Problems with parsing

enum TOKEN_TYPE{NUM,IDENT,COND,STRING};struct TokenStruct{    char delimiter;    std::string alpha;    std::string num;    std::string other;};class Scanner{    protected:        char m_delimiter;//defines the end of a line        std::string m_alpha;//letters in both Upper and Lower case        std::string m_num;//every numeric digit        std::string m_other;//everything else        TOKEN_TYPE token_type;//stores the type of token    public:        Scanner():m_delimiter(';'),m_alpha("_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"),m_num("0123456789"),m_other("#\"=+-*/.:!<>,'{}[]()"){}        Scanner(TokenStruct tokstr):m_delimiter(tokstr.delimiter),m_alpha(tokstr.alpha),m_num(tokstr.num),m_other(tokstr.other){}        ~Scanner(){}        //Note: the file pointer given to GetToken() must already be open.        std::string GetToken(std::ifstream& fin)        {            std::string token="";            char tempchar;            bool decimalpoint=false,whtspc=true;//decimalpoint is used to check if the token already has a decimal point. if it has, then the token is returned without the second decimal            //test if the file is actually open            if(!(fin.is_open()))                return std::string("file not open");            //get rid of whitespace            //do{                tempchar = fin.get();            //}while(isspace(tempchar)==0);            if(strcmp(&tempchar,""))                tempchar = fin.get();            //set token type            if(m_alpha.find(tempchar,0)!=std::string::npos)            {                token_type=IDENT;                token.append((const char*)tempchar);            }            else if(m_num.find(tempchar,0)!=std::string::npos)            {                token_type=NUM;            }            else if(m_other.find(tempchar,0)!=std::string::npos)            {                if(!strcmp((const char*)tempchar,"\""))                    token_type=STRING;                else                    token_type=COND;            }            else return std::string("token type not recognised");            return token;            //time to start reading in the token        }        std::vector<std::string> TokenizeLine(std::ifstream& fin)        {            std::vector<std::string> ret_vec;            std::string temp;            size_t line_size;            while(!(temp[0]==m_delimiter))            {                temp=GetToken(fin);            }            return ret_vec;        }};

#include "tokeniser.h"#include <fstream>using namespace std;int main(){    Scanner tokens;    ifstream fin;    fin.open("test.txt");    std::string myString;    std::vector<std::string> myVector;    myString=tokens.GetToken(fin);    //myVector=tokens.TokenizeLine(fin);    std::cout << myString << std::endl;    fin.close();}

#include <stdio.h>#include <conio.h>#include <ctype.h>#include <string.h>#ifndef NDEBUG    #define debug strcpy(argv[1],"errorlog.txt")#else    #define debug#endiftypedef enum{ident=0,num,string,punct,eof} type;/*    standard structure*/typedef struct{    type ttype;    char* value;    int size;} tok_type;int main(int argc,char** argv){    char            tempchar;    char*           token;    type            ttype;    int             size=0;    FILE            *in,*out;    debug;//used to set debug info    if(argc<2)        return 1;    else if(!(in = fopen(argv[1],"rt")))        return 1;    else if(!(out = fopen("out.tmp","wt+")))    {        fclose(in);        return 1;    }    while(!(feof(in)))    {        tempchar = fgetc(in);        if(isspace(tempchar))            continue;        if(size==0)        {            strcat(token,&tempchar);            size++;            if(tempchar=='"')                ttype=string;            else if(isalpha(tempchar))                ttype=ident;            else if(isdigit(tempchar))                ttype=num;            else if(ispunct(tempchar))                ttype=punct;            else                ttype=eof;        }        else        {            if(isspace(tempchar))                continue;            if(ttype == ident)            {                if(isalnum(tempchar)|| tempchar=='_')                {                    strcat(token,&tempchar);                    size++;                    continue;                }                else                {                    tok_type temp;                    temp.ttype=ttype;                    strcpy(temp.value,token);                    temp.size=size;                    fwrite(&temp,1,sizeof(temp),out);                    size=0;                    memset(&temp,0,sizeof(temp));                    ungetc(tempchar,in);                }            }            else if(ttype==num)            {                if(isdigit(tempchar))                {                    strcat(token,&tempchar);                    size++;                    continue;                }                else                {                    tok_type temp;                    temp.ttype=ttype;                    strcpy(temp.value,token);                    temp.size=size;                    fwrite(&temp,1,sizeof(temp),out);                    size=0;                    memset(&temp,0,sizeof(temp));                    ungetc(tempchar,in);                }            }            else if(ttype==string)            {                if(tempchar=='"')                {                    if(token[size-1]=='\\')                    {                        strcat(token,&tempchar);                        size++;                        continue;                    }                    else                    {                        tok_type temp;                        temp.ttype=ttype;                        strcpy(temp.value,token);                        temp.size=size;                        fwrite(&temp,1,sizeof(temp),out);                        size=0;                        memset(&temp,0,sizeof(temp));                        ungetc(tempchar,in);                    }                }                else                {                    strcat(token,&tempchar);                    size++;                    continue;                }            }        }    }    fclose(in);    fclose(out);    fflush(stdin);    return 0;}

#ifndef _LEX_H#define _LEX_H#include <iostream>#include <fstream>#include <string>//using operandsusing std::ifstream;using std::ofstream;using std::string;enum token_type{unknown=0,ident,number,punct,comment,strlit};//used by the lexer to work out how to act in different situationsstruct tok_type{    token_type token;    std::string value;};class str_test{    public:        bool    isAlpha(char);        bool    isNum(char);        bool    isSpace(char);        bool    isAlphaNum(char);        bool    isPunct(char);        bool    isUpper(char);        bool    isLower(char);        char    caseToggle(char);//NOTE: takes a single char and converts it to its alternate case(upper->lower,lower->upper)};//basic lexer classclass base_lexer{    public:        base_lexer(ifstream& _in):in(_in){}        ~base_lexer(){}        tok_type NextToken();//the main method of the lexer    protected:        char GetChar(){return in.get();}        void Unget(){in.unget();}        str_test str;        ifstream& in;};#endif

tok_type base_lexer::NextToken(){    static char temp=GetChar();    std::string token;    tok_type ret;    //ignore whitespace    if(str.isSpace(temp))    {        //use a do...while() loop so temp is always holding a new charcter        while(str.isSpace(temp))        {            temp=GetChar();        }    }    //is token a comment?    if(temp=='#')    {        while(temp!='\n'&&temp!='\r')        {            temp=GetChar();        }        temp=GetChar();    }    token.append(&temp);    //is the token an identifier?    if(str.isAlpha(temp) || temp=='_')    {        ret.token=ident;        while(!in.eof())        {            temp=GetChar();            if(str.isAlphaNum(temp) || temp=='_')            {                token.append(&temp);            }            else            {                ret.value=token;                return ret;            }        }    }    //is token a number?    else if(str.isNum(temp))    {        ret.token=number;        while(!in.eof())        {            temp=GetChar();            if(str.isNum(temp))            {                token.append(&temp);            }            else            {                ret.value=token;                return ret;            }        }    }    //is token a string const?    //NOTE: The string literal does not mark escape sequences. They are included    //into the body of the text as part of the literal string. The only    //reason special case code is needed is to allow the '"' character    else if(temp=='\"')    {        ret.token=strlit;        temp=GetChar();        token.append(&temp);        while(temp!='\"' && !in.eof())        {            if(temp=='\\')            {                temp=GetChar();                token.append(&temp);            }            temp=GetChar();            token.append(&temp);        }        ret.value=token;        temp=GetChar();        return ret;    }    //is token punctuation?    //NOTE: punctuation currently acts like numbers, in that the lexer keeps adding    //to the token for as long as the next char is punctuation.    //This will have to be changed to allow strings surrounded by other punctuation    else if(str.isPunct(temp))    {        ret.token=punct;        while(!in.eof())        {            temp=GetChar();            if(str.isPunct(temp))            {                token.append(&temp);            }            else            {                ret.value=token;                return ret;            }        }    }    //the token found is about as helpful as a caterer serving roadkill at a wedding...ToT    else    {        ret.token=unknown;        ret.value=temp;        return ret;    }}/**  * str_test:  *          The string testing class.  *  *          I thought of just making these functions wrappers of the functions  *          found in string.h and cstring, but as they are all single digits,  *          I can just test them quickly against ASCII values  */bool str_test::isAlpha(char c){    if((c>='A' && c<='Z') || (c>='a' && c<='z'))return true; else return false;}bool str_test::isNum(char c){    if(c>='0' && c<='9')return true;else return false;}bool str_test::isAlphaNum(char c){    if(isAlpha(c) || isNum(c))return true;else return false;}bool str_test::isLower(char c){    if(c>='a' && c<='z')return true;else return false;}bool str_test::isUpper(char c){    if(c>64 && c<91)return true; else return false;}bool str_test::isSpace(char c){    if(c==' ' || c=='\n' || c=='\r' || c=='\t')return true;else return false;}bool str_test::isPunct(char c){    if((c>32 && c<48) || (c>57 && c<65) || (c>90 && c<97) || (c>122 && c<127))return true;else return false;}char str_test::caseToggle(char c){    if(isUpper(c))return c+32;else if(isLower(c))return c-32;else return 0;}