/+
        Copyright (c) 1999-2006 by Digital Mars
        All Rights Reserved
        written by Walter Bright www.digitalmars.com
        License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.
        See the included readme.txt for details.
        D Language conversion by: J Duncan
 +/

/**
        d language lexer
 */

module dparser.Lexer;

import dparser.Root;

import dparser.Tokens;
import dparser.Token;
import dparser.Keyword;

import dparser.Types;

import dparser.Module;
import dparser.Identifier;
import dparser.unialpha;

import dparser.OutBuffer;

//private import std.ctype;
//private import std.string;
//import dwf.core.debugapi;

int errno = 0;

//#if _WIN32 && __DMC__
// from \dm\src\include\setlocal.h
//extern "C" char * __cdecl __locale_decpoint;
char *__locale_decpoint;

//#endif
//const uint LS = 0x2028;	// UTF line separator
//const uint PS = 0x2029;	// UTF paragraph separator

//extern int isUniAlpha(unsigned u);
//extern int HtmlNamedEntity(unsigned char *p, int length);

/**
        Lexer object
 */

class Lexer
{
    static           Identifier[char[]]       stringtable;
    static OutBuffer stringbuffer;
    static Token     *freelist;

    Token            token;        // current token
    Module           mod;          // current module
    Loc              loc;          // for error messages
    ubyte            *base;        // pointer to start of buffer
    ubyte            *end;         // past end of buffer
    ubyte            *p;           // current character
    int              doDocComment; // collect doc comment information
    int              anyToken;     // !=0 means seen at least one token
    int              commentToken; // !=0 means comments are TOKcomment's


    this(Module mod, ubyte *base, uint begoffset, uint endoffset, int doDocComment, int commentToken)
    {
        if (stringbuffer is null)
            stringbuffer = new OutBuffer;
        loc = Loc(mod, 1);

        this.base         = base;
        this.end          = base + endoffset;
        this.p            = base + begoffset;
        this.mod          = mod;
        this.doDocComment = doDocComment;
        this.commentToken = commentToken;

        /*
                If first line starts with '#!', ignore the line
         */

        if (p[0] == '#' && p[1] == '!')
        {
            p += 2;
            while (true)
            {
                ubyte c = *p;

                switch (c)
                {
                case '\n':
                    p++;
                    break;

                case '\r':
                    p++;
                    if (*p == '\n')
                        p++;
                    break;

                case 0:
                case 0x1A:
                    break;

                default:
                    if (c & 0x80)
                    {
                        uint u = decodeUTF();

                        if (u == PS || u == LS)
                            break;
                    }
                    p++;
                    continue;
                }
                break;
            }

            loc.linnum = 2;
        }
    }



    // generate a unique identifier for this string
    static Identifier idPool(in char[] str)
    {
//	    StringValue sv;
//	    uint len = s.length;
//	    StringValue sv = stringtable.update(s, len);
//	    Identifier* id = cast(Identifier*) sv.ptrvalue;
//	    if( id is null )
        if ((str in stringtable) == null)
        {
            stringtable[str] = new Identifier(str, TOK.TOKidentifier);
        }
        return stringtable[str];
    }

    static void initKeywords()
    {
        // build character map
        cmtable_init();

        // create keyword tokens & identifiers
        dparser.Keyword.initKeywords();

        // create standard lexer tokens
        dparser.Token.createLexerTokens();
    }

    // Combine two document comments into one.
    static char[] combineComments(char[] c1, char[] c2)
    {
        char[] c = c2;

        if (c1.length)
        {
            c = c1;
            if (c2.length)
            {
                c = c1 ~ "\n" ~ c2;
            }
        }
        return c;
    }

    // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.
    //! fix
    uint decodeUTF()
    {
        ubyte *s = p;
        ubyte c  = *s;

        assert(c & 0x80);
        if (!(c & 0x80))
            return c;

        return cast(uint)'X';
        /*
            dchar u;
            uint len;



            // Check length of remaining string up to 6 UTF-8 characters
            for( len = 1; len < 6 && s[len]; len++ )
            {

            }
                        /+
            uint idx = 0;
            char* msg = utf_decodeChar( s, len, &idx, &u );
            p += idx - 1;
            if( msg )
            {
                        error(msg);
            }
           +/
            return u;
         */
    }

    void error(...)
    {
        if ((mod !is null) && !global.gag)
        {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
               char[] p = loc.toChars();
               if( p.length )
                writef( "%s: ", p );
               writefx( stdout, _arguments, _argptr, 1 );
             */
            if (global.errors >= global.max_errors)                     // moderate blizzard of cascading messages
                throw new Exception("too many errors");
        }

        global.errors++;
    }

    void errorLoc(Loc loc, ...)
    {
        if ((mod !is null) && !global.gag)
        {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
               char[] p = loc.toChars();
               if( p.length )
                writef("%s: ", p);
               writefx(stdout, _arguments, _argptr, 1);
             */
            if (global.errors >= 20)                    // moderate blizzard of cascading messages
                throw new Exception("too many errors");
        }

        global.errors++;
    }


    TOK nextToken()
    {
        if (token.next)
        {
            Token *t = token.next;

            memcpy(&token, t, Token.sizeof);
//			t.next = freelist;
//			freelist = t;
        }
        else
        {
            scan(&token);
        }
//	    token.print();
        return token.value;
    }

    Token *peek(inout Token ct)
    {
        Token *t;

        if (ct.next)
            t = ct.next;
        else
        {
            t = new Token;
            scan(t);
            t.next  = null;
            ct.next = t;
        }
        return t;
    }

    // Turn next token in buffer into a token.

    void scan(Token *t)
    {
//		debug writefln("scan token");
        uint lastLine = loc.linnum;
        uint linnum;

        t.blockComment = null;
        t.lineComment  = null;
        while (true)
        {
            t.ptr = p;
//			debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );
            switch (*p)
            {
            case 0:
            case 0x1a:
                t.value = TOK.TOKeof;                                           // end of file
//					debug writefln( "    EOF" );
                return;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
//					debug writefln( "    whitespace" );
                continue;                                                               // skip white space

            case '\r':
//					debug writefln( "    cr" );
                p++;
                if (*p != '\n')                                                 // if CR stands by itself
                    loc.linnum++;
                continue;                                                       // skip white space

            case '\n':
//					debug writefln( "    nl" );
                p++;
                loc.linnum++;
                continue;                                                               // skip white space

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                t.value = number(t);
                return;

/*
   #if CSTRINGS
                            case '\'':
                                t.value = charConstant(t, 0);
                                return;

                            case '"':
                                t.value = stringConstant(t,0);
                                return;

                            case 'l':
                            case 'L':
                                if( p[1] == '\'')
                                {
                                    p++;
                                    t.value = charConstant(t, 1);
                                    return;
                                }
                                else if( p[1] == '"')
                                {
                                    p++;
                                    t.value = stringConstant(t, 1);
                                    return;
                                }
   #else
 */
            case '\'':
//					debug writefln( "    char" );
                t.value = charConstant(t, 0);
                return;

            case 'r':
//					debug writefln( "    wysiwyg" );
                if (p[1] != '"')
                    goto case_identifier;
                p++;

            case '`':
                t.value = wysiwygStringConstant(t, *p);
                return;

            case 'x':
//					debug writefln( "    hex string" );
                if (p[1] != '"')
                    goto case_identifier;
                p++;
                t.value = hexStringConstant(t);
                return;


            case '"':
//					debug writefln( "    string" );
                t.value = escapeStringConstant(t, 0);
//					debug writefln( t.ustring );
                return;

            case '\\':                                  // escaped string literal
//					debug writefln( "    escaped string literal" );
                uint c;

                stringbuffer.offset = 0;
                do
                {
                    p++;
                    c = escapeSequence();
                    stringbuffer.write(c);
                } while (*p == '\\');
//					t.len = stringbuffer.offset;
//					stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//					memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );
                t.postfix = 0;
                t.value   = TOK.TOKstring;
                return;

            case 'l':
            case 'L':
//	#endif

            case 'a':
            case 'b':
            case 'c':
            case 'd':
            case 'e':
            case 'f':
            case 'g':
            case 'h':
            case 'i':
            case 'j':
            case 'k':
            case 'm':
            case 'n':
            case 'o':
            case 'p':
            case 'q':                             /*case 'r':*/
            case 's':
            case 't':
            case 'u':
            case 'v':
            case 'w':                                         /*case 'x':*/
            case 'y':
            case 'z':
            case 'A':
            case 'B':
            case 'C':
            case 'D':
            case 'E':
            case 'F':
            case 'G':
            case 'H':
            case 'I':
            case 'J':
            case 'K':
            case 'M':
            case 'N':
            case 'O':
            case 'P':
            case 'Q':
            case 'R':
            case 'S':
            case 'T':
            case 'U':
            case 'V':
            case 'W':
            case 'X':
            case 'Y':
            case 'Z':
            case '_':
 case_identifier:
                {
//					debug writefln( "    identifier" );
                    ubyte c;

                    do
                    {
                        c = *++p;
                    } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF())));

//					sv = stringtable.update((char *)t.ptr, p - t.ptr);
                    char[] tmp;

                    tmp.length = p - t.ptr;
                    memcpy(tmp.ptr, t.ptr, p - t.ptr);
                    Identifier id;
                    Identifier *pid = tmp in stringtable;

                    if (pid)
                    {
                        id = *pid;
                    }

                    if (id is null)
                    {
                        id               = new Identifier(tmp, TOK.TOKidentifier);
                        stringtable[tmp] = id;
                    }

                    t.identifier = id;
                    t.value      = cast(TOK)id.value;
                    anyToken     = 1;

                    // if special identifier token
                    if (*t.ptr == '_')
                    {
                        static char date[11 + 1];
                        static char time[8 + 1];
                        static char timestamp[24 + 1];

                        if (!date[0])                           // lazy evaluation
                        {
                            //!!
                            /+
                                    time_t t;
                                    char *p;
                                    .time(&t);
                                    p = ctime(&t);
                                    assert(p);
                                    sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);
                                    sprintf(time.ptr, "%.8s", p + 11);
                                    sprintf(timestamp.ptr, "%.24s", p);
                             +/
                        }

                        if (mod && id is Id.FILE)
                        {
                            t.value = TOK.TOKstring;
                            if (loc.filename.length)
                                t.ustring = loc.filename;
                            else
                                t.ustring = mod.identifier.toChars();
                            goto Llen;
                        }
                        else if (mod && id == Id.LINE)
                        {
                            t.value      = TOK.TOKint64v;
                            t.uns64value = loc.linnum;
                        }
                        else if (id == Id.DATE)
                        {
                            t.value = TOK.TOKstring;
                            //! t.ustring = date;
                            goto Llen;
                        }
                        else if (id == Id.TIME)
                        {
                            t.value = TOK.TOKstring;
                            //! t.ustring = time;
                            goto Llen;
                        }
                        else if (id == Id.TIMESTAMP)
                        {
                            t.value = TOK.TOKstring;
                            //! t.ustring = timestamp;
 Llen:
                            t.postfix = 0;
//							t.len = strlen((char *)t.ustring);
                        }
                    }
                    //printf("t.value = %d\n",t.value);
                    return;
                }

            // comments
            case '/':
                p++;
                switch (*p)
                {
                case '=':
                    p++;
                    t.value = TOK.TOKdivass;
                    return;

                case '*':                               // '/*'
                    p++;
                    linnum = loc.linnum;
                    while (true)
                    {
                        while (true)
                        {
                            ubyte c = *p;

                            switch (c)
                            {
                            case '/':
                                break;

                            case '\n':
                                loc.linnum++;
                                p++;
                                continue;

                            case '\r':
                                p++;
                                if (*p != '\n')
                                    loc.linnum++;
                                continue;

                            case 0:
                            case 0x1A:
                                error("unterminated /* */ comment");
                                p       = end;
                                t.value = TOK.TOKeof;
                                return;

                            default:
                                if (c & 0x80)
                                {
                                    uint u = decodeUTF();

                                    if (u == PS || u == LS)
                                        loc.linnum++;
                                }
                                p++;
                                continue;
                            }
                            break;
                        }
                        p++;
                        if (p[-2] == '*' && p - 3 != t.ptr)
                            break;
                    }

                    if (commentToken)
                    {
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    // if /** but not /**/
                    else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr)
                        getDocComment(t, lastLine == linnum);                                           //! ?
                    continue;

                case '/':                                       // do // style comments
                    linnum = loc.linnum;
                    while (1)
                    {
                        ubyte c = *++p;

                        switch (c)
                        {
                        case '\n':
                            break;

                        case '\r':
                            if (p[1] == '\n')
                                p++;
                            break;

                        case 0:
                        case 0x1a:
                            if (commentToken)
                            {
                                p       = end;
                                t.value = TOK.TOKcomment;
                                return;
                            }
                            if (doDocComment && t.ptr[2] == '/')
                                getDocComment(t, lastLine == linnum);
                            p       = end;
                            t.value = TOK.TOKeof;
                            return;

                        default:
                            if (c & 0x80)
                            {
                                uint u = decodeUTF();

                                if (u == PS || u == LS)
                                    break;
                            }
                            continue;
                        }
                        break;
                    }

                    if (commentToken)
                    {
                        p++;
                        loc.linnum++;
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    if (doDocComment && t.ptr[2] == '/')
                        getDocComment(t, lastLine == linnum);

                    p++;
                    loc.linnum++;
                    continue;

                case '+':
                    {
                        int nest;

                        linnum = loc.linnum;
                        p++;
                        nest = 1;
                        while (1)
                        {
                            ubyte c = *p;

                            switch (c)
                            {
                            case '/':
                                p++;
                                if (*p == '+')
                                {
                                    p++;
                                    nest++;
                                }
                                continue;

                            case '+':
                                p++;
                                if (*p == '/')
                                {
                                    p++;
                                    if (--nest == 0)
                                        break;
                                }
                                continue;

                            case '\r':
                                p++;
                                if (*p != '\n')
                                    loc.linnum++;
                                continue;

                            case '\n':
                                loc.linnum++;
                                p++;
                                continue;

                            case 0:
                            case 0x1A:
                                error("unterminated /+ +/ comment");
                                p       = end;
                                t.value = TOK.TOKeof;
                                return;

                            default:
                                if (c & 0x80)
                                {
                                    uint u = decodeUTF();

                                    if (u == PS || u == LS)
                                        loc.linnum++;
                                }
                                p++;
                                continue;
                            }
                            break;
                        }
                        if (commentToken)
                        {
                            t.value = TOK.TOKcomment;
                            return;
                        }
                        if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr)
                        {
                            // if /++ but not /++/
                            getDocComment(t, lastLine == linnum);
                        }
                        continue;
                    }

                default:
                    break;
                }
                t.value = TOK.TOKdiv;
                return;

            case '.':
                p++;
                if (isdigit(*p))
                {
                    p--;
                    t.value = inreal(t);
                }
                else if (p[0] == '.')
                {
                    if (p[1] == '.')
                    {
                        p      += 2;
                        t.value = TOK.TOKdotdotdot;
                    }
                    else
                    {
                        p++;
                        t.value = TOK.TOKslice;
                    }
                }
                else
                    t.value = TOK.TOKdot;
                return;

            case '&':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKandass;
                }
                else if (*p == '&')
                {
                    p++;
                    t.value = TOK.TOKandand;
                }
                else
                    t.value = TOK.TOKand;
                return;

            // |, ||, |=
            case '|':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKorass;
                }
                else if (*p == '|')
                {
                    p++;
                    t.value = TOK.TOKoror;
                }
                else
                    t.value = TOK.TOKor;
                return;

            case '-':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKminass;
                }
                else if (*p == '-')
                {
                    p++;
                    t.value = TOK.TOKminusminus;
                }
                else
                    t.value = TOK.TOKmin;
                return;

            // +, +=, ++
            case '+':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKaddass;                                    // +=
                }
                else if (*p == '+')
                {
                    p++;
                    t.value = TOK.TOKplusplus;                                          // ++
                }
                else
                    t.value = TOK.TOKadd;                                                       // +
                return;

            // <, <=, <<=, <<, <>=, <>
            case '<':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKle;                                                // <=
                }
                else if (*p == '<')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKshlass;                                        // <<=
                    }
                    else
                        t.value = TOK.TOKshl;                                           // <<
                }
                else if (*p == '>')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKleg;                                           // <>=
                    }
                    else
                        t.value = TOK.TOKlg;                                            // <>
                }
                else
                    t.value = TOK.TOKlt;                                                // <
                return;

            // >, >>, >>>, >=, >>=, >>>=
            case '>':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKge;                                                // >=
                }
                else if (*p == '>')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKshrass;                                        // >>=
                    }
                    else if (*p == '>')
                    {
                        p++;
                        if (*p == '=')
                        {
                            p++;
                            t.value = TOK.TOKushrass;                           // >>>=
                        }
                        else
                            t.value = TOK.TOKushr;                                      // >>>
                    }
                    else
                        t.value = TOK.TOKshr;                                           // >>
                }
                else
                    t.value = TOK.TOKgt;                                                // >
                return;

            case '!':
                p++;
                if (*p == '=')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKnotidentity;                                   // !==
                    }
                    else
                        t.value = TOK.TOKnotequal;                                      // !=
                }
                else if (*p == '<')
                {
                    p++;
                    if (*p == '>')
                    {
                        p++;
                        if (*p == '=')
                        {
                            p++;
                            t.value = TOK.TOKunord;                             // !<>=
                        }
                        else
                            t.value = TOK.TOKue;                                // !<>
                    }
                    else if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKug;                                    // !<=
                    }
                    else
                        t.value = TOK.TOKuge;                                   // !<
                }
                else if (*p == '>')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKul;                                    // !>=
                    }
                    else
                        t.value = TOK.TOKule;                                   // !>
                }
                else
                    t.value = TOK.TOKnot;                                       // !
                return;

            case '=':
                p++;
                if (*p == '=')
                {
                    p++;
                    if (*p == '=')
                    {
                        p++;
                        t.value = TOK.TOKidentity;                                      // ===
                    }
                    else
                        t.value = TOK.TOKequal;                                         // ==
                }
                else
                    t.value = TOK.TOKassign;                                    // =
                return;

            case '~':
                p++;
                if (*p == '=')
                {
                    p++;
                    t.value = TOK.TOKcatass;                                            // ~=
                }
                else
                    t.value = TOK.TOKtilde;                                             // ~
                return;

            // SINGLE
            case '(': p++; t.value = TOK.TOKlparen;     return;

            case ')': p++; t.value = TOK.TOKrparen;     return;

            case '[': p++; t.value = TOK.TOKlbracket;   return;

            case ']': p++; t.value = TOK.TOKrbracket;   return;

            case '{': p++; t.value = TOK.TOKlcurly;     return;

            case '}': p++; t.value = TOK.TOKrcurly;     return;

            case '?': p++; t.value = TOK.TOKquestion;   return;

            case ',': p++; t.value = TOK.TOKcomma;              return;

            case ';': p++; t.value = TOK.TOKsemicolon;  return;

            case ':': p++; t.value = TOK.TOKcolon;              return;

            case '$': p++; t.value = TOK.TOKdollar;     return;

            // DOUBLE
            case '*': p++; if (*p == '=')
                {
                    p++; t.value = TOK.TOKmulass;
                }
                else
                    t.value = TOK.TOKmul;
                return;

            case '%': p++; if (*p == '=')
                {
                    p++; t.value = TOK.TOKmodass;
                }
                else
                    t.value = TOK.TOKmod;
                return;

            case '^': p++; if (*p == '=')
                {
                    p++; t.value = TOK.TOKxorass;
                }
                else
                    t.value = TOK.TOKxor;
                return;

// removed 148	case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;


            case '#':
                p++;
                Pragma();
                continue;

            default:
                {
                    debug writefln("    default char");
                    ubyte c = *p;

                    if (c & 0x80)
                    {
                        uint u = decodeUTF();

                        // Check for start of unicode identifier
                        if (isUniAlpha(u))
                            goto case_identifier;

                        if (u == PS || u == LS)
                        {
                            loc.linnum++;
                            p++;
                            continue;
                        }
                    }
                    if (isprint(c))
                        error("unsupported char '%s'", cast(char)c);
                    else
                        error("unsupported char 0x%02x", cast(ubyte)c);
                    p++;
                    continue;
                }
            }
        }
    }



    // Parse escape sequence.
    uint escapeSequence()
    {
        uint c;
        int  n;
        int  ndigits;

        c = *p;
        switch (c)
        {
        case '\'':
        case '"':
        case '?':
        case '\\':
 Lconsume:
            p++;
            break;

        case 'a':       c = 7;          goto Lconsume;

        case 'b':       c = 8;          goto Lconsume;

        case 'f':       c = 12;         goto Lconsume;

        case 'n':       c = 10;         goto Lconsume;

        case 'r':       c = 13;         goto Lconsume;

        case 't':       c = 9;          goto Lconsume;

        case 'v':       c = 11;         goto Lconsume;

        case 'u':
            ndigits = 4;
            goto Lhex;

        case 'U':
            ndigits = 8;
            goto Lhex;

        case 'x':
            ndigits = 2;
 Lhex:
            p++;
            c = *p;
            if (ishex(c))
            {
                uint v;

                n = 0;
                v = 0;
                while (1)
                {
                    if (isdigit(c))
                        c -= '0';
                    else if (islower(c))
                        c -= 'a' - 10;
                    else
                        c -= 'A' - 10;
                    v = v * 16 + c;
                    c = *++p;
                    if (++n == ndigits)
                        break;
                    if (!ishex(c))
                    {
                        error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
                        break;
                    }
                }
//!				    if( ndigits != 2 && !utf_isValidDchar(v))
//!						error("invalid UTF character \\U%08x", v);
                c = v;
            }
            else
                error("undefined escape hex sequence \\%s\n", c);
            break;

        case '&':                                       // named character entity
            for (ubyte *idstart = ++p; 1; p++)
            {
                switch (*p)
                {
                case ';':
                    //!!!
                    /+
                       c = HtmlNamedEntity(idstart, p - idstart);
                       if( c == ~0 )
                       {
                            error("unnamed character entity &%.*s;", p - idstart, idstart);
                            c = ' ';
                       }

                       p++;
                     +/
                    break;

                default:
                    if (isalpha(*p) || (p != idstart + 1 && isdigit(*p)))
                        continue;
                    error("unterminated named entity");
                    break;
                }
                break;
            }
            break;

        case 0:
        case 0x1a:                                      // end of file
            c = '\\';
            break;

        default:
            if (isoctal(c))
            {
                ubyte v;

                n = 0;
                do
                {
                    v = v * 8 + (c - '0');
                    c = *++p;
                } while (++n < 3 && isoctal(c));
                c = v;
            }
            else
                error("undefined escape sequence \\%s\n", c);
            break;
        }
        return c;
    }

    /**************************************
     */

    TOK wysiwygStringConstant(Token *t, int tc)
    {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        while (1)
        {
            c = *p++;
            switch (c)
            {
            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n')
                    continue;                           // ignore
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return TOK.TOKstring;

            case '"':
            case '`':
                if (c == tc)
                {
//				    t.len = stringbuffer.offset;
                    stringbuffer.write(cast(byte)0);
                    t.ustring = stringbuffer.toString;
//				    t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//				    memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                    stringPostfix(t);
                    return TOK.TOKstring;
                }
                break;

            default:
                if (c & 0x80)
                {
                    p--;
                    uint u = decodeUTF();

                    p++;
                    if (u == PS || u == LS)
                        loc.linnum++;
                    stringbuffer.write(u);
                    continue;
                }
                break;
            }
            stringbuffer.write(c);
        }
    }

    /**************************************
     * Lex hex strings:
     *	x"0A ae 34FE BD"
     */

    TOK hexStringConstant(Token *t)
    {
        uint c;
        Loc  start = loc;
        uint n     = 0;
        uint v;

        p++;
        stringbuffer.offset = 0;
        while (1)
        {
            c = *p++;
            switch (c)
            {
            case ' ':
            case '\t':
            case '\v':
            case '\f':
                continue;                                               // skip white space

            case '\r':
                if (*p == '\n')
                    continue;                                           // ignore

            // Treat isolated '\r' as if it were a '\n'
            case '\n':
                loc.linnum++;
                continue;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return TOK.TOKstring;

            case '"':
                if (n & 1)
                {
                    error("odd number (%d) of hex characters in hex string", n);
                    stringbuffer.write(v);
                }
//				t.len = stringbuffer.offset;
//				stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);
                return TOK.TOKstring;

            default:
                if (c >= '0' && c <= '9')
                    c -= '0';
                else if (c >= 'a' && c <= 'f')
                    c -= 'a' - 10;
                else if (c >= 'A' && c <= 'F')
                    c -= 'A' - 10;
                else if (c & 0x80)
                {
                    p--;
                    uint u = decodeUTF();

                    p++;
                    if (u == PS || u == LS)
                        loc.linnum++;
                    else
                        error("non-hex character \\u%x", u);
                }
                else
                    error("non-hex character '%s'", c);
                if (n & 1)
                {
                    v = (v << 4) | c;
                    stringbuffer.write(v);
                }
                else
                    v = c;
                n++;
                break;
            }
        }
    }

    /**************************************
     */

    TOK escapeStringConstant(Token *t, int wide)
    {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        //    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );
        while (1)
        {
            c = *p++;
            switch (c)
            {
            case '\\':
                switch (*p)
                {
                case 'u':
                case 'U':
                case '&':
                    c = escapeSequence();
                    stringbuffer.write(c);
                    continue;

                default:
                    c = escapeSequence();
                    break;
                }
                break;

            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n')
                    continue;                           // ignore
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case '"':
//			        writefln( "end of string: ", stringbuffer.toString );
                t.ustring = stringbuffer.toString().dup;
                //				t.len = stringbuffer.offset;
                //				stringbuffer.write(cast(byte)0);
                //				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
                //				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);

                return TOK.TOKstring;

            case 0:
            case 0x1a:
                p--;
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
//					t.len = 0;
                t.postfix = 0;
                return TOK.TOKstring;

            default:
                if (c & 0x80)
                {
                    p--;
                    c = decodeUTF();
                    if (c == LS || c == PS)
                    {
                        c = '\n';
                        loc.linnum++;
                    }
                    p++;
                    stringbuffer.write(cast(char)c);
                    continue;
                }
                break;
            }
            stringbuffer.write(cast(char)c);
//			writefln( stringbuffer.toString );
        }
    }

    //**************************************
    TOK charConstant(Token *t, int wide)
    {
        uint c;
        TOK  tk = TOK.TOKcharv;

        //printf("Lexer.charConstant\n");
        p++;
        c = *p++;
        switch (c)
        {
        case '\\':
            switch (*p)
            {
            case 'u':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKwcharv;
                break;

            case 'U':
            case '&':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKdcharv;
                break;

            default:
                t.uns64value = escapeSequence();
                break;
            }
            break;

        case '\n':
 L1:
            loc.linnum++;

        case '\r':
        case 0:
        case 0x1a:
        case '\'':
            error("unterminated character constant");
            return tk;

        default:
            if (c & 0x80)
            {
                p--;
                c = decodeUTF();
                p++;
                if (c == LS || c == PS)
                    goto L1;
                if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe))
                    tk = TOK.TOKwcharv;
                else
                    tk = TOK.TOKdcharv;
            }
            t.uns64value = c;
            break;
        }

        if (*p != '\'')
        {
            error("unterminated character constant");
            return tk;
        }
        p++;
        return tk;
    }

    // Get postfix of string literal.
    void stringPostfix(Token *t)
    {
        switch (*p)
        {
        case 'c':
        case 'w':
        case 'd':
            t.postfix = *p;
            p++;
            break;

        default:
            t.postfix = 0;
            break;
        }
    }

    /***************************************
     * Read \u or \U unicode sequence
     * Input:
     *	u	'u' or 'U'
     */
    /*
       uint Wchar(uint u)
       {
        uint value;
        uint n;
        ubyte c;
        uint nchars;

        nchars = (u == 'U') ? 8 : 4;
        value = 0;
        for (n = 0; 1; n++)
        {
       ++p;
                    if( n == nchars)
                        break;
                    c = *p;
                    if( !ishex(c))
                    {
                            error("\\%s sequence must be followed by %d hex characters", u, nchars);
                        break;
                    }
                    if( isdigit(c))
                        c -= '0';
                    else if( islower(c))
                        c -= 'a' - 10;
                    else
                        c -= 'A' - 10;
                    value <<= 4;
                    value |= c;
        }
        return value;
       }
     */

    /**************************************
     * Read in a number.
     * If it's an integer, store it in tok.TKutok.Vlong.
     *	integers can be decimal, octal or hex
     *	Handle the suffixes U, UL, LU, L, etc.
     * If it's double, store it in tok.TKutok.Vdouble.
     * Returns:
     *	TKnum
     *	TKdouble,...
     */

    TOK number(Token *t)
    {
        //debug writefln("Lexer.number()");
        // We use a state machine to collect numbers
        enum STATE
        {
            STATE_initial,
            STATE_0,
            STATE_decimal,
            STATE_octal,
            STATE_octale,
            STATE_hex,
            STATE_binary,
            STATE_hex0,
            STATE_binary0,
            STATE_hexh,
            STATE_error
        }

        enum FLAGS
        {
            FLAGS_decimal  = 1,                         // decimal
            FLAGS_unsigned = 2,                         // u or U suffix
            FLAGS_long     = 4,                         // l or L suffix
        }
        FLAGS flags = FLAGS.FLAGS_decimal;

        int   i;
        TOK   result;
        int   base;

        stringbuffer.offset = 0;
//		stringbuffer.data = null;
        STATE state  = STATE.STATE_initial;
        ubyte *start = p;

        TOK _isreal()
        {
            p = start;
            return inreal(t);
        }

        while (true)
        {
            char c = cast(char)*p;

            switch (state)
            {
            case STATE.STATE_initial:                           // opening state
                if (c == '0')
                    state = STATE.STATE_0;
                else
                    state = STATE.STATE_decimal;
                break;

            case STATE.STATE_0:
                flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal);
                switch (c)
                {
                //	#if ZEROH
//					    case 'H':			// 0h
//					    case 'h':
//							goto hexh;
                //	#endif
                case 'X':
                case 'x':
                    state = STATE.STATE_hex0;
                    break;

                case '.':
                    if (p[1] == '.')                                            // .. is a separate token
                        goto done;

                case 'i':
                case 'f':
                case 'F':
                    goto _Real;

                //	#if ZEROH
//					    case 'E':
//					    case 'e':
//							goto case_hex;
                //	#endif
                case 'B':
                case 'b':
                    state = STATE.STATE_binary0;
                    break;

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                    state = STATE.STATE_octal;
                    break;

                //	#if ZEROH
//					    case '8': case '9': case 'A':
//					    case 'C': case 'D': case 'F':
//					    case 'a': case 'c': case 'd': case 'f':
//						    case_hex:
//							state = STATE.STATE_hexh;
//							break;
                //	#endif
                case '_':
                    state = STATE.STATE_octal;
                    p++;
                    continue;

                default:
                    goto done;
                }
                break;

            case STATE.STATE_decimal:                           // reading decimal number

                // if its not a digit - decimal complete or not a decimal
                if (!isdigit(c))
                {
//						debug writefln( "\tnon-digit( %s )", c );
                    //	#if ZEROH
//					    if( ishex(c) || c == 'H' || c == 'h' )
//							goto hexh;
                    //	#endif
                    //! wtf ?
                    // ignore embedded _
                    if (c == '_')
                    {
                        p++;
                        continue;
                    }

                    // check decimal point - make real
                    if (c == '.' && p[1] != '.')
                        goto _Real;

                    // check for mantra - make real
                    if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E')
                    {
 _Real:                 // It's a real number. Back up and rescan as a real
                        p = start;
                        return inreal(t);
                    }

                    goto done;
                }
                break;

            case STATE.STATE_hex0:                              // reading hex number
            case STATE.STATE_hex:
                if (!ishex(c))
                {
                    if (c == '_')                                       // ignore embedded _
                    {
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.')
                        goto _Real;
                    if (c == 'P' || c == 'p' || c == 'i')
                        goto _Real;
                    if (state == STATE.STATE_hex0)
                        error("Hex digit expected, not '%s'", c);
                    goto done;
                }
                state = STATE.STATE_hex;
                break;

            //	#if ZEROH
//				    hexh:
//						state = STATE.STATE_hexh;
//
//				    case STATE.STATE_hexh:		// parse numbers like 0FFh
//						if( !ishex(c))
//						{
//						    if( c == 'H' || c == 'h')
//						    {
//								p++;
//								base = 16;
//								goto done;
//						    }
//						    else
//						    {
//								// Check for something like 1E3 or 0E24
//								if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))
//								    goto _Real;
//								error("Hex digit expected, not '%s'", c);
//								goto done;
//						    }
//						}
//						break;
            //		#endif

            case STATE.STATE_octal:                                     // reading octal number
            case STATE.STATE_octale:                                    // reading octal number with non-octal digits
                if (!isoctal(c))
                {
//				#if ZEROH
//						    if( ishex(c) || c == 'H' || c == 'h' )
//								goto hexh;
//				#endif
                    if (c == '_')                                               // ignore embedded _
                    {
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.')
                        goto _Real;
                    if (c == 'i')
                        goto _Real;
                    if (isdigit(c))
                        state = STATE.STATE_octale;
                    else
                        goto done;
                }
                break;

            case STATE.STATE_binary0:                                   // starting binary number
            case STATE.STATE_binary:                                    // reading binary number
                if (c != '0' && c != '1')
                {
                    //			#if ZEROH
//						    if( ishex(c) || c == 'H' || c == 'h' )
//								goto hexh;
                    //			#endif
                    if (c == '_')                                               // ignore embedded _
                    {
                        p++;
                        continue;
                    }
                    if (state == STATE.STATE_binary0)
                    {
                        error("binary digit expected");
                        state = STATE.STATE_error;
                        break;
                    }
                    else
                        goto done;
                }
                state = STATE.STATE_binary;
                break;

            case STATE.STATE_error:                                     // for error recovery
                if (!isdigit(c))                                        // scan until non-digit
                    goto done;
                break;

            default:
                assert(0);
            }
            stringbuffer.write(cast(char)c);
            p++;
        }
 done:
        stringbuffer.write(cast(char)0);                // terminate string

//		debug writefln( "\tdigit complete( %s )", stringbuffer.toString );

        if (state == STATE.STATE_octale)
            error("Octal digit expected");

        uinteger_t n;                           // unsigned >=64 bit integer type

        if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0))
            n = stringbuffer.data[0] - '0';
        else
        {
            // Convert string to integer
            char *p = cast(char *)stringbuffer.data.ptr;
            int  r  = 10;
            int  d;

            if (*p == '0')
            {
                if (p[1] == 'x' || p[1] == 'X')
                {
                    // "0x#"
                    p += 2;
                    r  = 16;
                }
                else if (p[1] == 'b' || p[1] == 'B')
                {
                    // "0b#" - binary
                    p += 2;
                    r  = 2;
                }
                else if (isdigit(p[1]))
                {
                    p += 1;
                    r  = 8;
                }
            }

            n = 0;

            while (true)
            {
                if (*p >= '0' && *p <= '9')
                    d = *p - '0';
                else if (*p >= 'a' && *p <= 'z')
                    d = *p - 'a' + 10;
                else if (*p >= 'A' && *p <= 'Z')
                    d = *p - 'A' + 10;
                else
                    break;

                if (d >= r)
                    break;

                if (n * r + d < n)
                {
                    error("integer overflow");
                    break;
                }

                n = n * r + d;
                p++;
            }

            // if n needs more than 64 bits
            if (n.sizeof > 8 && n > 0xffffffffffffffffL)
                error("integer overflow");
        }

        // Parse trailing 'u', 'U', 'l' or 'L' in any combination
        while (true)
        {
            ubyte f;

            switch (*p)
            {
            case 'U':
            case 'u':
                f = FLAGS.FLAGS_unsigned;
                goto L1;

            case 'L':
            case 'l':
                f = FLAGS.FLAGS_long;
 L1:
                p++;
                if (flags & f)
                    error("unrecognized token");
                flags = cast(FLAGS)(flags | f);
                continue;

            default:
                break;
            }
            break;
        }

        switch (flags)
        {
        case 0:
            /* Octal or Hexadecimal constant.
             * First that fits: int, uint, long, ulong
             */
            if (n & 0x8000000000000000L)
                result = TOK.TOKuns64v;
            else if (n & 0xffffffff00000000L)
                result = TOK.TOKint64v;
            else if (n & 0x80000000)
                result = TOK.TOKuns32v;
            else
                result = TOK.TOKint32v;
            break;

        case FLAGS.FLAGS_decimal:
            /* First that fits: int, long, long long
             */
            if (n & 0x8000000000000000L)
            {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else if (n & 0xffffffff80000000L)
                result = TOK.TOKint64v;
            else
                result = TOK.TOKint32v;
            break;

        case FLAGS.FLAGS_unsigned:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:
            /* First that fits: uint, ulong
             */
            if (n & 0xffffffff00000000L)
                result = TOK.TOKuns64v;
            else
                result = TOK.TOKuns32v;
            break;

        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L)
            {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else
                result = TOK.TOKint64v;
            break;

        case FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L)
                result = TOK.TOKuns64v;
            else
                result = TOK.TOKint64v;
            break;

        case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
            result = TOK.TOKuns64v;
            break;

        default:
            debug writefln("%x", flags);
            assert(0);
        }
        t.uns64value = n;
        return result;
    }

    /**************************************
     * Read in characters, converting them to real.
     * Bugs:
     *	Exponent overflow not detected.
     *	Too much requested precision is not detected.
     */

    TOK inreal(Token *t)
    {
        int  dblstate;
        uint c;
        char hex;                       // is this a hexadecimal-floating-constant?
        TOK  result;

        //printf("Lexer.inreal()\n");
        stringbuffer.offset = 0;
        dblstate            = 0;
        hex                 = 0;
 Lnext:
        while (1)
        {
            // Get next char from input
            c = *p++;
            //printf("dblstate = %d, c = '%s'\n", dblstate, c);
            while (1)
            {
                switch (dblstate)
                {
                case 0:                                 // opening state
                    if (c == '0')
                        dblstate = 9;
                    else if (c == '.')
                        dblstate = 3;
                    else
                        dblstate = 1;
                    break;

                case 9:
                    dblstate = 1;
                    if (c == 'X' || c == 'x')
                    {
                        hex++;
                        break;
                    }

                case 1:                                 // digits to left of .
                case 3:                                 // digits to right of .
                case 7:                                 // continuing exponent digits
                    if (!isdigit(c) && !(hex && isxdigit(c)))
                    {
                        if (c == '_')
                            goto Lnext;                         // ignore embedded '_'
                        dblstate++;
                        continue;
                    }
                    break;

                case 2:                                 // no more digits to left of .
                    if (c == '.')
                    {
                        dblstate++;
                        break;
                    }

                case 4:                                 // no more digits to right of .
                    if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p'))
                    {
                        dblstate = 5;
                        hex      = 0;                           // exponent is always decimal
                        break;
                    }
                    if (hex)
                        error("binary-exponent-part required");
                    goto done;

                case 5:                                 // looking immediately to right of E
                    dblstate++;
                    if (c == '-' || c == '+')
                        break;

                case 6:                                 // 1st exponent digit expected
                    if (!isdigit(c))
                        error("exponent expected");
                    dblstate++;
                    break;

                case 8:                                 // past end of exponent digits
                    goto done;
                }
                break;
            }
            stringbuffer.write(c);
        }
 done:
        p--;

        stringbuffer.write(cast(byte)0);

//	#if _WIN32 && __DMC__
        char *save = __locale_decpoint;

        __locale_decpoint = ".";
//	#endif
        t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);
        errno          = 0;
        switch (*p)
        {
        case 'F':
        case 'f':
            strtof(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat32v;
            p++;
            break;

        default:
            strtod(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat64v;
            break;

        case 'L':
        case 'l':
            result = TOK.TOKfloat80v;
            p++;
            break;
        }
        if (*p == 'i' || *p == 'I')
        {
            p++;
            switch (result)
            {
            case TOK.TOKfloat32v:
                result = TOK.TOKimaginary32v;
                break;

            case TOK.TOKfloat64v:
                result = TOK.TOKimaginary64v;
                break;

            case TOK.TOKfloat80v:
                result = TOK.TOKimaginary80v;
                break;
            }
        }
//	#if _WIN32 && __DMC__
        __locale_decpoint = save;
//	#endif
        if (errno == ERANGE)
            error("number is not representable");
        return result;
    }




    /*********************************************
     * Do pragma.
     * Currently, the only pragma supported is:
     *	#line linnum [filespec]
     */

    void Pragma()
    {
        Token  tok;
        int    linnum;
        char[] filespec;
        Loc    loc = this.loc;

        scan(&tok);

        if (tok.value != TOK.TOKidentifier || tok.identifier != Id.line)
            goto Lerr;

        scan(&tok);
        if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v)
            linnum = tok.uns64value - 1;
        else
            goto Lerr;

        while (1)
        {
            switch (*p)
            {
            case 0:
            case 0x1a:
            case '\n':
 Lnewline:
                this.loc.linnum = linnum;
                if (filespec.length)
                    this.loc.filename = filespec;
                return;

            case '\r':
                p++;
                if (*p != '\n')
                {
                    p--;
                    goto Lnewline;
                }
                continue;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
                continue;                                               // skip white space

            case '_':
                if (mod && memcmp(p, cast(char *)"__FILE__", 8) == 0)
                {
                    p += 8;
//!					    filespec = mem.strdup(loc.filename ? loc.filename : mod.identifier.toChars());
                }
                continue;

            case '"':
                if (filespec)
                    goto Lerr;
                stringbuffer.offset = 0;
                p++;
                while (1)
                {
                    uint c;

                    c = *p;
                    switch (c)
                    {
                    case '\n':
                    case '\r':
                    case 0:
                    case 0x1a:
                        goto Lerr;

                    case '"':
                        stringbuffer.write(cast(byte)0);
                        //						    filespec = mem.strdup((char *)stringbuffer.data);
                        filespec = stringbuffer.toString.dup;
                        p++;
                        break;

                    default:
                        if (c & 0x80)
                        {
                            uint u = decodeUTF();

                            if (u == PS || u == LS)
                                goto Lerr;
                        }
                        stringbuffer.write(c);
                        p++;
                        continue;
                    }
                    break;
                }
                continue;

            default:
                if (*p & 0x80)
                {
                    uint u = decodeUTF();

                    if (u == PS || u == LS)
                        goto Lnewline;
                }
                goto Lerr;
            }
        }

 Lerr:
        errorLoc(loc, "#line integer [\"filespec\"]\\n expected");
    }



    /***************************************************
     * Parse doc comment embedded between t.ptr and p.
     * Remove trailing blanks and tabs from lines.
     * Replace all newlines with \n.
     * Remove leading comment character from each line.
     * Decide if it's a lineComment or a blockComment.
     * Append to previous one for this token.
     */

    void getDocComment(Token *t, uint lineComment)
    {
        auto OutBuffer buf       = new OutBuffer;
        ubyte          ct        = t.ptr[2];
        ubyte          *q        = t.ptr + 3; // start of comment text
        int            linestart = 0;

        ubyte          *qend = p;

        if (ct == '*' || ct == '+')
            qend -= 2;

        // Scan over initial row of ****'s or ++++'s or ////'s
        for (; q < qend; q++)
        {
            if (*q != ct)
                break;
        }

        // Remove trailing row of ****'s or ++++'s
        if (ct != '/')
        {
            for (; q < qend; qend--)
            {
                if (qend[-1] != ct)
                    break;
            }
        }

        for (; q < qend; q++)
        {
            ubyte c = *q;

            switch (c)
            {
            case '*':
            case '+':
                if (linestart && c == ct)
                {
                    linestart = 0;
                    // Trim preceding whitespace up to preceding \n
                    while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))
                        buf.offset--;
                    continue;
                }
                break;

            case ' ':
            case '\t':
                break;

            case '\r':
                if (q[1] == '\n')
                    continue;                                   // skip the \r
                goto Lnewline;

            default:
                if (c == 226)
                {
                    // If LS or PS
                    if (q[1] == 128 &&
                        (q[2] == 168 || q[2] == 169))
                    {
                        q += 2;
                        goto Lnewline;
                    }
                }
                linestart = 0;
                break;

 Lnewline:
                c = '\n';                               // replace all newlines with \n

            case '\n':
                linestart = 1;

                // Trim trailing whitespace
                while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))
                    buf.offset--;

                break;
            }
            buf.write(c);
        }

        // Always end with a newline
        if (!buf.offset || buf.data[buf.offset - 1] != '\n')
            buf.writenl();

        //buf.write(cast(char)0);

        // It's a line comment if the start of the doc comment comes
        // after other non-whitespace on the same line.
//	    ubyte** dc = (lineComment && anyToken)
//				 ? &t.lineComment
//				 : &t.blockComment;

        char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;

        // Combine with previous doc comment, if any
        if (dc.length)
            dc = combineComments(dc, buf.toString().dup);
        else
            dc = buf.toString().dup;

//		writefln( dc );

        if (lineComment && anyToken)
            t.lineComment = dc;
        else
            t.blockComment = dc;
    }
}

// character maps
static ubyte[256] cmtable;

const int CMoctal  = 0x1;
const int CMhex    = 0x2;
const int CMidchar = 0x4;

ubyte isoctal(ubyte c)
{
    return cmtable[c] & CMoctal;
}
ubyte ishex(ubyte c)
{
    return cmtable[c] & CMhex;
}
ubyte isidchar(ubyte c)
{
    return cmtable[c] & CMidchar;
}

static void cmtable_init()
{
    for (uint c = 0; c < cmtable.length; c++)
    {
        if ('0' <= c && c <= '7')
            cmtable[c] |= CMoctal;
        if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F'))
            cmtable[c] |= CMhex;
        if (isalnum(c) || c == '_')
            cmtable[c] |= CMidchar;
    }
}


/+
   struct StringValue
   {
    union
    {
                int intvalue;
                void *ptrvalue;
                dchar *string;
    }

    char[] lstring;
   }
   #define CASE_BASIC_TYPES
        case TOKwchar: case TOKdchar:
        case TOKbit: case TOKbool: case TOKchar:
        case TOKint8: case TOKuns8:
        case TOKint16: case TOKuns16:
        case TOKint32: case TOKuns32:
        case TOKint64: case TOKuns64:
        case TOKfloat32: case TOKfloat64: case TOKfloat80:
        case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
        case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
        case TOKvoid:

   #define CASE_BASIC_TYPES_X(t)					\
        case TOKvoid:	 t = Type::tvoid;  goto LabelX;		\
        case TOKint8:	 t = Type::tint8;  goto LabelX;		\
        case TOKuns8:	 t = Type::tuns8;  goto LabelX;		\
        case TOKint16:	 t = Type::tint16; goto LabelX;		\
        case TOKuns16:	 t = Type::tuns16; goto LabelX;		\
        case TOKint32:	 t = Type::tint32; goto LabelX;		\
        case TOKuns32:	 t = Type::tuns32; goto LabelX;		\
        case TOKint64:	 t = Type::tint64; goto LabelX;		\
        case TOKuns64:	 t = Type::tuns64; goto LabelX;		\
        case TOKfloat32: t = Type::tfloat32; goto LabelX;	\
        case TOKfloat64: t = Type::tfloat64; goto LabelX;	\
        case TOKfloat80: t = Type::tfloat80; goto LabelX;	\
        case TOKimaginary32: t = Type::timaginary32; goto LabelX;	\
        case TOKimaginary64: t = Type::timaginary64; goto LabelX;	\
        case TOKimaginary80: t = Type::timaginary80; goto LabelX;	\
        case TOKcomplex32: t = Type::tcomplex32; goto LabelX;	\
        case TOKcomplex64: t = Type::tcomplex64; goto LabelX;	\
        case TOKcomplex80: t = Type::tcomplex80; goto LabelX;	\
        case TOKbit:	 t = Type::tbit;     goto LabelX;	\
        case TOKchar:	 t = Type::tchar;    goto LabelX;	\
        case TOKwchar:	 t = Type::twchar; goto LabelX;	\
        case TOKdchar:	 t = Type::tdchar; goto LabelX;	\
        LabelX
 +/
