%{
/*
    scanner.lpp

    Qore Programming Language

    Copyright (C) 2003 - 2023 Qore Technologies, s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

    Note that the Qore library is released under a choice of three open-source
    licenses: MIT (as above), LGPL 2+, or GPL 2+; see README-LICENSE for more
    information.

    Flex scanner:
        no string length limits,
        no include file depth limits,
        thread-safe

    requires flex 2.5.31 or better (2.5.37+ recommended, 2.5.4 will not work)
    so a thread-safe scanner can be generated

    see: http://flex.sourceforge.net/
*/

#include <qore/Qore.h>
#include "qore/intern/ParserSupport.h"
#include "qore/intern/qore_program_private.h"
#include "qore/intern/ModuleInfo.h"
#include "qore/intern/QoreParseHashNode.h"

#include "parser.hpp"

#include <cctype>
#include <climits>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#define YY_USER_ACTION { yylloc->update(yylineno, yyleng, yytext); }

int yyparse(yyscan_t yyscanner);

static const QoreProgramLocation* get_loc(YYLTYPE* yylloc) {
    return qore_program_private::get(*getProgram())->getLocation(yylloc->first_line, yylloc->last_line);
}

static void check_broken_op(YYLTYPE* yylloc, const char* got, const char* expected) {
    if (!parse_check_parse_option(PO_BROKEN_OPERATORS)) {
        parse_error(*get_loc(yylloc), "broken operator expression found; got \"%s\"; expected: \"%s\"; "
            "multi-character operators separated by whitespace are only allowed if %broken-operators "
            "(PO_BROKEN_OPERATORS) is set", got, expected);
    }
}

static int ascii_to_octal(char c) {
   assert(isoctaldigit(c));
   return c - '0';
}

static char* make_scoped_vref(const char* str) {
   size_t len = strlen(str);
   char* rv = (char*)malloc(sizeof(char) * len);
   size_t i = 0;
   while (*str) {
      if (*str != '$')
         rv[i++] = *str;
      ++str;
   }
   rv[i] = '\0';
   return rv;
}

// the return value of this function must be freed if non-0
static char* trim(const char* str) {
   while ((*str) == ' ' || (*str) == '\t')
      str++;
   // duplicate string
   char* n = strdup(str);
   // find end of string
   unsigned l = (unsigned)strlen(n);
   if (l) {
      char* e = n + l - 1;
      while ((*e) == ' ' || (*e) == '\t')
         *(e--) = '\0';
   }
   if (!n[0]) {
      free(n);
      n = 0;
   }
   return n;
}

// edits the string (even though it's a 'const char*') and returns a pointer to a string with leading and trailing
// whitespace removed
static char* trim_inplace(const char* str) {
   while ((*str) == ' ' || (*str) == '\t')
      str++;
   // find end of string
   unsigned l = (unsigned)strlen(str);
   if (l) {
      const char* e = str + l - 1;
      while ((*e) == ' ' || (*e) == '\t')
         *((char*)(e--)) = '\0';
   }
   return str[0] ? (char*)str : 0;
}

#if 0
// everybody likes to get the last word - now there's a function for it
static const char* get_last_word(const char* str) {
   unsigned l = (unsigned)strlen(str);
   assert(l);

   const char* e = str + l - 2;

   while (e != str && *e != ' ' && *e != '\t' && *e != '\r' && *e != '\n')
      --e;

   //printd(0, "get_last_word() '%s' -> '%s'\n", str, e + 1);
   return e == str ? e : e + 1;
}
#endif

// sets the time zone at parse time - this function is exported in the library
void parse_set_time_zone(const char* zone) {
   zone = trim_inplace(zone);
   if (!zone) {
      parse_error(QoreProgramLocation(), "empty argument to %set-time-zone");
      return;
   }

   getProgram()->parseSetTimeZone(zone);
}

static void scanner_process_path(QoreString& npath, char* ip) {
    // scan through paths and:
    // 1: do environment variable substitution
    // 2: remove the paths that don't exist
    char* lp, *p;
    p = lp = ip;

    while (true) {
#ifdef _Q_WINDOWS
       // do not assume ':' separates paths on windows if it's the second character in a path
       if ((*p == ':' && p != lp + 1) || !*p) {
#else
       if (*p == ':' || !*p) {
#endif
          // skip if at beginning of new path
          if (p != lp) {
             QoreString tpath;
             tpath.concat(lp, p - lp);
             const char* sp = tpath.c_str();
             //printd(5, "got path string: '%s'\n", sp);

             const char* i, *spt;
             spt = sp;

             while ((i = strchr(spt, '$'))) {
                // find end of environment variable
                char* ep = (char*)++i;
                while (*ep && (*ep == '_' || isalnum(*ep)))
                   ++ep;
                spt = i + 1;
                // perform variable substitution
                if (ep != i) {
                   char save = *ep;
                   *ep = '\0';
                   TempString val(SystemEnvironment::get(i));
                   //printd(5, "set_include_path() found variable '$%s' = %s\n", i, val->c_str());
                   *ep = save;
                   if (val && val->strlen()) {
                      // see if environment value ends in the directory separator char, if so, and the
                      // rest of the string starts with the directory separator char, then make sure the
                      // character is not doubled (can cause problems on windows)
                      if (val->c_str()[val->strlen() - 1] == QORE_DIR_SEP
                            && *ep == QORE_DIR_SEP)
                         ++ep;

                      // replace with value
                      int pos = (int)(i - sp);
                      //printd(5, "got $%s = '%s' (replacing %d char(s))\n", i, val->c_str(), ep - i + 1);
                      tpath.replace(pos - 1, ep - i + 1, val->c_str());
                      // re-assign sp in case it's changed
                      int diff = (int)(pos + val->strlen());
                      sp = tpath.c_str();
                      spt = sp + diff;
                      //printd(5, "new string = '%s' ('%s')\n", sp, spt);
                   }
                }
             }
             struct stat sb;

             if (tpath[0] == '.' && tpath[1] == '.') {
                // substitue program path if available
                const char* sp = getProgram()->parseGetScriptDir();
                if (sp) {
                   size_t l = strlen(sp);
                   if (tpath[0] != '/' && tpath[0] != '\\' && sp[l-1] != '/' && sp[l-1] != '\\')
                      tpath.replace(0, 0, QORE_DIR_SEP_STR);
                   tpath.replace(0, 0, sp);
                }
             }

             //printd(5, "set_include_path() trying '%s'\n", sp);
             // add to path list if the directory exists
             if (!stat(sp, &sb)) {
                //printd(5, "OK: adding '%s' to path list\n", sp);
                if (npath.strlen())
                   npath.concat(':');
                npath.concat(&tpath);
             }

             if (!*p)
                break;
          }
          lp = ++p;
          continue;
       }
#ifdef _Q_WINDOWS
       // transform '/ characters to '\' characters
       if (*p == '/')
          *p = '\\';
#endif

       ++p;
    }
    //printd(5, "ip: '%s' npath: '%s'\n", ip, npath.c_str());
}

static QoreString* get_include_file_name(char* file) {
#ifdef _Q_WINDOWS
    // convert UNIX-style '/' character to '\'
    for (char* p = file; *p; ++p)
        if (*p == '/')
            *p = '\\';
#endif

    // if the filename begins with an environment variable, then try to do environment variable substitution
    if (file[0] == '$') {
        QoreString* rv = new QoreString;

        QoreString var;
        char* p = file + 1;
        while (*p && (*p == '_' || isalnum(*p))) {
            var.concat(*p);
            ++p;
        }

        QoreString val;
        if (SystemEnvironment::get(var.c_str(), val)) {
            rv->concat(file);
        } else {
            rv->concat(&val);
            rv->concat(file + var.size() + 1);
        }

        return rv;
    }

    //printd(5, "get_include_file_name(%s)\n", file);
    // FIXME: UNIX-specific - need to check for drive name on windows
    if (file[0] == QORE_DIR_SEP) {
        return new QoreString(file);
    }

    QoreString *rv;

    // check in current directory of script first
    const char* sp = getProgram()->parseGetScriptDir();
    if (sp) {
        rv = findFileInPath(file, sp);
        if (rv) {
            return rv;
        }
    }

    rv = findFileInEnvPath(file, "QORE_INCLUDE_DIR");
    if (!rv) {
        const char* pp = getProgram()->parseGetIncludePath();
        if (pp) {
            rv = findFileInPath(file, pp);
        }
        if (!rv) {
            rv = new QoreString(file);
        }
    }

    return rv;
}

static void append_module_path(const char* path) {
    char* ip = trim(path);
    if (!ip) {
        return;
    }
    ON_BLOCK_EXIT(free, ip);

    QoreString npath; // for new path

    scanner_process_path(npath, ip);

    //printd(5, "appending module path: '%s'\n", npath.c_str());
    QMM.addModuleDirList(npath.c_str());
    //getProgram()->parseSetIncludePath(npath.c_str());
}

static void set_include_path(const char* path) {
    char* ip = trim(path);
    if (!ip) {
        return;
    }
    ON_BLOCK_EXIT(free, ip);

    QoreString npath; // for new path

    scanner_process_path(npath, ip);

    //printd(5, "setting include path: '%s'\n", npath.c_str());
    getProgram()->parseSetIncludePath(npath.c_str());
}

static char* remove_quotes(char* str) {
    str[strlen(str) - 1] = '\0';
    return str + 1;
}

static int check_reexport(YYLTYPE* yylloc, const char* directive, char*& cn, bool& reexport) {
    if (*cn != '(') {
        return 0;
    }
    char* p = strchr(cn, ')');
    if (!p) {
        parse_error(*get_loc(yylloc), "missing end parenthesis in %%%s directive", directive);
        return -1;
    }

    const char* ropt = cn + 1;
    *p = '\0';
    cn = p + 1;
    if (strcmp(ropt, "reexport")) {
        parse_error(*get_loc(yylloc), "unrecognized %%%s option '%s'; expected 'reexport'", directive, ropt);
        return -1;
    }

    reexport = true;
    cn = trim_inplace(cn);
    if (!cn) {
        parse_error(*get_loc(yylloc), "missing argument to %%%s", directive);
        return -1;
    }

    return 0;
}

static DateTimeNode *makeYears(int years) {
    return new DateTimeNode(years, 0, 0, 0, 0, 0, 0, true);
}

static DateTimeNode *makeMonths(int months) {
    return new DateTimeNode(0, months, 0, 0, 0, 0, 0, true);
}

static DateTimeNode *makeDays(int days) {
    return new DateTimeNode(0, 0, days, 0, 0, 0, 0, true);
}

static DateTimeNode *makeHours(int hours) {
    return new DateTimeNode(0, 0, 0, hours, 0, 0, 0, true);
}

static DateTimeNode *makeMinutes(int minutes) {
    return new DateTimeNode(0, 0, 0, 0, minutes, 0, 0, true);
}

static DateTimeNode* makeSeconds(const char* txt) {
    long secs = strtol(txt, 0, 10);
    double us = q_strtod(txt) - secs;
    if (us) {
        us *= 1000000.0;
    }

    return DateTimeNode::makeRelative(0, 0, 0, 0, 0, secs, (int)us);
}

static DateTimeNode *makeMilliseconds(int ms) {
    return new DateTimeNode(0, 0, 0, 0, 0, 0, ms, true);
}

static DateTimeNode *makeMicroseconds(int us) {
    return DateTimeNode::makeRelative(0, 0, 0, 0, 0, 0, us);
}

static int64 parse_get_integer(YYLTYPE* yylloc, const char* str) {
    errno = 0;
    int64 rv = strtoll(str, 0, 10);
    if (errno == ERANGE) {
        parse_error(*get_loc(yylloc), "numeric overflow in integer '%s': cannot represent in a 64-bit signed number; "
            "please use a floating-point or arbitrary-precision number instead (ex: '%sn'); integers must be in the "
            "range " QLLD " (MININT) to " QLLD " (MAXINT)", str, str, -LLONG_MAX, LLONG_MAX);
    }
    return rv;
}

static inline int get_int_2(const char*& p) {
    int rv = (*p - '0') * 10;
    ++p;
    rv += (*p - '0');
    ++p;
    return rv;
}

static inline int get_int_4(const char*& p) {
    int rv = (*p - '0') * 1000;
    ++p;
    rv += (*p - '0') * 100;
    ++p;
    rv += (*p - '0') * 10;
    ++p;
    rv += (*p - '0');
    ++p;
    return rv;
}

static void parse_time(const char*& p, int &hour, int &minute, int &second, int &us) {
    assert(!hour && !minute && !second && !us);

    // get hour
    hour = get_int_2(p);
    ++p;

    // get minute
    minute = get_int_2(p);

    if ((*p) == ':') {
        ++p;
        // get second
        second = get_int_2(p);

        // get milliseconds or microseconds if present
        if ((*p) == '.') {
            ++p;

            // read all digits
            int len = 0;
            while (isdigit(*p)) {
                us *= 10;
                us += *p - '0';
                ++len;
                ++p;
            }

            // adjust to microseconds
            while (len < 6) {
                us *= 10;
                ++len;
            }
            while (len > 6) {
                us /= 10;
                --len;
            }
        }
    }
}

static DateTimeNode *makeDateTime(char* str, bool has_time) {
    const char* p = str;

    // get year
    int year = get_int_4(p);
    ++p;

    // get month
    int month = get_int_2(p);
    ++p;

    // get day
    int day = get_int_2(p);

    int hour = 0, minute = 0, second = 0, us = 0;

    // parse time, if any
    if (*p == 'T' || (*p == '-' && has_time)) {
        ++p;
        parse_time(p, hour, minute, second, us);
    }

    // get zone if present, otherwise assume local zone
    const AbstractQoreZoneInfo *zone;
    if ((*p) == '+' || (*p) == '-')
        zone = QTZM.findCreateOffsetZone(p);
    else if ((*p) == 'Z')
        zone = 0;
    else
        zone = currentTZ();

    return DateTimeNode::makeAbsolute(zone, year, month, day, hour, minute, second, us);
}

static inline DateTimeNode *makeTime(char* str) {
    const char* p = str;

    int hour = 0, minute = 0, second = 0, us = 0;
    parse_time(p, hour, minute, second, us);

    // get zone if present, otherwise assume local zone
    const AbstractQoreZoneInfo *zone;
    if ((*p) == '+' || (*p) == '-')
        zone = QTZM.findCreateOffsetZone(p);
    else if ((*p) == 'Z')
        zone = 0;
    else
        zone = currentTZ();

    return DateTimeNode::makeAbsolute(zone, 1970, 01, 01, hour, minute, second, us);
}

static inline DateTimeNode *makeRelativeDateTime(char* str) {
    const char* p = str;

    // get year
    int year = get_int_4(p);
    ++p;

    // get month
    int month = get_int_2(p);
    ++p;

    // get day
    int day = get_int_2(p);

    int hour = 0, minute = 0, second = 0, us = 0;
    if (*p) {
        ++p;
        parse_time(p, hour, minute, second, us);
    }

    return DateTimeNode::makeRelative(year, month, day, hour, minute, second, us);
}

static DateTimeNode* makeRelativeTime(char* str) {
    const char* p = str;

    int hour = 0, minute = 0, second = 0, us = 0;
    parse_time(p, hour, minute, second, us);

    return DateTimeNode::makeRelative(0, 0, 0, hour, minute, second, us);
}

static bool isRegexModifier(QoreRegex* qr, int c) {
    if (c == 'i') {
        qr->setCaseInsensitive();
    } else if (c == 's') {
        qr->setDotAll();
    } else if (c == 'x') {
        qr->setExtended();
    } else if (c == 'm') {
        qr->setMultiline();
    } else if (c == 'u') {
        qr->setUnicode();
    } else {
        return false;
    }
    return true;
}

static bool isRegexExtractModifier(QoreRegex* qr, int c) {
    if (isRegexModifier(qr, c)) {
        return true;
    }
    if (c == 'g') {
        qr->setGlobal();
    } else {
        return false;
    }
    return true;
}

static bool isRegexSubstModifier(QoreRegexSubst* qr, int c) {
    if (c == 'g') {
        qr->setGlobal();
    } else if (c == 'i') {
        qr->setCaseInsensitive();
    } else if (c == 's') {
        qr->setDotAll();
    } else if (c == 'x') {
        qr->setExtended();
    } else if (c == 'm') {
        qr->setMultiline();
    } else if (c == 'u') {
        qr->setUnicode();
    } else {
        return false;
    }
    return true;
}

static char* make_cast(const char* s) {
    QoreString str(s + 5);
    str.terminate(str.size() - 1);
    str.trim();
    //printd(5, "make_cast() '%s'\n", str.c_str());
    return str.giveBuffer();
}

static void qore_parse_define(YYLTYPE* yylloc, const char* pstr) {
    QoreString str(pstr);
    // trim trailing and leading whitespace
    str.trim();
    const char* p = strchr(str.c_str(), ' ');
    QoreString arg;
    if (p) {
        // copy arg to arg string
        arg.set(p + 1);
        // trim leading whitespace, if any
        arg.trim_leading();
        // remove argument from define string
        str.terminate(p - str.c_str());
    }
    // check define string
    if (str.empty()) {
        parse_error(*get_loc(yylloc), "missing argument to %%define");
        return;
    }

    QoreValue v;
    if (!arg.empty()) {
        bool ok;
        v = qore_parse_get_define_value(get_loc(yylloc), str.c_str(), arg, ok);
        if (!ok)
            return;
    }

    qore_program_private::parseDefine(getProgram(), get_loc(yylloc), str.c_str(), v);
}

static bool parse_is_defined(YYLTYPE* yylloc, const char* pstr, bool def = true) {
    QoreString str(pstr);
    str.trim();
    if (str.empty()) {
        parse_error(*get_loc(yylloc), "missing argument to %%if%sdef", def ? "" : "n");
        return false;
    }
    return qore_program_private::parseIsDefined(getProgram(), str.c_str());
}

static bool is_identifier(const char *str) {
    if (!(getProgram()->getParseOptions64() & PO_REQUIRE_BARE_REFS)) {
        if (*str++ != '$') {
            return false;
        }
    }
    if (!isalpha(*str) && *str != '_') {
        return false;
    }
    while (*++str) {
        if (!isalpha(*str) && !isdigit(*str) && *str != '_') {
            return false;
        }
    }
    return true;
}

static void parse_set_parse_options(YYLTYPE* yylloc, int64 po) {
    qore_program_private::get(*getProgram())->parseSetParseOptions(get_loc(yylloc), po);
}

static void parse_disable_parse_options(YYLTYPE* yylloc, int64 po) {
    qore_program_private::get(*getProgram())->parseDisableParseOptions(get_loc(yylloc), po);
}

static bool angle_balanced(const char* p) {
    //printd(5, "angle_balanced() '%s'\n", p);
    int ob = 0;
    int cb = 0;
    while (*p) {
        switch (*p) {
            case '<': ++ob; break;
            case '>': ++cb; break;
        }
        ++p;
    }
    //printd(5, "angle_balanced() %d == %d\n", ob, cb);

    return ob == cb;
}

#define DO_DOT_KW_IDENTIFIER(kw) { yylval->String = new QoreStringNode(kw); size_t pos = yylval->String->size() + 1; if (yytext[pos]) unput(yytext[pos]); return DOT_KW_IDENTIFIER; }

#define QORE_FLEX_DO_EOF { /* delete current buffer */ yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner); \
    /* get last parse state */ YY_BUFFER_STATE yybs = (YY_BUFFER_STATE)endParsing(); \
    if (yybs) { /* need to close file and switch to previous buffer */ \
        fclose(yyin); if (yybs) yy_switch_to_buffer(yybs, yyscanner); \
    } else yyterminate();  \
}
%}

%option noyywrap nomain noyy_top_state warn
%option reentrant bison-bridge bison-locations
%option stack
%option yylineno
%option noyy_push_state
%option noyy_pop_state

%x str_state regex_state incl p_def p_ifdef p_ifndef p_skip_to_endif p_skip_to_endif_or_else case_state regex_googleplex regex_negative_universe regex_subst1 regex_subst2 line_comment block_comment1 block_comment2 exec_class_state requires regex_trans1 regex_trans2 regex_extract_state disable_warning enable_warning append_path_state append_module_state module_cmd set_zone try_module p_skip_to_endtry angle_state

HEX_DIGIT       [0-9A-Fa-f]
HEX_CONST       0x{HEX_DIGIT}+
OCTAL_CONST     \\[0-7]{1,3}
DIGIT           [0-9]
WORD            [a-zA-Z_][a-zA-Z0-9_]*
WS              [ \t\r]
WSNL            [ \t\r\n]
YEAR            [0-9]{4}
MONTH           (0[1-9])|(1[012])
DAY             ((0[1-9])|([12][0-9])|(3[01]))
HOUR            ([01][0-9])|(2[0-3])
MSEC            [0-5][0-9]
US              [0-9]{1,6}
D2              [0-9]{2}
BINARY          <({HEX_DIGIT}{HEX_DIGIT})+>
TZ              (Z|([+-][01][0-9](:{MSEC}(:{MSEC})?)?))
RDATE           P(-?[0-9]+(\.[0-9]+)?[YMDHSu])+
RDATETIME       P(-?[0-9]+(\.[0-9]+)?[YMD])+T(-?[0-9]+(\.[0-9]+)?[HMSu])+
RTIME           PT(-?[0-9]+(\.[0-9]+)?[HMSu])+

%%
^%no-global-vars{WS}*$                  parse_set_parse_options(yylloc, PO_NO_GLOBAL_VARS);
^%no-subroutine-defs{WS}*$              parse_set_parse_options(yylloc, PO_NO_SUBROUTINE_DEFS);
^%no-threads{WS}*$                      parse_set_parse_options(yylloc, PO_NO_THREADS);
^%no-thread-classes{WS}*$               parse_set_parse_options(yylloc, PO_NO_THREAD_CLASSES);
^%no-thread-control{WS}*$               parse_set_parse_options(yylloc, PO_NO_THREAD_CONTROL);
^%no-thread-info{WS}*$                  parse_set_parse_options(yylloc, PO_NO_THREAD_INFO);
^%no-top-level{WS}*$                    parse_set_parse_options(yylloc, PO_NO_TOP_LEVEL_STATEMENTS);
^%no-class-defs{WS}*$                   parse_set_parse_options(yylloc, PO_NO_CLASS_DEFS);
^%no-namespace-defs{WS}*$               parse_set_parse_options(yylloc, PO_NO_NAMESPACE_DEFS);
^%no-external-process{WS}*$             parse_set_parse_options(yylloc, PO_NO_EXTERNAL_PROCESS);
^%no-external-info{WS}*$                parse_set_parse_options(yylloc, PO_NO_EXTERNAL_INFO);
^%no-external-access{WS}*$              parse_set_parse_options(yylloc, PO_NO_EXTERNAL_ACCESS);
^%no-filesystem{WS}*$                   parse_set_parse_options(yylloc, PO_NO_FILESYSTEM);
^%no-io{WS}*$                           parse_set_parse_options(yylloc, PO_NO_IO);
^%lockdown{WS}*$                        parse_set_parse_options(yylloc, PO_LOCKDOWN);
^%lock-options{WS}*$                    getProgram()->lockOptions();
^%lock-warnings{WS}*$                   parse_set_parse_options(yylloc, PO_LOCK_WARNINGS);
^%no-locale-control{WS}*$               parse_set_parse_options(yylloc, PO_NO_LOCALE_CONTROL);
^%no-process-control{WS}*$              parse_set_parse_options(yylloc, PO_NO_PROCESS_CONTROL);
^%no-constant-defs{WS}*$                parse_set_parse_options(yylloc, PO_NO_CONSTANT_DEFS);
^%no-new{WS}*$                          parse_set_parse_options(yylloc, PO_NO_NEW);
^%no-network{WS}*$                      parse_set_parse_options(yylloc, PO_NO_NETWORK);
^%no-child-restrictions{WS}*$           parse_set_parse_options(yylloc, PO_NO_CHILD_PO_RESTRICTIONS);
^%allow-injection{WS}*$                 parse_set_parse_options(yylloc, PO_ALLOW_INJECTION);
^%no-database{WS}*$                     parse_set_parse_options(yylloc, PO_NO_DATABASE);
^%no-gui{WS}*$                          parse_set_parse_options(yylloc, PO_NO_GUI);
^%no-terminal-io{WS}*$                  parse_set_parse_options(yylloc, PO_NO_TERMINAL_IO);
^%no-modules{WS}*$                      parse_set_parse_options(yylloc, PO_NO_MODULES);
^%require-our{WS}*$                     parse_set_parse_options(yylloc, PO_REQUIRE_OUR);
^%require-types{WS}*$                   parse_set_parse_options(yylloc, PO_REQUIRE_TYPES);
^%require-prototypes{WS}*$              parse_set_parse_options(yylloc, PO_REQUIRE_PROTOTYPES);
^%strict-args{WS}*$                     parse_set_parse_options(yylloc, PO_STRICT_ARGS);
^%allow-bare-refs{WS}*$                 parse_set_parse_options(yylloc, PO_ALLOW_BARE_REFS);
^%require-dollar{WS}*$                  parse_disable_parse_options(yylloc, PO_ALLOW_BARE_REFS);
^%assume-local{WS}*$                    parse_set_parse_options(yylloc, PO_ASSUME_LOCAL);
^%assume-global{WS}*$                   parse_disable_parse_options(yylloc, PO_ASSUME_LOCAL);
^%new-style{WS}*$                       parse_set_parse_options(yylloc, PO_NEW_STYLE);
^%old-style{WS}*$                       parse_disable_parse_options(yylloc, PO_NEW_STYLE);
^%perl-bool-eval{WS}*$                  parse_disable_parse_options(yylloc, PO_STRICT_BOOLEAN_EVAL);
^%strict-bool-eval{WS}*$                parse_set_parse_options(yylloc, PO_STRICT_BOOLEAN_EVAL);
^%strong-encapsulation{WS}*$            parse_set_parse_options(yylloc, PO_STRONG_ENCAPSULATION);
^%no-uncontrolled-apis{WS}*$            parse_set_parse_options(yylloc, PO_NO_UNCONTROLLED_APIS);
^%no-debugging{WS}*$                    parse_set_parse_options(yylloc, PO_NO_DEBUGGING);
^%allow-weak-references{WS}*$           parse_set_parse_options(yylloc, PO_ALLOW_WEAK_REFERENCES);
^%allow-debugger{WS}*$                  parse_set_parse_options(yylloc, PO_ALLOW_DEBUGGER);
^%broken-list-parsing{WS}*$             parse_set_parse_options(yylloc, PO_BROKEN_LIST_PARSING);
^%broken-logic-precedence{WS}*$         parse_set_parse_options(yylloc, PO_BROKEN_LOGIC_PRECEDENCE);
^%broken-loop-statement{WS}*$           parse_set_parse_options(yylloc, PO_BROKEN_LOOP_STATEMENT);
^%broken-int-assignments{WS}*$          parse_set_parse_options(yylloc, PO_BROKEN_INT_ASSIGNMENTS);
^%broken-operators{WS}*$                parse_set_parse_options(yylloc, PO_BROKEN_OPERATORS);
^%correct-list-parsing{WS}*$            parse_disable_parse_options(yylloc, PO_BROKEN_LIST_PARSING);
^%correct-logic-precedence{WS}*$        parse_disable_parse_options(yylloc, PO_BROKEN_LOGIC_PRECEDENCE);
^%correct-loop-statement{WS}*$          parse_disable_parse_options(yylloc, PO_BROKEN_LOOP_STATEMENT);
^%correct-int-assignments{WS}*$         parse_disable_parse_options(yylloc, PO_BROKEN_INT_ASSIGNMENTS);
^%correct-operators{WS}*$               parse_disable_parse_options(yylloc, PO_BROKEN_OPERATORS);
^%loose-args{WS}*$                      parse_disable_parse_options(yylloc, PO_STRICT_ARGS);
^%broken-range{WS}*$                    parse_set_parse_options(yylloc, PO_BROKEN_RANGE);
^%correct-range{WS}*$                   parse_disable_parse_options(yylloc, PO_BROKEN_RANGE);
^%broken-references{WS}*$               parse_set_parse_options(yylloc, PO_BROKEN_REFERENCES);
^%correct-references{WS}*$              parse_disable_parse_options(yylloc, PO_BROKEN_REFERENCES);
^%broken-sprintf{WS}*$                  parse_set_parse_options(yylloc, PO_BROKEN_SPRINTF);
^%correct-sprintf{WS}*$                 parse_disable_parse_options(yylloc, PO_BROKEN_SPRINTF);
^%broken-cast{WS}*$                     parse_set_parse_options(yylloc, PO_BROKEN_CAST);
^%correct-cast{WS}*$                    parse_disable_parse_options(yylloc, PO_BROKEN_CAST);
^%broken-varargs{WS}*$                  parse_set_parse_options(yylloc, PO_BROKEN_VARARGS);
^%correct-varargs{WS}*$                 parse_disable_parse_options(yylloc, PO_BROKEN_VARARGS);
^%allow-returns{WS}*$                   parse_set_parse_options(yylloc, PO_ALLOW_RETURNS);
^%allow-statement-no-effect{WS}*$       parse_set_parse_options(yylloc, PO_ALLOW_STATEMENT_NO_EFFECT);
^%strict-types{WS}*$                    parse_set_parse_options(yylloc, PO_STRICT_TYPES);
^%loose-types{WS}*$                     parse_disable_parse_options(yylloc, PO_STRICT_TYPES);
^%no-reflection{WS}*$                   parse_set_parse_options(yylloc, PO_NO_REFLECTION);
^%no-transient{WS}*$                    parse_set_parse_options(yylloc, PO_NO_TRANSIENT);
^%push-parse-options{WS}*$              push_parse_options();
^%append-include-path{WS}*$             { parse_error(*get_loc(yylloc), "missing argument to %%append-include-path"); }
^%append-include-path{WS}+              BEGIN(append_path_state);
<append_path_state>[^\n\r]+             {
                                           set_include_path(yytext);
                                           BEGIN(INITIAL);
                                        }
<append_path_state><<EOF>>              {
                                           QORE_FLEX_DO_EOF
                                        }
^%append-module-path{WS}*$              { parse_error(*get_loc(yylloc), "missing argument to %%append-module-path"); }
^%append-module-path{WS}+               BEGIN(append_module_state);
<append_module_state>[^\n\r]+           {
                                           append_module_path(yytext);
                                           BEGIN(INITIAL);
                                        }
<append_module_state><<EOF>>            {
                                           QORE_FLEX_DO_EOF
                                        }
^%set-time-zone{WS}*$                   { parse_error(*get_loc(yylloc), "missing argument to %%set-time-zone"); }
^%set-time-zone{WS}+                    BEGIN(set_zone);
<set_zone>[^\n\r]+                      {
                                           yytext = trim_inplace(yytext);
                                           if (!yytext)
                                              parse_error(*get_loc(yylloc), "empty argument to %set-time-zone");
                                           else
                                              getProgram()->parseSetTimeZone(yytext);
                                           BEGIN(INITIAL);
                                        }
<set_zone><<EOF>>                       {
                                           QORE_FLEX_DO_EOF
                                        }
^%enable-all-warnings{WS}*$             {
                                           if (getProgram()->setWarningMask(-1))
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_WARNING_MASK_UNCHANGED, "CANNOT-UPDATE-WARNING-MASK", "this program has its warning mask locked; cannot enable all warnings");
                                        }
^%disable-all-warnings{WS}*$            {
                                           if (getProgram()->setWarningMask(0))
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_WARNING_MASK_UNCHANGED, "CANNOT-UPDATE-WARNING-MASK", "this program has its warning mask locked; cannot disable all warnings");
                                        }
^%disable-warning{WS}*$                 { parse_error(*get_loc(yylloc), "missing argument to %%disable-warning"); }
^%disable-warning{WS}+                  BEGIN(disable_warning);
<disable_warning>[^\n\r]+               {
                                           char* cn = trim(yytext);
                                           //printd(5, "scanner: disable warning '%s'\n", cn);
                                           int code = get_warning_code(cn);
                                           if (!code)
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_UNKNOWN_WARNING, "UNKNOWN-WARNING", "cannot disable unknown warning '%s'", cn);
                                           else if (getProgram()->disableWarning(code))
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_WARNING_MASK_UNCHANGED, "CANNOT-UPDATE-WARNING-MASK", "this program has its warning mask locked; cannot disable warning '%s'", cn);
                                           //printd(5, "scanner: disable warning '%s' code=%d\n", cn, code);
                                           free(cn);
                                           BEGIN(INITIAL);
                                        }
<disable_warning><<EOF>>                {
                                           QORE_FLEX_DO_EOF
                                        }
^%enable-warning{WS}*$                  { parse_error(*get_loc(yylloc), "missing argument to %%enable-warning"); }
^%enable-warning{WS}+                   BEGIN(enable_warning);
<enable_warning>[^\n\r]+                {
                                           char* cn = trim(yytext);
                                           //printd(5, "scanner: enable warning '%s'\n", cn);
                                           int code = get_warning_code(cn);
                                           if (!code)
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_UNKNOWN_WARNING, "UNKNOWN-WARNING", "cannot enable unknown warning '%s'", cn);
                                           else if (getProgram()->enableWarning(code))
                                              qore_program_private::makeParseWarning(getProgram(), *get_loc(yylloc), QP_WARN_WARNING_MASK_UNCHANGED, "CANNOT-UPDATE-WARNING-MASK", "this program has its warning mask locked; cannot enable warning '%s'", cn);
                                           free(cn);
                                           BEGIN(INITIAL);
                                        }
<enable_warning><<EOF>>                 {
                                           QORE_FLEX_DO_EOF
                                        }
^%exec-class{WS}*$                      { parse_error(*get_loc(yylloc), "missing argument to %%exec-class"); }
^%exec-class{WS}+                       BEGIN(exec_class_state);
^%try-reexport-module{WS}*$             {
                                           parse_error(*get_loc(yylloc), "missing argument to %%try-reexport-module");
                                           BEGIN(p_skip_to_endtry);
                                        }
^%try-reexport-module                   {
                                           parse_set_try_reexport(true);
                                           BEGIN(try_module);
                                        }
^%try-module{WS}*$                      {
                                           parse_error(*get_loc(yylloc), "missing argument to %%try-module");
                                           BEGIN(p_skip_to_endtry);
                                        }
^%try-module                            {
                                           parse_set_try_reexport(false);
                                           BEGIN(try_module);
                                        }
<try_module>[^\n\r]+                    {
                                           parse_try_module_inc();
                                           QoreString str(yytext);
                                           bool ok = true;

                                           // we need to check the first character before we trim the string
                                           // if it's not whitespace and not an open paren, then throw an error
                                           if (str.size() && str[0] != '(' && str[0] != ' '
                                               && str[0] != '\t' && str[0] != '\n' && str[0] != '\r') {
                                              parse_error(*get_loc(yylloc), "invalid %%try-module declaration c: %c", str[0]);
                                              ok = false;
                                           } else {
                                              str.trim();
                                              QoreString var;
                                              bool require_dollar = !(getProgram()->getParseOptions64() & PO_REQUIRE_BARE_REFS);
                                              if (str.empty()) {
                                                 parse_error(*get_loc(yylloc), "missing argument to %%try-module");
                                                 ok = false;
                                              } else {
                                                 const char* p = str.c_str();
                                                 const char* op = strchr(p, '(');
                                                 if (op) {
                                                    if (op != p) {
                                                       parse_error(*get_loc(yylloc), "%%try-module not in format '%%try-module (%svarname) <module>'; extra text before '('; text given: '%%try-module %s'", require_dollar ? "$" : "", str.c_str());
                                                       ok = false;
                                                    }
                                                    ++p;
                                                    const char* cp = strchr(p, ')');
                                                    if (!cp) {
                                                       parse_error(*get_loc(yylloc), "%%try-module not in format '%%try-module (%svarname) <module>'; missing close parenthesis ')' after variable name; text given: '%%try-module %s'", require_dollar ? "$" : "", str.c_str());
                                                       ok = false;
                                                    } else {
                                                       // get variable name
                                                       var.concat(p, cp - p);
                                                       var.trim("\t ");

                                                       if (var.empty()) {
                                                          parse_error(*get_loc(yylloc), "%%try-module not in format '%%try-module (%svarname) <module>'; missing exception variable name; text given: '%%try-module %s'", require_dollar ? "$" : "", str.c_str());
                                                          ok = false;
                                                       } else {
                                                          // remove preface from string leaving only the rest
                                                          str.replace(0, cp - str.c_str() + 1, (const char*)0);
                                                          if (!is_identifier(var.c_str())) {
                                                             parse_error(*get_loc(yylloc), "%%try-module not in format '%%try-module (%svarname) <module>'; invalid exception variable name; text given: '%%try-module (%s)'", require_dollar ? "$" : "", var.c_str(), str.c_str());
                                                             ok = false;
                                                          }
                                                       }
                                                    }
                                                 }
                                              }

                                              if (!ok) {
                                                 BEGIN(p_skip_to_endtry);
                                                 //BEGIN(INITIAL);
                                              } else {
                                                 str.trim();
                                                 if (str.empty()) {
                                                    if (var.empty())
                                                       parse_error(*get_loc(yylloc), "%%try-module not in format %%try-module <module>; missing module name; text given: '%%try-module %s'", str.c_str());
                                                    else
                                                       parse_error(*get_loc(yylloc), "%%try-module not in format %%try-module (%svarname) <module>; missing module name; text given: '%%try-module %s'", require_dollar ? "$" : "", str.c_str());
                                                    BEGIN(INITIAL);
                                                 } else {
                                                    //printd(5, "scanner requesting feature: '%s'\n", cn);
                                                    QoreProgram* pgm = getProgram();
                                                    ExceptionSink xsink;
                                                    QMM.parseLoadModule(xsink, xsink, str.c_str(), pgm, parse_get_try_reexport());
                                                    //printd(5, "'%s': xsink: %d err: %d var: '%s'\n", str.c_str(), (bool)xsink, err, var.c_str());
                                                    if (!xsink) {
                                                       BEGIN(p_skip_to_endtry);
                                                    } else {
                                                       BEGIN(INITIAL);
                                                       if (var.empty()) {
                                                          // ignore exception in this case
                                                          xsink.clear();
                                                       } else {
                                                          yylval->trymod = new TryModuleError(var.giveBuffer(), xsink.catchException());
                                                          return TRY_MODULE_ERROR;
                                                       }
                                                    }
                                                 }
                                              }
                                           }
                                        }
<try_module><<EOF>>                     {
                                           QORE_FLEX_DO_EOF
                                        }
<p_skip_to_endtry>{
   ^%endtry{WS}*$                       {
                                           if (parse_try_module_dec(get_loc(yylloc))) {
                                              //printd(5, "p_skip_to_endtry: found %%endtry (%s) (final)\n", yytext);
                                              BEGIN(INITIAL);
                                           }
#ifdef DEBUG
                                           //else
                                           //printd(5, "p_skip_to_endif: found %%endif (%s) (not final)\n", yytext);
#endif
                                        }
   ^.*                                  {
                                           if (!strncmp("%try-module", yytext, 11)) {
                                              //printd(5, "p_skip_to_endtry: found %%try-module (%s); skipping\n", yytext);
                                              parse_try_module_inc();
                                           }
#ifdef DEBUG
                                           // ignore other text
                                           //printd(5, "p_skip_to_endif: %s\n", yytext);
#endif
                                        }
   \n                                   // ignore
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
^%endtry{WS}*$                          parse_try_module_dec(get_loc(yylloc));
^%requires{WS}*$                        { parse_error(*get_loc(yylloc), "missing argument to %%requires"); }
^%requires                              BEGIN(requires);
<requires>[^\n\r]+                      {
                                           char* cn = trim_inplace(yytext);
                                           if (!cn)
                                              parse_error(*get_loc(yylloc), "missing argument to %%requires");
                                           else {
                                              bool reexport = false;
                                              if (!check_reexport(yylloc, "requires", cn, reexport)) {
                                                 //printd(5, "scanner requesting feature: '%s'\n", cn);
                                                 QoreProgram* pgm = getProgram();
                                                 ExceptionSink xsink;
                                                 QMM.parseLoadModule(xsink, xsink, cn, pgm, reexport);
                                                 pgm->getParseExceptionSink()->assimilate(xsink);
                                              }
                                           }
                                           BEGIN(INITIAL);
                                        }
<requires><<EOF>>                       {
                                           QORE_FLEX_DO_EOF
                                        }
^%module-cmd{WS}*$                      { parse_error(*get_loc(yylloc), "missing argument to %%module-cmd"); }
^%module-cmd                            BEGIN(module_cmd);
<module_cmd>{WS}*                       // ignore white space
<module_cmd>[^\n\r]+                    {
                                           QoreString str(yytext);
                                           str.trim();
                                           const char* p = str.c_str();
                                           const char* op = strchr(p, '(');
                                           if (!op)
                                              parse_error(*get_loc(yylloc), "%%module-cmd not in format '%%module-cmd(<module-name>) <command>'; missing open parenthesis '(' before module name; text given: '%%module-cmd %s'", p);
                                           else {
                                              p = op + 1;
                                              const char* cp = strchr(p, ')');
                                              if (!cp)
                                                 parse_error(*get_loc(yylloc), "%%module-cmd not in format '%%module-cmd(<module-name>) <command>'; missing close parenthesis ')' after module name; text given: '%%module-cmd %s'", str.c_str());
                                              else {
                                                 // get module name
                                                 QoreString module(p, cp - p);
                                                 module.trim();

                                                 // remove preface from string leaving only command
                                                 str.replace(0, cp - str.c_str() + 1, (const char*)0);
                                                 str.trim();
                                                 const QoreProgramLocation* loc = get_loc(yylloc);
                                                 if (str.empty())
                                                    parse_error(*loc, "%%module-cmd not in format %%module-cmd(<module-name>) <command>; missing command; text given: '%%module-cmd %s'", str.c_str());
                                                 else {
                                                    QMM.issueParseCmd(loc, module.c_str(), str);
                                                 }
                                              }
                                           }
                                           BEGIN(INITIAL);
                                        }
<module_cmd><<EOF>>                     {
                                           QORE_FLEX_DO_EOF
                                        }
^%define{WS}*$                          { parse_error(*get_loc(yylloc), "missing argument to %%define"); }
^%define{WS}+                           BEGIN(p_def);
<p_def>{WS}*                            // ignore white space
<p_def>[^\n\r]+                         {
                                           qore_parse_define(yylloc, yytext);
                                           BEGIN(INITIAL);
                                        }
^%ifdef{WS}*$                           { parse_error(*get_loc(yylloc), "missing argument to %%ifdef"); }
^%ifdef{WS}+                            BEGIN(p_ifdef);
<p_ifdef>{
   [^\n\r]+$                            {
                                           parse_cond_push(true);
                                           if (parse_is_defined(yylloc, yytext)) {
                                              //printd(5, "ifdef: '%s' is defined\n", yytext);
                                              BEGIN(INITIAL);
                                           } else {
                                              //printd(5, "ifdef: '%s' is not defined\n", yytext);
                                              // skip to the next %else or %endif
                                              BEGIN(p_skip_to_endif_or_else);
                                           }
                                        }
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
^%ifndef{WS}*$                           { parse_error(*get_loc(yylloc), "missing argument to %%ifndef"); }
^%ifndef{WS}+                            BEGIN(p_ifndef);
<p_ifndef>{
   [^\n\r]+$                            {
                                           parse_cond_push(true);
                                           if (!parse_is_defined(yylloc, yytext)) {
                                              //printd(5, "ifndef: '%s' is not defined\n", yytext);
                                              BEGIN(INITIAL);
                                           } else {
                                              //printd(5, "ifndef: '%s' is defined\n", yytext);
                                              // skip to the next %else or %endif
                                              BEGIN(p_skip_to_endif_or_else);
                                           }
                                        }
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
^%else{WS}*$                            {
                                              if (!parse_cond_else()) {
                                                 parse_error(*get_loc(yylloc), "%%else found without %%if");
                                              } else {
                                                 BEGIN(p_skip_to_endif);
                                              }
                                        }
<p_skip_to_endif>{
   ^%endif{WS}*$                        {
                                           if (parse_cond_pop(get_loc(yylloc))) {
                                              printd(5, "p_skip_to_endif: found %%endif (%s) (final)\n", yytext);
                                              BEGIN(INITIAL);
                                           }
#ifdef DEBUG
                                           //else
                                           //printd(5, "p_skip_to_endif: found %%endif (%s) (not final)\n", yytext);
#endif
                                        }
   ^.*                                  {
                                           if (!strncmp("%ifdef", yytext, 6)) {
                                              //printd(5, "p_skip_to_endif: found %%ifdef (%s); skipping\n", yytext);
                                              parse_cond_push();
                                           }
#ifdef DEBUG
                                           // ignore other text
                                           //printd(5, "p_skip_to_endif: %s\n", yytext);
#endif
                                        }
   \n                                   // ignore
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
<p_skip_to_endif_or_else>{
   ^%else{WS}*$                         {
                                           //printd(5, "p_skip_to_endif_or_else: found %else\n");
                                           if (parse_cond_test(get_loc(yylloc))) {
                                              BEGIN(INITIAL);
                                           }
                                        }
   ^%endif{WS}*$                        {
                                           if (parse_cond_pop(get_loc(yylloc))) {
                                              //printd(5, "p_skip_to_endif_or_else: found %%endif (%s) (final)\n", yytext);
                                              BEGIN(INITIAL);
                                           }
#ifdef DEBUG
                                           //else
                                           //printd(5, "p_skip_to_endif_or_else: found %%endif (%s) (not final)\n", yytext);
#endif
                                        }
   ^.*                                  {
                                           if (!strncmp("%ifdef", yytext, 6)) {
                                              //printd(5, "p_skip_to_endif_or_else: found %%ifdef (%s); skipping\n", yytext);
                                              parse_cond_push();
                                           }
#ifdef DEBUG
                                           // ignore other text
                                           //printd(5, "p_skip_to_endif_or_else: %s\n", yytext);
#endif
                                        }
   \n                                   // ignore
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
^%endif{WS}*$                           parse_cond_pop(get_loc(yylloc));
^%include{WS}*$                         { parse_error(*get_loc(yylloc), "missing argument to %%include"); }
^%include{WS}+                          BEGIN(incl);
<exec_class_state>{
   [^\n\r]+                             {
                                           char* cn = trim(yytext);
                                           //printf("setting class name to: '%s'\n", cn);
                                           getProgram()->setExecClass(cn);
                                           parse_set_parse_options(yylloc, PO_NO_TOP_LEVEL_STATEMENTS);
                                           free(cn);
                                           BEGIN(INITIAL);
                                        }
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
<incl>{WS}*                             // ignore white space
<incl>[^\n\r]+                          {
                                           TempString fname(get_include_file_name(yytext));
                                           const char* fn = fname->c_str();
                                           // remove enclosing quotes if any
                                           if (fname->strlen()
                                               && ((fn[0] == '\"' && fn[fname->strlen() - 1] == '\"')
                                                   || (fn[0] == '\'' && fn[fname->strlen() - 1] == '\''))) {
                                              fname->trim(fn[0]);
                                           }

                                           QoreProgram* pgm = getProgram();

                                           if (pgm->getParseOptions64() & PO_NO_FILESYSTEM) {
                                              parse_error(*get_loc(yylloc), "cannot use the %%include parse directive when PO_NO_FILESYSTEM is in force");
                                              BEGIN(INITIAL);
                                           } else if (fname->empty()) {
                                              parse_error(*get_loc(yylloc), "missing argument to %%include", yytext);
                                              BEGIN(INITIAL);
                                           } else {
                                              // re-get the buffer pointer
                                              fn = fname->c_str();
                                              // check if regular file
                                              struct stat sbuf;
                                              int rc = stat(fn, &sbuf);
                                              if (rc) {
                                                 ExceptionSink* xsink = pgm->getParseExceptionSink();
                                                 if (xsink)
                                                    xsink->raiseErrnoException("PARSE-ERROR", errno, "stat() failed on include file: \"%s\"", fn);
                                                 BEGIN(INITIAL);
                                              } else {
                                                 //printd(0, "%s: mode=%o, s_ifmt=%o, &=%o, reg=%o comp=%s\n", fname->c_str(), sbuf.st_mode, S_IFMT, sbuf.st_mode & S_IFMT, S_IFREG, (sbuf.st_mode & S_IFMT) != S_IFREG ? "true" : "false");
                                                 if ((sbuf.st_mode & S_IFMT) != S_IFREG) {
                                                    parse_error(*get_loc(yylloc), "cannot include \"%s\"; is not a regular file", fn);
                                                    BEGIN(INITIAL);
                                                 } else {
                                                    FILE *save_yyin = yyin;
                                                    yyin = fopen(fn, "r");

                                                    if (!yyin) {
                                                       parse_error(*get_loc(yylloc), "cannot open include file \"%s\"", fn);
                                                       yyin = save_yyin;
                                                       BEGIN(INITIAL);
                                                    } else {
                                                       // save file name string in QoreProgram's list
                                                       const char* str = qore_program_private::get(*pgm)->addString(fname->c_str());
                                                       beginParsing(str, (void*)YY_CURRENT_BUFFER);
                                                       yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), yyscanner);
                                                       BEGIN(INITIAL);
                                                    }
                                                 }
                                              }
                                           }
                                        }
<incl><<EOF>>                           {
                                           QORE_FLEX_DO_EOF
                                        }
<INITIAL><<EOF>>                        {
                                           QORE_FLEX_DO_EOF
                                        }
\"                                      yylval->String = new QoreStringNode; yylloc->saveFirst(); BEGIN(str_state);
<str_state>{
      \"                                yylloc->restoreFirst(); BEGIN(INITIAL); return QUOTED_WORD;
      \n                                yylval->String->concat('\n');
      {OCTAL_CONST}                     {
                                           int token_length = strlen(yytext);
                                           int val;
                                           if (token_length == 4) {
                                              val = ascii_to_octal(*(yytext + 3))
                                                 + (8 * ascii_to_octal(*(yytext + 2)))
                                                 + (64 * ascii_to_octal(*(yytext + 1)));
                                           } else if (token_length == 3) {
                                              val = ascii_to_octal(*(yytext + 2))
                                                 + (8 * ascii_to_octal(*(yytext + 1)));
                                           } else { // here token_length == 2
                                              val = ascii_to_octal(*(yytext + 1));
                                           }
                                           if (val >= 0400)
                                              parse_error(*get_loc(yylloc), "trying to escape invalid octal value (in the range 400-777): %o", val);
                                           yylval->String->concat((char)val);
                                        }
      \\a                               yylval->String->concat('\a');
      \\b                               yylval->String->concat('\b');
      \\e                               yylval->String->concat('\e');
      \\f                               yylval->String->concat('\f');
      \\n                               yylval->String->concat('\n');
      \\r                               yylval->String->concat('\r');
      \\t                               yylval->String->concat('\t');
      \\v                               yylval->String->concat('\v');
      \\.                               yylval->String->concat(yytext[1]);
      \\\n                              yylval->String->concat('\n');
      [^\\\n\"]+                        {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->String->concat(*(yptr++));
                                        }
      <<EOF>>                           {
                                            yylval->String->deref();
                                            QORE_FLEX_DO_EOF
                                        }
}
<regex_subst2>{
      \/                                {
                                            // get regex modifiers
                                            int c;
                                            do {
                                                c = yyinput(yyscanner);
                                            } while (isRegexSubstModifier(yylval->RegexSubst, c));
                                            // issue #2096: handle EOF while scanning chars manually
                                            if (!c) {
                                                yylval->RegexSubst->deref();
                                                QORE_FLEX_DO_EOF
                                            } else {
                                                unput(c);
                                                yylloc->restoreFirst();
                                                BEGIN(INITIAL);
                                                yylval->RegexSubst->parse();
                                                return REGEX_SUBST;
                                            }
                                        }
      \n                                {
                                           yylval->RegexSubst->concatTarget('\n');
                                        }
      \\\/                              yylval->RegexSubst->concatTarget('/');
      \\\.                              yylval->RegexSubst->concatTarget('.');
      \\.                               { yylval->RegexSubst->concatTarget('\\'); yylval->RegexSubst->concatTarget(yytext[1]); }
      [^\n\\/]+                         {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->RegexSubst->concatTarget(*(yptr++));
                                        }
      <<EOF>>                           {
                                            yylval->RegexSubst->deref();
                                            QORE_FLEX_DO_EOF
                                        }
}
<regex_subst1>{
      \/                                BEGIN(regex_subst2);
      \n                                yylval->RegexSubst->concatSource('\n');
      \\\/                              yylval->RegexSubst->concatSource('/');
      \\.                               { yylval->RegexSubst->concatSource('\\'); yylval->RegexSubst->concatSource(yytext[1]); }
      [^\n\\/]+                         {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->RegexSubst->concatSource(*(yptr++));
                                        }
      <<EOF>>                           {
                                            yylval->RegexSubst->deref();
                                            QORE_FLEX_DO_EOF
                                        }
}
<regex_trans2>{
      -                                 yylval->Trans->setTargetRange();
      \/                                {
                                           yylloc->restoreFirst();
                                           BEGIN(INITIAL);
                                           yylval->Trans->finishTarget();
                                           return REGEX_TRANS;
                                        }
      \n                                yylval->Trans->concatTarget('\n');
      \\n                               yylval->Trans->concatTarget('\n');
      \\t                               yylval->Trans->concatTarget('\t');
      \\r                               yylval->Trans->concatTarget('\r');
      \\b                               yylval->Trans->concatTarget('\b');
      \\f                               yylval->Trans->concatTarget('\f');
      \\\n                              yylval->Trans->concatTarget('\n');
      \\\/                              yylval->Trans->concatTarget('/');
      \\.                               yylval->Trans->concatTarget(yytext[1]);
      [^\n\\/\-]+                               {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->Trans->concatTarget(*(yptr++));
                                        }
      <<EOF>>                           {
                                           yylval->Trans->deref();
                                           QORE_FLEX_DO_EOF
                                        }
}
<regex_trans1>{
      -                                 yylval->Trans->setSourceRange();
      \/                                BEGIN(regex_trans2); yylval->Trans->finishSource();
      \n                                yylval->Trans->concatSource('\n');
      \\n                               yylval->Trans->concatSource('\n');
      \\t                               yylval->Trans->concatSource('\t');
      \\r                               yylval->Trans->concatSource('\r');
      \\b                               yylval->Trans->concatSource('\b');
      \\f                               yylval->Trans->concatSource('\f');
      \\\n                              yylval->Trans->concatSource('\n');
      \\\/                              yylval->Trans->concatSource('/');
      \\.                               yylval->Trans->concatSource(yytext[1]);
      [^\n\\/\-]+                       {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->Trans->concatSource(*(yptr++));
                                        }
      <<EOF>>                           {
                                           yylval->Trans->deref();
                                           QORE_FLEX_DO_EOF
                                        }
}
<regex_state>{
      \/                                {
                                            // get regex modifiers
                                            int c;
                                            do {
                                                c = yyinput(yyscanner);
                                            } while (isRegexModifier(yylval->Regex, c));
                                            // issue #2096: handle EOF while scanning chars manually
                                            if (!c) {
                                                yylval->Regex->deref();
                                                QORE_FLEX_DO_EOF
                                            } else {
                                                unput(c);
                                                yylloc->restoreFirst();
                                                BEGIN(INITIAL);
                                                yylval->Regex->parse([&] () { return get_loc(yylloc); });
                                                return REGEX;
                                            }
                                        }
      \n                                yylval->Regex->concat('\n');
      \\\/                              yylval->Regex->concat('/');
      \\.                               { yylval->Regex->concat('\\'); yylval->Regex->concat(yytext[1]); }
      [^\n\\/]+                         {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->Regex->concat(*(yptr++));
                                        }
      <<EOF>>                           {
                                           yylval->Regex->deref();
                                           QORE_FLEX_DO_EOF
                                        }
}
<regex_extract_state>{
      \/                                {
                                            // get regex modifiers
                                            int c;
                                            do {
                                                c = yyinput(yyscanner);
                                            } while (isRegexExtractModifier(yylval->Regex, c));
                                            // issue #2096: handle EOF while scanning chars manually
                                            if (!c) {
                                                yylval->Regex->deref();
                                                QORE_FLEX_DO_EOF
                                            } else {
                                                unput(c);
                                                yylloc->restoreFirst();
                                                BEGIN(INITIAL);
                                                yylval->Regex->parse([&] () { return get_loc(yylloc); });
                                                return REGEX_EXTRACT;
                                            }
                                        }
      \n                                yylval->Regex->concat('\n');
      \\\/                              yylval->Regex->concat('/');
      \\.                               {
                                            yylval->Regex->concat('\\');
                                            yylval->Regex->concat(yytext[1]);
                                        }
      [^\n\\/]+                         {
                                           char* yptr = yytext;
                                           while (*yptr)
                                              yylval->Regex->concat(*(yptr++));
                                        }
      <<EOF>>                           {
                                           yylval->Regex->deref();
                                           QORE_FLEX_DO_EOF
                                        }
}
\#                                      BEGIN(line_comment);
<line_comment>{
      [\n\r]                            BEGIN(INITIAL);
      <<EOF>>                           QORE_FLEX_DO_EOF
      [^\n\r]                           // ignore any other character until EOL
}
where                                   return TOK_WHERE;
private                                 return TOK_PRIVATE;
private:internal                        return TOK_PRIVATEINTERNAL;
private:hierarchy                       return TOK_PRIVATEHIERARCHY;
public                                  return TOK_PUBLIC;
abstract                                return TOK_ABSTRACT;
unreference                             return TOK_UNREFERENCE;
new                                     return TOK_NEW;
shift                                   return TOK_SHIFT;
unshift                                 return TOK_UNSHIFT;
do                                      return TOK_DO;
while                                   return TOK_WHILE;
if                                      return TOK_IF;
else                                    return TOK_ELSE;
for                                     return TOK_FOR;
foreach                                 return TOK_FOREACH;
in                                      return TOK_IN;
elements                                return TOK_ELEMENTS;
keys                                    return TOK_KEYS;
sub                                     { new_argv_ref(); return TOK_SUB; }
const                                   return TOK_CONST;
return                                  return TOK_RETURN;
returns                                 return TOK_RETURNS;
thread_local                            return TOK_THREAD_LOCAL;
my                                      return TOK_MY;
our                                     return TOK_OUR;
break                                   return TOK_BREAK;
continue                                return TOK_CONTINUE;
try                                     return TOK_TRY;
throw                                   return TOK_THROW;
rethrow                                 return TOK_RETHROW;
catch                                   return TOK_CATCH;
final                                   return TOK_FINAL;
find                                    return TOK_FIND;
delete                                  return TOK_DELETE;
remove                                  return TOK_REMOVE;
background                              return TOK_BACKGROUND;
synchronized                            return TOK_SYNCHRONIZED;
deprecated                              return TOK_DEPRECATED;
thread_exit                             return TOK_THREAD_EXIT;
exists                                  return TOK_EXISTS;
context                                 return TOK_CONTEXT;
summarize                               return TOK_SUMMARIZE;
subcontext                              return TOK_SUB_CONTEXT;
sortBy                                  return TOK_SORT_BY;
sortDescendingBy                        return TOK_SORT_DESCENDING_BY;
by                                      return TOK_BY;
switch                                  return TOK_SWITCH;
case                                    BEGIN(case_state); return TOK_CASE;
<case_state>{
   \/                                   yylval->Regex = new QoreRegex(); yylloc->saveFirst(); BEGIN(regex_state);
   {WS}+                                /* ignore */
   [^\/]                                yyless(0); BEGIN(INITIAL);
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
default                                 return TOK_DEFAULT;
inherits                                return TOK_INHERITS;
push                                    return TOK_PUSH;
pop                                     return TOK_POP;
splice                                  return TOK_SPLICE;
extract                                 return TOK_EXTRACT;
instanceof                              return TOK_INSTANCEOF;
chomp                                   return TOK_CHOMP;
trim                                    return TOK_TRIM;
on_exit                                 return TOK_ON_EXIT;
on_success                              return TOK_ON_SUCCESS;
on_error                                return TOK_ON_ERROR;
hashdecl                                return TOK_HASHDECL;
transient                               {
                                            if (getProgram()->getParseOptions64() & PO_NO_TRANSIENT) {
                                                yylval->string = strdup("transient"); return IDENTIFIER;
                                            }
                                            return TOK_TRANSIENT;
                                        }
map                                     { push_ignore_numeric_argv_ref(); return TOK_MAP; }
foldr                                   { push_ignore_numeric_argv_ref(); return TOK_FOLDR; }
foldl                                   { push_ignore_numeric_argv_ref(); return TOK_FOLDL; }
select                                  { push_ignore_numeric_argv_ref(); return TOK_SELECT; }
static                                  return TOK_STATIC;
(::(Qore::)?)?NOTHING                   return TOK_NOTHING;
(::(Qore::)?)?NULL                      return TOK_NULL;
(::(Qore::)?)?True                      return TOK_TRUE;
(::(Qore::)?)?False                     return TOK_FALSE;
class\(                                 yylval->string = strdup("class"); return KW_IDENTIFIER_OPENPAREN;
private\(                               yylval->string = strdup("private"); return KW_IDENTIFIER_OPENPAREN;
new\(                                   yylval->string = strdup("new"); return KW_IDENTIFIER_OPENPAREN;
delete\(                                yylval->string = strdup("delete"); return KW_IDENTIFIER_OPENPAREN;
case\(                                  yylval->string = strdup("case"); return KW_IDENTIFIER_OPENPAREN;
chomp\(                                 yylval->string = strdup("chomp"); return KW_IDENTIFIER_OPENPAREN;
final\(                                 yylval->string = strdup("final"); return KW_IDENTIFIER_OPENPAREN;
find\(                                  yylval->string = strdup("find"); return KW_IDENTIFIER_OPENPAREN;
trim\(                                  yylval->string = strdup("trim"); return KW_IDENTIFIER_OPENPAREN;
push\(                                  yylval->string = strdup("push"); return KW_IDENTIFIER_OPENPAREN;
pop\(                                   yylval->string = strdup("pop"); return KW_IDENTIFIER_OPENPAREN;
splice\(                                yylval->string = strdup("splice"); return KW_IDENTIFIER_OPENPAREN;
shift\(                                 yylval->string = strdup("shift"); return KW_IDENTIFIER_OPENPAREN;
unshift\(                               yylval->string = strdup("unshift"); return KW_IDENTIFIER_OPENPAREN;
background\(                            yylval->string = strdup("background"); return KW_IDENTIFIER_OPENPAREN;
exists\(                                yylval->string = strdup("exists"); return KW_IDENTIFIER_OPENPAREN;
map\(                                   yylval->string = strdup("map"); return KW_IDENTIFIER_OPENPAREN;
foldr\(                                 yylval->string = strdup("foldr"); return KW_IDENTIFIER_OPENPAREN;
foldl\(                                 yylval->string = strdup("foldl"); return KW_IDENTIFIER_OPENPAREN;
select\(                                yylval->string = strdup("select"); return KW_IDENTIFIER_OPENPAREN;
inherits\(                              yylval->string = strdup("inherits"); return KW_IDENTIFIER_OPENPAREN;
default{WS}*\(                          yylval->string = strdup("default"); return KW_IDENTIFIER_OPENPAREN;
static{WS}*\(                           yylval->string = strdup("static"); return KW_IDENTIFIER_OPENPAREN;
returns{WS}*\(                          yylval->string = strdup("returns"); return KW_IDENTIFIER_OPENPAREN;
public{WS}*\(                           yylval->string = strdup("public"); return KW_IDENTIFIER_OPENPAREN;
deprecated{WS}*\(                       yylval->string = strdup("deprecated"); return KW_IDENTIFIER_OPENPAREN;
module{WS}*\(                           yylval->string = strdup("module"); return KW_IDENTIFIER_OPENPAREN;
\.returns[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("returns")
\.new[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("new")
\.private[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("private")
\.public[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("public")
\.class[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("class")
\.elements[^A-Za-z_0-9]                 DO_DOT_KW_IDENTIFIER("elements")
\.delete[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("delete")
\.case[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("case")
\.default[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("default")
\.map[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("map")
\.select[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("select")
\.keys[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("keys")
\.chomp[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("chomp")
\.trim[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("trim")
\.push[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("push")
\.pop[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("pop")
\.splice[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("splice")
\.shift[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("shift")
\.unshift[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("unshift")
\.background[^A-Za-z_0-9]               DO_DOT_KW_IDENTIFIER("background")
\.find[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("find")
\.exists[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("exists")
\.foldr[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("foldr")
\.foldl[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("foldl")
\.static[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("static")
\.abstract[^A-Za-z_0-9]                 DO_DOT_KW_IDENTIFIER("abstract")
\.deprecated[^A-Za-z_0-9]               DO_DOT_KW_IDENTIFIER("deprecated")
\.summarize[^A-Za-z_0-9]                DO_DOT_KW_IDENTIFIER("summarize")
\.sub[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("sub")
\.subcontext[^A-Za-z_0-9]               DO_DOT_KW_IDENTIFIER("subcontext")
\.sortBy[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("sortBy")
\.sortDescendingBy[^A-Za-z_0-9]         DO_DOT_KW_IDENTIFIER("sortDescendingBy")
\.by[^A-Za-z_0-9]                       DO_DOT_KW_IDENTIFIER("by")
\.switch[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("switch")
\.return[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("return")
\.rethrow[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("rethrow")
\.break[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("break")
\.catch[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("catch")
\.const[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("const")
\.context[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("context")
\.continue[^A-Za-z_0-9]                 DO_DOT_KW_IDENTIFIER("continue")
\.do[^A-Za-z_0-9]                       DO_DOT_KW_IDENTIFIER("do")
\.else[^A-Za-z_0-9]                     DO_DOT_KW_IDENTIFIER("else")
\.final[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("final")
\.for[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("for")
\.foreach[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("foreach")
\.if[^A-Za-z_0-9]                       DO_DOT_KW_IDENTIFIER("if")
\.in[^A-Za-z_0-9]                       DO_DOT_KW_IDENTIFIER("in")
\.inherits[^A-Za-z_0-9]                 DO_DOT_KW_IDENTIFIER("inherits")
\.instanceof[^A-Za-z_0-9]               DO_DOT_KW_IDENTIFIER("instanceof")
\.namespace[^A-Za-z_0-9]                DO_DOT_KW_IDENTIFIER("namespace")
\.my[^A-Za-z_0-9]                       DO_DOT_KW_IDENTIFIER("my")
\.on_error[^A-Za-z_0-9]                 DO_DOT_KW_IDENTIFIER("on_error")
\.on_exit[^A-Za-z_0-9]                  DO_DOT_KW_IDENTIFIER("on_exit")
\.on_success[^A-Za-z_0-9]               DO_DOT_KW_IDENTIFIER("on_success")
\.our[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("our")
\.synchronized[^A-Za-z_0-9]             DO_DOT_KW_IDENTIFIER("synchronized")
\.thread_exit[^A-Za-z_0-9]              DO_DOT_KW_IDENTIFIER("thread_exit")
\.throw[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("throw")
\.try[^A-Za-z_0-9]                      DO_DOT_KW_IDENTIFIER("try")
\.where[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("where")
\.while[^A-Za-z_0-9]                    DO_DOT_KW_IDENTIFIER("while")
\.module[^A-Za-z_0-9]                   DO_DOT_KW_IDENTIFIER("module")

{YEAR}-{MONTH}-{DAY}T{HOUR}:{MSEC}(:{MSEC}(\.{US})?)?{TZ}?   yylval->datetime = makeDateTime(yytext, true); return DATETIME;
{YEAR}-{MONTH}-{DAY}-{HOUR}:{MSEC}:{MSEC}(\.{US})?{TZ}?      yylval->datetime = makeDateTime(yytext, true); return DATETIME;
{YEAR}-{MONTH}-{DAY}?{TZ}?                                   yylval->datetime = makeDateTime(yytext, false); return DATETIME;
{HOUR}:{MSEC}(:{MSEC}(\.{US})?)?{TZ}?   yylval->datetime = makeTime(yytext); return DATETIME;
P{YEAR}-{D2}-{D2}(T{D2}:{D2}(:{D2}(\.{US})?)?)?   yylval->datetime = makeRelativeDateTime(yytext+1); return DATETIME;
PT{D2}:{D2}(:{D2}(\.{US})?)?            yylval->datetime = makeRelativeTime(yytext+2); return DATETIME;
P{D2}:{D2}(:{D2}(\.{US})?)?             yylval->datetime = makeRelativeTime(yytext+1); return DATETIME;
{RDATETIME}                             yylval->datetime = new DateTimeNode(yytext); return DATETIME;
{RDATE}                                 yylval->datetime = new DateTimeNode(yytext); return DATETIME;
{RTIME}                                 yylval->datetime = new DateTimeNode(yytext); return DATETIME;
({WORD}::)+{WORD}                       yylval->string = strdup(yytext); return SCOPED_REF;
(::{WORD})+                             yylval->string = strdup(yytext); return SCOPED_REF;
({WORD}::)+\${WORD}                     yylval->string = make_scoped_vref(yytext); return SCOPED_VREF;
(::{WORD})?::\${WORD}                   yylval->string = make_scoped_vref(yytext); return SCOPED_VREF;
cast<{WS}*({WORD}::)*{WORD}{WS}*(<[a-zA-Z<>:_0-9 \t,\*]+>)?{WS}*> yylval->string = make_cast(yytext); return QORE_CAST;
cast<\*{WS}*({WORD}::)*{WORD}{WS}*(<[a-zA-Z<>:_0-9 \t,\*]+>)?{WS}*> yylval->string = make_cast(yytext + 1); return QORE_OR_NOTHING_CAST;
({WORD}::)+\$\.{WORD}                   yylval->nscope = new NamedScope(strdup(yytext)); yylval->nscope->fixBCCall(); return BASE_CLASS_CALL;
@[Nn][Aa][Nn]@                          {
#ifdef NAN
                                           yylval->decimal = NAN;
#else
                                           yylval->decimal = q_strtod("NAN");
#endif
                                           return QFLOAT;
                                        }
@[Ii][Nn][Ff]@                          {
#ifdef INFINITY
                                           yylval->decimal = INFINITY;
#else
                                           yylval->decimal = q_strtod("INFINITY");
#endif
                                           return QFLOAT;
                                        }
{DIGIT}+"."{DIGIT}+                     yylval->decimal = q_strtod(yytext); return QFLOAT;
{DIGIT}+[eE][+-]?{DIGIT}+               yylval->decimal = q_strtod(yytext); return QFLOAT;
{DIGIT}+"."{DIGIT}+[eE][+-]?{DIGIT}+    yylval->decimal = q_strtod(yytext); return QFLOAT;
@[Nn][Aa][Nn]@n                         yylval->num = new QoreNumberNode("@nan@"); return NUMBER;
@[Ii][Nn][Ff]@n                         yylval->num = new QoreNumberNode("@inf@"); return NUMBER;
{DIGIT}+n                               yylval->num = new QoreNumberNode(yytext); return NUMBER;
{DIGIT}+"."{DIGIT}+n                    yylval->num = new QoreNumberNode(yytext); return NUMBER;
{DIGIT}+[eE][+-]?{DIGIT}+n              yylval->num = new QoreNumberNode(yytext); return NUMBER;
{DIGIT}+"."{DIGIT}+[eE][+-]?{DIGIT}+n   yylval->num = new QoreNumberNode(yytext); return NUMBER;
0[0-7]+                                 yylval->integer = strtoll(yytext+1, 0, 8); return INTEGER;
{DIGIT}+                                yylval->integer = parse_get_integer(yylloc, yytext); return INTEGER;
{DIGIT}+Y                               yylval->datetime = makeYears(strtol(yytext, 0, 10));   return DATETIME;
{DIGIT}+M                               yylval->datetime = makeMonths(strtol(yytext, 0, 10));  return DATETIME;
{DIGIT}+D                               yylval->datetime = makeDays(strtol(yytext, 0, 10));    return DATETIME;
{DIGIT}+h                               yylval->datetime = makeHours(strtol(yytext, 0, 10));   return DATETIME;
{DIGIT}+ms                              yylval->datetime = makeMilliseconds(strtol(yytext, 0, 10)); return DATETIME;
{DIGIT}+us                              yylval->datetime = makeMicroseconds(strtol(yytext, 0, 10)); return DATETIME;
{DIGIT}+m                               yylval->datetime = makeMinutes(strtol(yytext, 0, 10)); return DATETIME;
{DIGIT}+(\.{DIGIT}+)?s                  yylval->datetime = makeSeconds(yytext); return DATETIME;
{HEX_CONST}                             yylval->integer = strtoll(yytext, 0, 16); return INTEGER;
{BINARY}                                yylval->binary = parseHex(get_loc(yylloc), yytext + 1, strlen(yytext + 1) - 1); return BINARY;
\$\.{WORD}                              yylval->string = strdup(yytext + 2); return SELF_REF;
\${WORD}                                yylval->string = strdup(yytext + 1); return VAR_REF;
class{WS}+({WORD}::)+{WORD}             { yylval->nscope = new NamedScope(trim(yytext + 6)); parse_push_name(yylval->nscope->getIdentifier()); return CLASS_SCOPED_REF; }
class{WS}+(::{WORD})+                   { yylval->nscope = new NamedScope(trim(yytext + 6)); parse_push_name(yylval->nscope->getIdentifier()); return CLASS_SCOPED_REF; }
class{WS}+{WORD}                        { yylval->string = trim(yytext + 6); parse_push_name(yylval->string); return CLASS_STRING; }
namespace{WS}+{WORD}                    { const char* ns = trim_inplace(yytext + 10); parse_push_ns_name(ns); return TOK_NAMESPACE; }
module{WS}+{WORD}                       { const char* mod = trim_inplace(yytext + 7); parse_set_module_def_context_name(mod); return TOK_MODULE; }
{WORD}                                  yylval->string = strdup(yytext); return IDENTIFIER;
\<{WORD}\>\{                            {
                                            yylval->string = strdup(yytext + 1);
                                            yylval->string[strlen(yylval->string) - 2] = '\0';
                                            return HASHDECL_IDENTIFIER_OPENCURLY;
                                        }
hash\<{WORD}\>\{                        {
                                            yylval->string = strdup(yytext + 5);
                                            yylval->string[strlen(yylval->string) - 2] = '\0';
                                            return HASHDECL_IDENTIFIER_OPENCURLY;
                                        }
{WORD}<[a-zA-Z\*<:_0-9 \t,]+>           {
                                           if (angle_balanced(yytext)) {
                                               yylval->string = strdup(yytext);
                                               return ANGLE_IDENTIFIER;
                                           }

                                           yylval->String = new QoreStringNode(yytext);
                                           yylloc->saveFirst();
                                           BEGIN(angle_state);
}
<angle_state>{
      >                                 {
                                           yylval->String->concat('>');
                                           if (angle_balanced(yylval->String->c_str())) {
                                              char* str = yylval->String->giveBuffer();
                                              yylval->String->deref();
                                              yylval->string = str;
                                              yylloc->restoreFirst();
                                              BEGIN(INITIAL);
                                              return ANGLE_IDENTIFIER;
                                           }
                                        }
      [^>]                              yylval->String->concat(yytext);
      <<EOF>>                           {
                                           yylval->String->deref();
                                           QORE_FLEX_DO_EOF
                                        }
}
\%{WORD}                                yylval->string = strdup(yytext + 1); return CONTEXT_REF;
\%{WORD}\:{WORD}                        yylval->string = strdup(yytext + 1); return COMPLEX_CONTEXT_REF;
\%\%                                    return TOK_CONTEXT_ROW;
\`[^`]*\`                               yylval->string = strdup(remove_quotes(yytext)); return BACKQUOTE;
\'[^\']*\'                              yylval->String = new QoreStringNode(remove_quotes(yytext)); return QUOTED_WORD;
\$\$                                    yylval->implicit_arg = new QoreImplicitArgumentNode(get_loc(yylloc), -1); return IMPLICIT_ARG_REF;
\$[0-9][0-9]*                           yylval->implicit_arg = new QoreImplicitArgumentNode(get_loc(yylloc), strtol(yytext + 1, 0, 0)); return IMPLICIT_ARG_REF;
\$#                                     return IMPLICIT_ELEMENT;
\?\?                                    return NULL_COALESCING;
\?\*                                    return VALUE_COALESCING;
\<=                                     return LOGICAL_LE;
\>=                                     return LOGICAL_GE;
\!=                                     return LOGICAL_NE;
\<\>                                    return LOGICAL_NE;
===                                     return ABSOLUTE_EQ;
\!==                                    return ABSOLUTE_NE;
==                                      return LOGICAL_EQ;
:=                                      return WEAK_ASSIGNMENT;
\<=\>                                   return LOGICAL_CMP;
&&                                      return LOGICAL_AND;
\|\|                                    return (getProgram()->getParseOptions64() & PO_BROKEN_LOGIC_PRECEDENCE) ? BROKEN_LOGICAL_OR : LOGICAL_OR;
\|                                      return (getProgram()->getParseOptions64() & PO_BROKEN_LOGIC_PRECEDENCE) ? BROKEN_BINARY_OR : '|';
\^                                      return (getProgram()->getParseOptions64() & PO_BROKEN_LOGIC_PRECEDENCE) ? BROKEN_BINARY_XOR : '^';
\>\>                                    return SHIFT_RIGHT;
\<\<                                    return SHIFT_LEFT;
\+\+                                    return P_INCREMENT;
--                                      return P_DECREMENT;
\+=                                     return PLUS_EQUALS;
-=                                      return MINUS_EQUALS;
&=                                      return AND_EQUALS;
\|=                                     return OR_EQUALS;
\%=                                     return MODULA_EQUALS;
\*=                                     return MULTIPLY_EQUALS;
\/=                                     return DIVIDE_EQUALS;
\^=                                     return XOR_EQUALS;
\>\>=                                   return SHIFT_RIGHT_EQUALS;
\<\<=                                   return SHIFT_LEFT_EQUALS;
\.\.                                    return RANGE;
\.\.\.                                  return ELLIPSES;

\<{WS}+=                                check_broken_op(yylloc, yytext, "<="); return LOGICAL_LE;
\>{WS}+=                                check_broken_op(yylloc, yytext, ">="); return LOGICAL_GE;
\!{WS}+=                                check_broken_op(yylloc, yytext, "!="); return LOGICAL_NE;
\<{WS}+\>                               check_broken_op(yylloc, yytext, "<>"); return LOGICAL_NE;
={WS}+=                                 check_broken_op(yylloc, yytext, "=="); return LOGICAL_EQ;

={WS}+={WS}*=                           check_broken_op(yylloc, yytext, "==="); return ABSOLUTE_EQ;
={WS}*={WS}+=                           check_broken_op(yylloc, yytext, "==="); return ABSOLUTE_EQ;
\!{WS}+={WS}*=                          check_broken_op(yylloc, yytext, "!=="); return ABSOLUTE_NE;
\!{WS}*={WS}+=                          check_broken_op(yylloc, yytext, "!=="); return ABSOLUTE_NE;
\<{WS}+={WS}*\>                         check_broken_op(yylloc, yytext, "<=>"); return LOGICAL_CMP;
\<{WS}*={WS}+\>                         check_broken_op(yylloc, yytext, "<=>"); return LOGICAL_CMP;

&{WS}+&                                 check_broken_op(yylloc, yytext, "&&"); return LOGICAL_AND;
\|{WS}+\|                               check_broken_op(yylloc, yytext, "||"); return (getProgram()->getParseOptions64() & PO_BROKEN_LOGIC_PRECEDENCE) ? BROKEN_LOGICAL_OR : LOGICAL_OR;
\>{WS}+\>                               check_broken_op(yylloc, yytext, ">>"); return SHIFT_RIGHT;
\<{WS}+\<                               check_broken_op(yylloc, yytext, "<<"); return SHIFT_LEFT;
\+{WS}+=                                check_broken_op(yylloc, yytext, "+="); return PLUS_EQUALS;
-{WS}+=                                 check_broken_op(yylloc, yytext, "-="); return MINUS_EQUALS;
&{WS}+=                                 check_broken_op(yylloc, yytext, "&="); return AND_EQUALS;
\|{WS}+=                                check_broken_op(yylloc, yytext, "|="); return OR_EQUALS;
\%{WS}+=                                check_broken_op(yylloc, yytext, "%="); return MODULA_EQUALS;
\*{WS}+=                                check_broken_op(yylloc, yytext, "*="); return MULTIPLY_EQUALS;
\/{WS}+=                                check_broken_op(yylloc, yytext, "/="); return DIVIDE_EQUALS;
\^{WS}+=                                check_broken_op(yylloc, yytext, "^="); return XOR_EQUALS;
\>{WS}+\>{WS}*=                         check_broken_op(yylloc, yytext, ">>="); return SHIFT_RIGHT_EQUALS;
\>{WS}*\>{WS}+=                         check_broken_op(yylloc, yytext, ">>="); return SHIFT_RIGHT_EQUALS;
\<{WS}+\<{WS}*=                         check_broken_op(yylloc, yytext, "<<="); return SHIFT_LEFT_EQUALS;
\<{WS}*\<{WS}+=                         check_broken_op(yylloc, yytext, "<<="); return SHIFT_LEFT_EQUALS;

\/\*                                    BEGIN(block_comment1);
<block_comment1>{
      \*                                BEGIN(block_comment2);
      <<EOF>>                           {
                                           // do not throw an extra exception here; "unexpected $end" raised due to premature EOF
                                           QORE_FLEX_DO_EOF;
                                        }
      [^\*]                             // ignore other input
}
<block_comment2>{
      \/                                BEGIN(INITIAL);
      \*                                // ignore
      <<EOF>>                           {
                                           // do not throw an extra exception here; "unexpected $end" raised due to premature EOF
                                           QORE_FLEX_DO_EOF;
                                        }
      [^\/\*]                           BEGIN(block_comment1);
}
<regex_googleplex>{
   s\/                                  yylval->RegexSubst = new QoreRegexSubst; yylloc->saveFirst(); BEGIN(regex_subst1);
   x\/                                  yylval->Regex = new QoreRegex; yylloc->saveFirst(); BEGIN(regex_extract_state);
   tr\/                                 yylval->Trans = new QoreTransliteration(get_loc(yylloc)); yylloc->saveFirst(); BEGIN(regex_trans1);
   m\/                                  yylval->Regex = new QoreRegex; yylloc->saveFirst(); BEGIN(regex_state);
   \/                                   yylval->Regex = new QoreRegex; yylloc->saveFirst(); BEGIN(regex_state);
   {WSNL}+                              /* ignore whitespace */
   [^sxmt\/]                            parse_error(*get_loc(yylloc), "missing regular expression after =~"); BEGIN(INITIAL);
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
<regex_negative_universe>{
   m\/                                  yylval->Regex = new QoreRegex; yylloc->saveFirst(); BEGIN(regex_state);
   \/                                   yylval->Regex = new QoreRegex; yylloc->saveFirst(); BEGIN(regex_state);
   {WSNL}+                              /* ignore whitespace */
   [^m\/]                               parse_error(*get_loc(yylloc), "missing regular expression after !~"); BEGIN(INITIAL);
   <<EOF>>                              {
                                           QORE_FLEX_DO_EOF
                                        }
}
=\~                                     BEGIN(regex_googleplex); return REGEX_MATCH;
\!\~                                    BEGIN(regex_negative_universe); return REGEX_NMATCH;
{WSNL}+                                 /* ignore whitespace */
.                                       return yytext[0];
%%
