/* Copyright (c) 2006, Sun Microsystems, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Sun Microsystems, Inc. nor the names of its
 *       contributors may be used to endorse or promote products derived from
 *       this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.javacc.parser;

import java.io.File;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Vector;

/**
 * Generate lexer.
 */
public class LexGen extends JavaCCGlobals implements JavaCCParserConstants
{
  static private java.io.PrintWriter ostr;
  static private String staticString;
  static private String tokMgrClassName;

  // Hashtable of vectors
  static Hashtable allTpsForState = new Hashtable();
  public static int lexStateIndex = 0;
  static int[] kinds;
  public static int maxOrdinal = 1;
  public static String lexStateSuffix;
  static String[] newLexState;
  public static int[] lexStates;
  public static boolean[] ignoreCase;
  public static Action[] actions;
  public static Hashtable initStates = new Hashtable();
  public static int stateSetSize;
  public static int maxLexStates;
  public static String[] lexStateName;
  static NfaState[] singlesToSkip;
  public static long[] toSkip;
  public static long[] toSpecial;
  public static long[] toMore;
  public static long[] toToken;
  public static int defaultLexState;
  public static RegularExpression[] rexprs;
  public static int[] maxLongsReqd;
  public static int[] initMatch;
  public static int[] canMatchAnyChar;
  public static boolean hasEmptyMatch;
  public static boolean[] canLoop;
  public static boolean[] stateHasActions;
  public static boolean hasLoop = false;
  public static boolean[] canReachOnMore;
  public static boolean[] hasNfa;
  public static boolean[] mixed;
  public static NfaState initialState;
  public static int curKind;
  static boolean hasSkipActions = false;
  static boolean hasMoreActions = false;
  static boolean hasTokenActions = false;
  static boolean hasSpecial = false;
  static boolean hasSkip = false;
  static boolean hasMore = false;
  public static RegularExpression curRE;
  public static boolean keepLineCol;

  static void PrintClassHead()
  {
    int i, j;

    try {
      File tmp = new File(Options.getOutputDirectory(), tokMgrClassName + ".java");
      ostr = new java.io.PrintWriter(
          new java.io.BufferedWriter(
              new java.io.FileWriter(tmp),
              8092
          )
      );
      Vector tn = (Vector)(toolNames.clone());
      tn.addElement(toolName);

      ostr.println("/* " + getIdString(tn, tokMgrClassName + ".java") + " */");

      int l = 0, kind;
      i = 1;
      for (;;)
      {
        if (cu_to_insertion_point_1.size() <= l)
          break;

        kind = ((Token)cu_to_insertion_point_1.elementAt(l)).kind;
        if(kind == PACKAGE || kind == IMPORT) {
          for (; i < cu_to_insertion_point_1.size(); i++) {
            kind = ((Token)cu_to_insertion_point_1.elementAt(i)).kind;
            if (kind == SEMICOLON ||
                kind == ABSTRACT ||
                kind == FINAL ||
                kind == PUBLIC ||
                kind == CLASS ||
                kind == INTERFACE)
            {
              cline = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginLine;
              ccol = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginColumn;
              for (j = l; j < i; j++) {
                printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
              }
              if (kind == SEMICOLON)
                printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
              ostr.println("");
              break;
            }
          }
          l = ++i;
        }
        else
          break;
      }

      ostr.println("");
      ostr.println("/** Token Manager. */");
      ostr.println("public class " + tokMgrClassName + " implements " +
          cu_name + "Constants");
      ostr.println("{"); // }
    }
    catch (java.io.IOException err) {
      JavaCCErrors.semantic_error("Could not create file : " + tokMgrClassName + ".java\n");
      throw new Error();
    }

    if (token_mgr_decls != null && token_mgr_decls.size() > 0)
    {
      Token t = (Token)token_mgr_decls.elementAt(0);
      boolean commonTokenActionSeen = false;
      boolean commonTokenActionNeeded = Options.getCommonTokenAction();

      printTokenSetup((Token)token_mgr_decls.elementAt(0));
      ccol = 1;

      for (j = 0; j < token_mgr_decls.size(); j++)
      {
        t = (Token)token_mgr_decls.elementAt(j);
        if (t.kind == IDENTIFIER &&
            commonTokenActionNeeded &&
            !commonTokenActionSeen)
          commonTokenActionSeen = t.image.equals("CommonTokenAction");

        printToken(t, ostr);
      }

      ostr.println("");
      if (commonTokenActionNeeded && !commonTokenActionSeen)
        JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +
            "But it appears you have not defined the method :\n"+
            "      " + staticString + "void CommonTokenAction(Token t)\n" +
        "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");

    }
    else if (Options.getCommonTokenAction())
    {
      JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. " +
          "But you have not defined the method :\n"+
          "      " + staticString + "void CommonTokenAction(Token t)\n" +
      "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
    }

    ostr.println("");
    ostr.println("  /** Debug output. */");
    ostr.println("  public " + staticString + " java.io.PrintStream debugStream = System.out;");
    ostr.println("  /** Set debug output. */");
    ostr.println("  public " + staticString + " void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }");

    if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
      ostr.println("");
      ostr.println("  /** The parser. */");
      ostr.println("  public " + cu_name + " parser = null;");
    }
  }

  static void DumpDebugMethods()
  {

    ostr.println("  " + staticString + " int kindCnt = 0;");
    ostr.println("  protected " + staticString + " final String jjKindsForBitVector(int i, long vec)");
    ostr.println("  {");
    ostr.println("    String retVal = \"\";");
    ostr.println("    if (i == 0)");
    ostr.println("       kindCnt = 0;");
    ostr.println("    for (int j = 0; j < 64; j++)");
    ostr.println("    {");
    ostr.println("       if ((vec & (1L << j)) != 0L)");
    ostr.println("       {");
    ostr.println("          if (kindCnt++ > 0)");
    ostr.println("             retVal += \", \";");
    ostr.println("          if (kindCnt % 5 == 0)");
    ostr.println("             retVal += \"\\n     \";");
    ostr.println("          retVal += tokenImage[i * 64 + j];");
    ostr.println("       }");
    ostr.println("    }");
    ostr.println("    return retVal;");
    ostr.println("  }");
    ostr.println("");

    ostr.println("  protected " + staticString + " final String jjKindsForStateVector(" +
    "int lexState, int[] vec, int start, int end)");
    ostr.println("  {");
    ostr.println("    boolean[] kindDone = new boolean[" + maxOrdinal + "];");
    ostr.println("    String retVal = \"\";");
    ostr.println("    int cnt = 0;");
    ostr.println("    for (int i = start; i < end; i++)");
    ostr.println("    {");
    ostr.println("     if (vec[i] == -1)");
    ostr.println("       continue;");
    ostr.println("     int[] stateSet = statesForState[curLexState][vec[i]];");
    ostr.println("     for (int j = 0; j < stateSet.length; j++)");
    ostr.println("     {");
    ostr.println("       int state = stateSet[j];");
    ostr.println("       if (!kindDone[kindForState[lexState][state]])");
    ostr.println("       {");
    ostr.println("          kindDone[kindForState[lexState][state]] = true;");
    ostr.println("          if (cnt++ > 0)");
    ostr.println("             retVal += \", \";");
    ostr.println("          if (cnt % 5 == 0)");
    ostr.println("             retVal += \"\\n     \";");
    ostr.println("          retVal += tokenImage[kindForState[lexState][state]];");
    ostr.println("       }");
    ostr.println("     }");
    ostr.println("    }");
    ostr.println("    if (cnt == 0)");
    ostr.println("       return \"{  }\";");
    ostr.println("    else");
    ostr.println("       return \"{ \" + retVal + \" }\";");
    ostr.println("  }");
    ostr.println("");
  }

  static void BuildLexStatesTable()
  {
    Enumeration e = rexprlist.elements();
    TokenProduction tp;
    int i;

    String[] tmpLexStateName = new String[lexstate_I2S.size()];
    while (e.hasMoreElements())
    {
      tp = (TokenProduction)e.nextElement();
      Vector respecs = tp.respecs;
      Vector tps;

      for (i = 0; i < tp.lexStates.length; i++)
      {
        if ((tps = (Vector)allTpsForState.get(tp.lexStates[i])) == null)
        {
          tmpLexStateName[maxLexStates++] = tp.lexStates[i];
          allTpsForState.put(tp.lexStates[i], tps = new Vector());
        }

        tps.addElement(tp);
      }

      if (respecs == null || respecs.size() == 0)
        continue;

      RegularExpression re;
      for (i = 0; i < respecs.size(); i++)
        if (maxOrdinal <= (re = ((RegExprSpec)respecs.elementAt(i)).rexp).ordinal)
          maxOrdinal = re.ordinal + 1;
    }

    kinds = new int[maxOrdinal];
    toSkip = new long[maxOrdinal / 64 + 1];
    toSpecial = new long[maxOrdinal / 64 + 1];
    toMore = new long[maxOrdinal / 64 + 1];
    toToken = new long[maxOrdinal / 64 + 1];
    toToken[0] = 1L;
    actions = new Action[maxOrdinal];
    actions[0] = actForEof;
    hasTokenActions = actForEof != null;
    initStates = new Hashtable();
    canMatchAnyChar = new int[maxLexStates];
    canLoop = new boolean[maxLexStates];
    stateHasActions = new boolean[maxLexStates];
    lexStateName = new String[maxLexStates];
    singlesToSkip = new NfaState[maxLexStates];
    System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);

    for (i = 0; i < maxLexStates; i++)
      canMatchAnyChar[i] = -1;

    hasNfa = new boolean[maxLexStates];
    mixed = new boolean[maxLexStates];
    maxLongsReqd = new int[maxLexStates];
    initMatch = new int[maxLexStates];
    newLexState = new String[maxOrdinal];
    newLexState[0] = nextStateForEof;
    hasEmptyMatch = false;
    lexStates = new int[maxOrdinal];
    ignoreCase = new boolean[maxOrdinal];
    rexprs = new RegularExpression[maxOrdinal];
    RStringLiteral.allImages = new String[maxOrdinal];
    canReachOnMore = new boolean[maxLexStates];
  }

  static int GetIndex(String name)
  {
    for (int i = 0; i < lexStateName.length; i++)
      if (lexStateName[i] != null && lexStateName[i].equals(name))
        return i;

    throw new Error(); // Should never come here
  }

  public static void AddCharToSkip(char c, int kind)
  {
    singlesToSkip[lexStateIndex].AddChar(c);
    singlesToSkip[lexStateIndex].kind = kind;
  }

  public static void start()
  {
    if (!Options.getBuildTokenManager() ||
        Options.getUserTokenManager() ||
        JavaCCErrors.get_error_count() > 0)
      return;

    keepLineCol = Options.getKeepLineColumn();
    Vector choices = new Vector();
    Enumeration e;
    TokenProduction tp;
    int i, j;

    staticString = (Options.getStatic() ? "static " : "");
    tokMgrClassName = cu_name + "TokenManager";

    PrintClassHead();
    BuildLexStatesTable();

    e = allTpsForState.keys();

    boolean ignoring = false;

    while (e.hasMoreElements())
    {
      NfaState.ReInit();
      RStringLiteral.ReInit();

      String key = (String)e.nextElement();

      lexStateIndex = GetIndex(key);
      lexStateSuffix = "_" + lexStateIndex;
      Vector allTps = (Vector)allTpsForState.get(key);
      initStates.put(key, initialState = new NfaState());
      ignoring = false;

      singlesToSkip[lexStateIndex] = new NfaState();
      singlesToSkip[lexStateIndex].dummy = true;

      if (key.equals("DEFAULT"))
        defaultLexState = lexStateIndex;

      for (i = 0; i < allTps.size(); i++)
      {
        tp = (TokenProduction)allTps.elementAt(i);
        int kind = tp.kind;
        boolean ignore = tp.ignoreCase;
        Vector rexps = tp.respecs;

        if (i == 0)
          ignoring = ignore;

        for (j = 0; j < rexps.size(); j++)
        {
          RegExprSpec respec = (RegExprSpec)rexps.elementAt(j);
          curRE = respec.rexp;

          rexprs[curKind = curRE.ordinal] = curRE;
          lexStates[curRE.ordinal] = lexStateIndex;
          ignoreCase[curRE.ordinal] = ignore;

          if (curRE.private_rexp)
          {
            kinds[curRE.ordinal] = -1;
            continue;
          }

          if (curRE instanceof RStringLiteral &&
              !((RStringLiteral)curRE).image.equals(""))
          {
            ((RStringLiteral)curRE).GenerateDfa(ostr, curRE.ordinal);
            if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)
              mixed[lexStateIndex] = true;
          }
          else if (curRE.CanMatchAnyChar())
          {
            if (canMatchAnyChar[lexStateIndex] == -1 ||
                canMatchAnyChar[lexStateIndex] > curRE.ordinal)
              canMatchAnyChar[lexStateIndex] = curRE.ordinal;
          }
          else
          {
            Nfa temp;

            if (curRE instanceof RChoice)
              choices.addElement(curRE);

            temp = curRE.GenerateNfa(ignore);
            temp.end.isFinal = true;
            temp.end.kind = curRE.ordinal;
            initialState.AddMove(temp.start);
          }

          if (kinds.length < curRE.ordinal)
          {
            int[] tmp = new int[curRE.ordinal + 1];

            System.arraycopy(kinds, 0, tmp, 0, kinds.length);
            kinds = tmp;
          }
          //System.out.println("   ordina : " + curRE.ordinal);

          kinds[curRE.ordinal] = kind;

          if (respec.nextState != null &&
              !respec.nextState.equals(lexStateName[lexStateIndex]))
            newLexState[curRE.ordinal] = respec.nextState;

          if (respec.act != null && respec.act.action_tokens != null &&
              respec.act.action_tokens.size() > 0)
            actions[curRE.ordinal] = respec.act;

          switch(kind)
          {
          case TokenProduction.SPECIAL :
            hasSkipActions |= (actions[curRE.ordinal] != null) ||
            (newLexState[curRE.ordinal] != null);
            hasSpecial = true;
            toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
            toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
            break;
          case TokenProduction.SKIP :
            hasSkipActions |= (actions[curRE.ordinal] != null);
            hasSkip = true;
            toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
            break;
          case TokenProduction.MORE :
            hasMoreActions |= (actions[curRE.ordinal] != null);
            hasMore = true;
            toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);

            if (newLexState[curRE.ordinal] != null)
              canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;
            else
              canReachOnMore[lexStateIndex] = true;

            break;
          case TokenProduction.TOKEN :
            hasTokenActions |= (actions[curRE.ordinal] != null);
            toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
            break;
          }
        }
      }

      // Generate a static block for initializing the nfa transitions
      NfaState.ComputeClosures();

      for (i = 0; i < initialState.epsilonMoves.size(); i++)
        ((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();

      if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))
      {
        initialState.GenerateCode();
        initialState.GenerateInitMoves(ostr);
      }

      if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)
      {
        if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||
            (toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
          hasSkipActions = true;
        else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
          hasMoreActions = true;
        else
          hasTokenActions = true;

        if (initMatch[lexStateIndex] == 0 ||
            initMatch[lexStateIndex] > initialState.kind)
        {
          initMatch[lexStateIndex] = initialState.kind;
          hasEmptyMatch = true;
        }
      }
      else if (initMatch[lexStateIndex] == 0)
        initMatch[lexStateIndex] = Integer.MAX_VALUE;

      RStringLiteral.FillSubString();

      if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
        RStringLiteral.GenerateNfaStartStates(ostr, initialState);

      RStringLiteral.DumpDfaCode(ostr);

      if (hasNfa[lexStateIndex])
        NfaState.DumpMoveNfa(ostr);

      if (stateSetSize < NfaState.generatedStates)
        stateSetSize = NfaState.generatedStates;
    }

    for (i = 0; i < choices.size(); i++)
      ((RChoice)choices.elementAt(i)).CheckUnmatchability();

    NfaState.DumpStateSets(ostr);
    CheckEmptyStringMatch();
    NfaState.DumpNonAsciiMoveMethods(ostr);
    RStringLiteral.DumpStrLiteralImages(ostr);
    DumpStaticVarDeclarations();
    DumpFillToken();
    DumpGetNextToken();

    if (Options.getDebugTokenManager())
    {
      NfaState.DumpStatesForKind(ostr);
      DumpDebugMethods();
    }

    if (hasLoop)
    {
      ostr.println(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");
      ostr.println(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");
      ostr.println(staticString + "boolean[] jjbeenHere = new boolean[" + maxLexStates + "];");
    }

    if (hasSkipActions)
      DumpSkipActions();
    if (hasMoreActions)
      DumpMoreActions();
    if (hasTokenActions)
      DumpTokenActions();

    NfaState.PrintBoilerPlate(ostr);
    ostr.println(/*{*/ "}");
    ostr.close();
  }

  static void CheckEmptyStringMatch()
  {
    int i, j, k, len;
    boolean[] seen = new boolean[maxLexStates];
    boolean[] done = new boolean[maxLexStates];
    String cycle;
    String reList;

    Outer:
      for (i = 0; i < maxLexStates; i++)
      {
        if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||
            canMatchAnyChar[i] != -1)
          continue;

        done[i] = true;
        len = 0;
        cycle = "";
        reList = "";

        for (k = 0; k < maxLexStates; k++)
          seen[k] = false;

        j = i;
        seen[i] = true;
        cycle += lexStateName[j] + "-->";
        while (newLexState[initMatch[j]] != null)
        {
          cycle += newLexState[initMatch[j]];
          if (seen[j = GetIndex(newLexState[initMatch[j]])])
            break;

          cycle += "-->";
          done[j] = true;
          seen[j] = true;
          if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||
              canMatchAnyChar[j] != -1)
            continue Outer;
          if (len != 0)
            reList += "; ";
          reList += "line " + rexprs[initMatch[j]].line + ", column " +
          rexprs[initMatch[j]].column;
          len++;
        }

        if (newLexState[initMatch[j]] == null)
          cycle += lexStateName[lexStates[initMatch[j]]];

        for (k = 0; k < maxLexStates; k++)
          canLoop[k] |= seen[k];

        hasLoop = true;
        if (len == 0)
          JavaCCErrors.warning(rexprs[initMatch[i]],
              "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
                  ? "" : (" for " + rexprs[initMatch[i]].label)) +
                  " can be matched by the empty string (\"\") in lexical state " +
                  lexStateName[i] + ". This can result in an endless loop of " +
          "empty string matches.");
        else
        {
          JavaCCErrors.warning(rexprs[initMatch[i]],
              "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
                  ? "" : (" for " + rexprs[initMatch[i]].label)) +
                  " can be matched by the empty string (\"\") in lexical state " +
                  lexStateName[i] + ". This regular expression along with the " +
                  "regular expressions at " + reList + " forms the cycle \n   " +
                  cycle + "\ncontaining regular expressions with empty matches." +
          " This can result in an endless loop of empty string matches.");
        }
      }
  }

  static void PrintArrayInitializer(int noElems)
  {
    ostr.print("{");
    for (int i = 0; i < noElems; i++)
    {
      if (i % 25 == 0)
        ostr.print("\n   ");
      ostr.print("0, ");
    }
    ostr.println("\n};");
  }

  static void DumpStaticVarDeclarations()
  {
    int i;
    String charStreamName;

    ostr.println("");
    ostr.println("/** Lexer state names. */");
    ostr.println("public static final String[] lexStateNames = {");
    for (i = 0; i < maxLexStates; i++)
      ostr.println("   \"" + lexStateName[i] + "\", ");
    ostr.println("};");

    if (maxLexStates > 1)
    {
      ostr.println("");
      ostr.println("/** Lex State array. */");
      ostr.print("public static final int[] jjnewLexState = {");

      for (i = 0; i < maxOrdinal; i++)
      {
        if (i % 25 == 0)
          ostr.print("\n   ");

        if (newLexState[i] == null)
          ostr.print("-1, ");
        else
          ostr.print(GetIndex(newLexState[i]) + ", ");
      }
      ostr.println("\n};");
    }

    if (hasSkip || hasMore || hasSpecial)
    {
      // Bit vector for TOKEN
      ostr.print("static final long[] jjtoToken = {");
      for (i = 0; i < maxOrdinal / 64 + 1; i++)
      {
        if (i % 4 == 0)
          ostr.print("\n   ");
        ostr.print("0x" + Long.toHexString(toToken[i]) + "L, ");
      }
      ostr.println("\n};");
    }

    if (hasSkip || hasSpecial)
    {
      // Bit vector for SKIP
      ostr.print("static final long[] jjtoSkip = {");
      for (i = 0; i < maxOrdinal / 64 + 1; i++)
      {
        if (i % 4 == 0)
          ostr.print("\n   ");
        ostr.print("0x" + Long.toHexString(toSkip[i]) + "L, ");
      }
      ostr.println("\n};");
    }

    if (hasSpecial)
    {
      // Bit vector for SPECIAL
      ostr.print("static final long[] jjtoSpecial = {");
      for (i = 0; i < maxOrdinal / 64 + 1; i++)
      {
        if (i % 4 == 0)
          ostr.print("\n   ");
        ostr.print("0x" + Long.toHexString(toSpecial[i]) + "L, ");
      }
      ostr.println("\n};");
    }

    if (hasMore)
    {
      // Bit vector for MORE
      ostr.print("static final long[] jjtoMore = {");
      for (i = 0; i < maxOrdinal / 64 + 1; i++)
      {
        if (i % 4 == 0)
          ostr.print("\n   ");
        ostr.print("0x" + Long.toHexString(toMore[i]) + "L, ");
      }
      ostr.println("\n};");
    }

    if (Options.getUserCharStream())
      charStreamName = "CharStream";
    else
    {
      if (Options.getJavaUnicodeEscape())
        charStreamName = "JavaCharStream";
      else
        charStreamName = "SimpleCharStream";
    }

    ostr.println(staticString + "protected " + charStreamName + " input_stream;");

    ostr.println(staticString + "private final int[] jjrounds = " +
        "new int[" + stateSetSize + "];");
    ostr.println(staticString + "private final int[] jjstateSet = " +
        "new int[" + (2 * stateSetSize) + "];");

    if (hasMoreActions || hasSkipActions || hasTokenActions)
    {
      ostr.println("private " + staticString + "final " + Options.stringBufOrBuild() + " jjimage = new " + Options.stringBufOrBuild() + "();");
      ostr.println("private " + staticString + Options.stringBufOrBuild() + " image = jjimage;");
      ostr.println("private " + staticString + "int jjimageLen;");
      ostr.println("private " + staticString + "int lengthOfMatch;");
    }

    ostr.println(staticString + "protected char curChar;");

    if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
      ostr.println("");
      ostr.println("/** Constructor with parser. */");
      ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream){");
      ostr.println("   parser = parserArg;");
    } else {
      ostr.println("/** Constructor. */");
      ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream){");
    }

    if (Options.getStatic() && !Options.getUserCharStream())
    {
      ostr.println("   if (input_stream != null)");
      ostr.println("      throw new TokenMgrError(\"ERROR: Second call to constructor of static lexer. " +
      "You must use ReInit() to initialize the static variables.\", TokenMgrError.STATIC_LEXER_ERROR);");
    }
    else if (!Options.getUserCharStream())
    {
      if (Options.getJavaUnicodeEscape())
        ostr.println("   if (JavaCharStream.staticFlag)");
      else
        ostr.println("   if (SimpleCharStream.staticFlag)");

      ostr.println("      throw new Error(\"ERROR: Cannot use a static CharStream class with a " +
      "non-static lexical analyzer.\");");
    }

    ostr.println("   input_stream = stream;");

    ostr.println("}");

    if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
      ostr.println("");
      ostr.println("/** Constructor with parser. */");
      ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " +
          charStreamName + " stream, int lexState){");
      ostr.println("   this(parserArg, stream);");
    } else {
      ostr.println("");
      ostr.println("/** Constructor. */");
      ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream, int lexState){");
      ostr.println("   this(stream);");
    }
    ostr.println("   SwitchTo(lexState);");
    ostr.println("}");

    // Reinit method for reinitializing the parser (for static parsers).
    ostr.println("");
    ostr.println("/** Reinitialise parser. */");
    ostr.println(staticString + "public void ReInit(" + charStreamName + " stream)");
    ostr.println("{");
    ostr.println("   jjmatchedPos = jjnewStateCnt = 0;");
    ostr.println("   curLexState = defaultLexState;");
    ostr.println("   input_stream = stream;");
    ostr.println("   ReInitRounds();");
    ostr.println("}");

    // Method to reinitialize the jjrounds array.
    ostr.println(staticString + "private void ReInitRounds()");
    ostr.println("{");
    ostr.println("   int i;");
    ostr.println("   jjround = 0x" + Integer.toHexString(Integer.MIN_VALUE + 1)+ ";");
    ostr.println("   for (i = " + stateSetSize + "; i-- > 0;)");
    ostr.println("      jjrounds[i] = 0x" + Integer.toHexString(Integer.MIN_VALUE) + ";");
    ostr.println("}");

    // Reinit method for reinitializing the parser (for static parsers).
    ostr.println("");
    ostr.println("/** Reinitialise parser. */");
    ostr.println(staticString + "public void ReInit(" + charStreamName + " stream, int lexState)");
    ostr.println("{");
    ostr.println("   ReInit(stream);");
    ostr.println("   SwitchTo(lexState);");
    ostr.println("}");

    ostr.println("");
    ostr.println("/** Switch to specified lex state. */");
    ostr.println(staticString + "public void SwitchTo(int lexState)");
    ostr.println("{");
    ostr.println("   if (lexState >= " + lexStateName.length + " || lexState < 0)");
    ostr.println("      throw new TokenMgrError(\"Error: Ignoring invalid lexical state : \"" +
    " + lexState + \". State unchanged.\", TokenMgrError.INVALID_LEXICAL_STATE);");
    ostr.println("   else");
    ostr.println("      curLexState = lexState;");
    ostr.println("}");

    ostr.println("");
  }

  // Assumes l != 0L
  static char MaxChar(long l)
  {
    for (int i = 64; i-- > 0; )
      if ((l & (1L << i)) != 0L)
        return (char)i;

    return 0xffff;
  }

  static void DumpFillToken()
  {
    final double tokenVersion = JavaFiles.getVersion("Token.java");
    final boolean hasBinaryNewToken = tokenVersion > 4.09;

    ostr.println(staticString + "protected Token jjFillToken()");
    ostr.println("{");
    ostr.println("   final Token t;");
    ostr.println("   final String curTokenImage;");
    if (keepLineCol)
    {
      ostr.println("   final int beginLine;");
      ostr.println("   final int endLine;");
      ostr.println("   final int beginColumn;");
      ostr.println("   final int endColumn;");
    }

    if (hasEmptyMatch)
    {
      ostr.println("   if (jjmatchedPos < 0)");
      ostr.println("   {");
      ostr.println("      if (image == null)");
      ostr.println("         curTokenImage = \"\";");
      ostr.println("      else");
      ostr.println("         curTokenImage = image.toString();");

      if (keepLineCol)
      {
        ostr.println("      beginLine = endLine = input_stream.getBeginLine();");
        ostr.println("      beginColumn = endColumn = input_stream.getBeginColumn();");
      }

      ostr.println("   }");
      ostr.println("   else");
      ostr.println("   {");
      ostr.println("      String im = jjstrLiteralImages[jjmatchedKind];");
      ostr.println("      curTokenImage = (im == null) ? input_stream.GetImage() : im;");

      if (keepLineCol)
      {
        ostr.println("      beginLine = input_stream.getBeginLine();");
        ostr.println("      beginColumn = input_stream.getBeginColumn();");
        ostr.println("      endLine = input_stream.getEndLine();");
        ostr.println("      endColumn = input_stream.getEndColumn();");
      }

      ostr.println("   }");
    }
    else
    {
      ostr.println("   String im = jjstrLiteralImages[jjmatchedKind];");
      ostr.println("   curTokenImage = (im == null) ? input_stream.GetImage() : im;");
      if (keepLineCol)
      {
        ostr.println("   beginLine = input_stream.getBeginLine();");
        ostr.println("   beginColumn = input_stream.getBeginColumn();");
        ostr.println("   endLine = input_stream.getEndLine();");
        ostr.println("   endColumn = input_stream.getEndColumn();");
      }
    }

    if (Options.getTokenFactory().length() > 0) {
      ostr.println("   t = " + Options.getTokenFactory() + ".newToken(jjmatchedKind, curTokenImage);");
    } else if (hasBinaryNewToken)
    {
      ostr.println("   t = Token.newToken(jjmatchedKind, curTokenImage);");
    }
    else
    {
      ostr.println("   t = Token.newToken(jjmatchedKind);");
      ostr.println("   t.kind = jjmatchedKind;");
      ostr.println("   t.image = curTokenImage;");
    }

    if (keepLineCol) {
      ostr.println("");
      ostr.println("   t.beginLine = beginLine;");
      ostr.println("   t.endLine = endLine;");
      ostr.println("   t.beginColumn = beginColumn;");
      ostr.println("   t.endColumn = endColumn;");
    }

    ostr.println("");
    ostr.println("   return t;");
    ostr.println("}");
  }

  static void DumpGetNextToken()
  {
    int i;

    ostr.println("");
    ostr.println(staticString + "int curLexState = " + defaultLexState + ";");
    ostr.println(staticString + "int defaultLexState = " + defaultLexState + ";");
    ostr.println(staticString + "int jjnewStateCnt;");
    ostr.println(staticString + "int jjround;");
    ostr.println(staticString + "int jjmatchedPos;");
    ostr.println(staticString + "int jjmatchedKind;");
    ostr.println("");
    ostr.println("/** Get the next Token. */");
    ostr.println("public " + staticString + "Token getNextToken()" +
    " ");
    ostr.println("{");
    if (hasSpecial) {
      ostr.println("  Token specialToken = null;");
    }
    ostr.println("  Token matchedToken;");
    ostr.println("  int curPos = 0;");
    ostr.println("");
    ostr.println("  EOFLoop :\n  for (;;)");
    ostr.println("  {   ");
    ostr.println("   try   ");
    ostr.println("   {     ");
    ostr.println("      curChar = input_stream.BeginToken();");
    ostr.println("   }     ");
    ostr.println("   catch(java.io.IOException e)");
    ostr.println("   {        ");

    if (Options.getDebugTokenManager())
      ostr.println("      debugStream.println(\"Returning the <EOF> token.\");");

    ostr.println("      jjmatchedKind = 0;");
    ostr.println("      matchedToken = jjFillToken();");

    if (hasSpecial)
      ostr.println("      matchedToken.specialToken = specialToken;");

    if (nextStateForEof != null || actForEof != null)
      ostr.println("      TokenLexicalActions(matchedToken);");

    if (Options.getCommonTokenAction())
      ostr.println("      CommonTokenAction(matchedToken);");

    ostr.println("      return matchedToken;");
    ostr.println("   }");

    if (hasMoreActions || hasSkipActions || hasTokenActions)
    {
      ostr.println("   image = jjimage;");
      ostr.println("   image.setLength(0);");
      ostr.println("   jjimageLen = 0;");
    }

    ostr.println("");

    String prefix = "";
    if (hasMore)
    {
      ostr.println("   for (;;)");
      ostr.println("   {");
      prefix = "  ";
    }

    String endSwitch = "";
    String caseStr = "";
    // this also sets up the start state of the nfa
    if (maxLexStates > 1)
    {
      ostr.println(prefix + "   switch(curLexState)");
      ostr.println(prefix + "   {");
      endSwitch = prefix + "   }";
      caseStr = prefix + "     case ";
      prefix += "    ";
    }

    prefix += "   ";
    for(i = 0; i < maxLexStates; i++)
    {
      if (maxLexStates > 1)
        ostr.println(caseStr + i + ":");

      if (singlesToSkip[i].HasTransitions())
      {
        // added the backup(0) to make JIT happy
        ostr.println(prefix + "try { input_stream.backup(0);");
        if (singlesToSkip[i].asciiMoves[0] != 0L &&
            singlesToSkip[i].asciiMoves[1] != 0L)
        {
          ostr.println(prefix + "   while ((curChar < 64" + " && (0x" +
              Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
              "L & (1L << curChar)) != 0L) || \n" +
              prefix + "          (curChar >> 6) == 1" +
              " && (0x" +
              Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
          "L & (1L << (curChar & 077))) != 0L)");
        }
        else if (singlesToSkip[i].asciiMoves[1] == 0L)
        {
          ostr.println(prefix + "   while (curChar <= " +
              (int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +
              Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
          "L & (1L << curChar)) != 0L)");
        }
        else if (singlesToSkip[i].asciiMoves[0] == 0L)
        {
          ostr.println(prefix + "   while (curChar > 63 && curChar <= " +
              ((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +
              " && (0x" +
              Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
          "L & (1L << (curChar & 077))) != 0L)");
        }

        if (Options.getDebugTokenManager())
        {
          ostr.println(prefix + "{");
          ostr.println("      debugStream.println(" +
              (maxLexStates > 1 ?
                  "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
                  "\"Skipping character : \" + " +
          "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \")\");");
        }
        ostr.println(prefix + "      curChar = input_stream.BeginToken();");

        if (Options.getDebugTokenManager())
          ostr.println(prefix + "}");

        ostr.println(prefix + "}");
        ostr.println(prefix + "catch (java.io.IOException e1) { continue EOFLoop; }");
      }

      if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
      {
        if (Options.getDebugTokenManager())
          ostr.println("      debugStream.println(\"   Matched the empty string as \" + tokenImage[" +
              initMatch[i] + "] + \" token.\");");

        ostr.println(prefix + "jjmatchedKind = " + initMatch[i] + ";");
        ostr.println(prefix + "jjmatchedPos = -1;");
        ostr.println(prefix + "curPos = 0;");
      }
      else
      {
        ostr.println(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
        ostr.println(prefix + "jjmatchedPos = 0;");
      }

      if (Options.getDebugTokenManager())
        ostr.println("      debugStream.println(" +
            (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
            "\"Current character : \" + " +
            "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") " +
        "at line \" + input_stream.getEndLine() + \" column \" + input_stream.getEndColumn());");

      ostr.println(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");

      if (canMatchAnyChar[i] != -1)
      {
        if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
          ostr.println(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +
              canMatchAnyChar[i] + "))");
        else
          ostr.println(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +
              canMatchAnyChar[i] + ")");
        ostr.println(prefix + "{");

        if (Options.getDebugTokenManager())
          ostr.println("           debugStream.println(\"   Current character matched as a \" + tokenImage[" +
              canMatchAnyChar[i] + "] + \" token.\");");
        ostr.println(prefix + "   jjmatchedKind = " + canMatchAnyChar[i] + ";");

        if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
          ostr.println(prefix + "   jjmatchedPos = 0;");

        ostr.println(prefix + "}");
      }

      if (maxLexStates > 1)
        ostr.println(prefix + "break;");
    }

    if (maxLexStates > 1)
      ostr.println(endSwitch);
    else if (maxLexStates == 0)
      ostr.println("       jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");

    if (maxLexStates > 1)
      prefix = "  ";
    else
      prefix = "";

    if (maxLexStates > 0)
    {
      ostr.println(prefix + "   if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
      ostr.println(prefix + "   {");
      ostr.println(prefix + "      if (jjmatchedPos + 1 < curPos)");

      if (Options.getDebugTokenManager())
      {
        ostr.println(prefix + "      {");
        ostr.println(prefix + "         debugStream.println(" +
        "\"   Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");
      }

      ostr.println(prefix + "         input_stream.backup(curPos - jjmatchedPos - 1);");

      if (Options.getDebugTokenManager())
        ostr.println(prefix + "      }");

      if (Options.getDebugTokenManager())
      {
        if (Options.getJavaUnicodeEscape() ||
            Options.getUserCharStream())
          ostr.println("    debugStream.println(" +
              "\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +
              "(\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +
          "\") ******\\n\");");
        else
          ostr.println("    debugStream.println(" +
              "\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH " +
              "(\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + " +
          "\") ******\\n\");");
      }

      if (hasSkip || hasMore || hasSpecial)
      {
        ostr.println(prefix + "      if ((jjtoToken[jjmatchedKind >> 6] & " +
        "(1L << (jjmatchedKind & 077))) != 0L)");
        ostr.println(prefix + "      {");
      }

      ostr.println(prefix + "         matchedToken = jjFillToken();");

      if (hasSpecial)
        ostr.println(prefix + "         matchedToken.specialToken = specialToken;");

      if (hasTokenActions)
        ostr.println(prefix + "         TokenLexicalActions(matchedToken);");

      if (maxLexStates > 1)
      {
        ostr.println("       if (jjnewLexState[jjmatchedKind] != -1)");
        ostr.println(prefix + "       curLexState = jjnewLexState[jjmatchedKind];");
      }

      if (Options.getCommonTokenAction())
        ostr.println(prefix + "         CommonTokenAction(matchedToken);");

      ostr.println(prefix + "         return matchedToken;");

      if (hasSkip || hasMore || hasSpecial)
      {
        ostr.println(prefix + "      }");

        if (hasSkip || hasSpecial)
        {
          if (hasMore)
          {
            ostr.println(prefix + "      else if ((jjtoSkip[jjmatchedKind >> 6] & " +
            "(1L << (jjmatchedKind & 077))) != 0L)");
          }
          else
            ostr.println(prefix + "      else");

          ostr.println(prefix + "      {");

          if (hasSpecial)
          {
            ostr.println(prefix + "         if ((jjtoSpecial[jjmatchedKind >> 6] & " +
            "(1L << (jjmatchedKind & 077))) != 0L)");
            ostr.println(prefix + "         {");

            ostr.println(prefix + "            matchedToken = jjFillToken();");

            ostr.println(prefix + "            if (specialToken == null)");
            ostr.println(prefix + "               specialToken = matchedToken;");
            ostr.println(prefix + "            else");
            ostr.println(prefix + "            {");
            ostr.println(prefix + "               matchedToken.specialToken = specialToken;");
            ostr.println(prefix + "               specialToken = (specialToken.next = matchedToken);");
            ostr.println(prefix + "            }");

            if (hasSkipActions)
              ostr.println(prefix + "            SkipLexicalActions(matchedToken);");

            ostr.println(prefix + "         }");

            if (hasSkipActions)
            {
              ostr.println(prefix + "         else ");
              ostr.println(prefix + "            SkipLexicalActions(null);");
            }
          }
          else if (hasSkipActions)
            ostr.println(prefix + "         SkipLexicalActions(null);");

          if (maxLexStates > 1)
          {
            ostr.println("         if (jjnewLexState[jjmatchedKind] != -1)");
            ostr.println(prefix + "         curLexState = jjnewLexState[jjmatchedKind];");
          }

          ostr.println(prefix + "         continue EOFLoop;");
          ostr.println(prefix + "      }");
        }

        if (hasMore)
        {
          if (hasMoreActions)
            ostr.println(prefix + "      MoreLexicalActions();");
          else if (hasSkipActions || hasTokenActions)
            ostr.println(prefix + "      jjimageLen += jjmatchedPos + 1;");

          if (maxLexStates > 1)
          {
            ostr.println("      if (jjnewLexState[jjmatchedKind] != -1)");
            ostr.println(prefix + "      curLexState = jjnewLexState[jjmatchedKind];");
          }
          ostr.println(prefix + "      curPos = 0;");
          ostr.println(prefix + "      jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");

          ostr.println(prefix + "      try {");
          ostr.println(prefix + "         curChar = input_stream.readChar();");

          if (Options.getDebugTokenManager())
            ostr.println("   debugStream.println(" +
                (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") +
                "\"Current character : \" + " +
                "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") " +
            "at line \" + input_stream.getEndLine() + \" column \" + input_stream.getEndColumn());");
          ostr.println(prefix + "         continue;");
          ostr.println(prefix + "      }");
          ostr.println(prefix + "      catch (java.io.IOException e1) { }");
        }
      }

      ostr.println(prefix + "   }");
      ostr.println(prefix + "   int error_line = input_stream.getEndLine();");
      ostr.println(prefix + "   int error_column = input_stream.getEndColumn();");
      ostr.println(prefix + "   String error_after = null;");
      ostr.println(prefix + "   boolean EOFSeen = false;");
      ostr.println(prefix + "   try { input_stream.readChar(); input_stream.backup(1); }");
      ostr.println(prefix + "   catch (java.io.IOException e1) {");
      ostr.println(prefix + "      EOFSeen = true;");
      ostr.println(prefix + "      error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
      ostr.println(prefix + "      if (curChar == '\\n' || curChar == '\\r') {");
      ostr.println(prefix + "         error_line++;");
      ostr.println(prefix + "         error_column = 0;");
      ostr.println(prefix + "      }");
      ostr.println(prefix + "      else");
      ostr.println(prefix + "         error_column++;");
      ostr.println(prefix + "   }");
      ostr.println(prefix + "   if (!EOFSeen) {");
      ostr.println(prefix + "      input_stream.backup(1);");
      ostr.println(prefix + "      error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
      ostr.println(prefix + "   }");
      ostr.println(prefix + "   throw new TokenMgrError(" +
      "EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);");
    }

    if (hasMore)
      ostr.println(prefix + " }");

    ostr.println("  }");
    ostr.println("}");
    ostr.println("");
  }

  public static void DumpSkipActions()
  {
    Action act;

    ostr.println(staticString + "void SkipLexicalActions(Token matchedToken)");
    ostr.println("{");
    ostr.println("   switch(jjmatchedKind)");
    ostr.println("   {");

    Outer:
      for (int i = 0; i < maxOrdinal; i++)
      {
        if ((toSkip[i / 64] & (1L << (i % 64))) == 0L)
          continue;

        for (;;)
        {
          if (((act = (Action)actions[i]) == null ||
              act.action_tokens == null ||
              act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
            continue Outer;

          ostr.println("      case " + i + " :");

          if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
          {
            ostr.println("         if (jjmatchedPos == -1)");
            ostr.println("         {");
            ostr.println("            if (jjbeenHere[" + lexStates[i] + "] &&");
            ostr.println("                jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
            ostr.println("                jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
            ostr.println("               throw new TokenMgrError(" +
                "(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
                "at line \" + input_stream.getBeginLine() + \", " +
            "column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
            ostr.println("            jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
            ostr.println("            jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
            ostr.println("            jjbeenHere[" + lexStates[i] + "] = true;");
            ostr.println("         }");
          }

          if ((act = (Action)actions[i]) == null ||
              act.action_tokens.size() == 0)
            break;

          ostr.print(  "         image.append");
          if (RStringLiteral.allImages[i] != null) {
            ostr.println("(jjstrLiteralImages[" + i + "]);");
            ostr.println("        lengthOfMatch = jjstrLiteralImages[" + i + "].length();");
          } else {
            ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
          }

          printTokenSetup((Token)act.action_tokens.elementAt(0));
          ccol = 1;

          for (int j = 0; j < act.action_tokens.size(); j++)
            printToken((Token)act.action_tokens.elementAt(j), ostr);
          ostr.println("");

          break;
        }

        ostr.println("         break;");
      }

    ostr.println("      default :");
    ostr.println("         break;");
    ostr.println("   }");
    ostr.println("}");
  }

  public static void DumpMoreActions()
  {
    Action act;

    ostr.println(staticString + "void MoreLexicalActions()");
    ostr.println("{");
    ostr.println("   jjimageLen += (lengthOfMatch = jjmatchedPos + 1);");
    ostr.println("   switch(jjmatchedKind)");
    ostr.println("   {");

    Outer:
      for (int i = 0; i < maxOrdinal; i++)
      {
        if ((toMore[i / 64] & (1L << (i % 64))) == 0L)
          continue;

        for (;;)
        {
          if (((act = (Action)actions[i]) == null ||
              act.action_tokens == null ||
              act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
            continue Outer;

          ostr.println("      case " + i + " :");

          if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
          {
            ostr.println("         if (jjmatchedPos == -1)");
            ostr.println("         {");
            ostr.println("            if (jjbeenHere[" + lexStates[i] + "] &&");
            ostr.println("                jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
            ostr.println("                jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
            ostr.println("               throw new TokenMgrError(" +
                "(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
                "at line \" + input_stream.getBeginLine() + \", " +
            "column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
            ostr.println("            jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
            ostr.println("            jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
            ostr.println("            jjbeenHere[" + lexStates[i] + "] = true;");
            ostr.println("         }");
          }

          if ((act = (Action)actions[i]) == null ||
              act.action_tokens.size() == 0)
          {
            break;
          }

          ostr.print(  "         image.append");

          if (RStringLiteral.allImages[i] != null)
            ostr.println("(jjstrLiteralImages[" + i + "]);");
          else
            ostr.println("(input_stream.GetSuffix(jjimageLen));");

          ostr.println("         jjimageLen = 0;");
          printTokenSetup((Token)act.action_tokens.elementAt(0));
          ccol = 1;

          for (int j = 0; j < act.action_tokens.size(); j++)
            printToken((Token)act.action_tokens.elementAt(j), ostr);
          ostr.println("");

          break;
        }

        ostr.println("         break;");
      }

    ostr.println("      default : ");
    ostr.println("         break;");

    ostr.println("   }");
    ostr.println("}");
  }

  public static void DumpTokenActions()
  {
    Action act;
    int i;

    ostr.println(staticString + "void TokenLexicalActions(Token matchedToken)");
    ostr.println("{");
    ostr.println("   switch(jjmatchedKind)");
    ostr.println("   {");

    Outer:
      for (i = 0; i < maxOrdinal; i++)
      {
        if ((toToken[i / 64] & (1L << (i % 64))) == 0L)
          continue;

        for (;;)
        {
          if (((act = (Action)actions[i]) == null ||
              act.action_tokens == null ||
              act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
            continue Outer;

          ostr.println("      case " + i + " :");

          if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
          {
            ostr.println("         if (jjmatchedPos == -1)");
            ostr.println("         {");
            ostr.println("            if (jjbeenHere[" + lexStates[i] + "] &&");
            ostr.println("                jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
            ostr.println("                jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
            ostr.println("               throw new TokenMgrError(" +
                "(\"Error: Bailing out of infinite loop caused by repeated empty string matches " +
                "at line \" + input_stream.getBeginLine() + \", " +
            "column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
            ostr.println("            jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
            ostr.println("            jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
            ostr.println("            jjbeenHere[" + lexStates[i] + "] = true;");
            ostr.println("         }");
          }

          if ((act = (Action)actions[i]) == null ||
              act.action_tokens.size() == 0)
            break;

          if (i == 0)
          {
            ostr.println("      image.setLength(0);"); // For EOF no image is there
          }
          else
          {
            ostr.print(  "        image.append");

            if (RStringLiteral.allImages[i] != null) {
              ostr.println("(jjstrLiteralImages[" + i + "]);");
              ostr.println("        lengthOfMatch = jjstrLiteralImages[" + i + "].length();");
            } else {
              ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
            }
          }

          printTokenSetup((Token)act.action_tokens.elementAt(0));
          ccol = 1;

          for (int j = 0; j < act.action_tokens.size(); j++)
            printToken((Token)act.action_tokens.elementAt(j), ostr);
          ostr.println("");

          break;
        }

        ostr.println("         break;");
      }

    ostr.println("      default : ");
    ostr.println("         break;");
    ostr.println("   }");
    ostr.println("}");
  }

  public static void reInit()
  {
    ostr = null;
    staticString = null;
    tokMgrClassName = null;
    allTpsForState = new Hashtable();
    lexStateIndex = 0;
    kinds = null;
    maxOrdinal = 1;
    lexStateSuffix = null;
    newLexState = null;
    lexStates = null;
    ignoreCase = null;
    actions = null;
    initStates = new Hashtable();
    stateSetSize = 0;
    maxLexStates = 0;
    lexStateName = null;
    singlesToSkip = null;
    toSkip = null;
    toSpecial = null;
    toMore = null;
    toToken = null;
    defaultLexState = 0;
    rexprs = null;
    maxLongsReqd = null;
    initMatch = null;
    canMatchAnyChar = null;
    hasEmptyMatch = false;
    canLoop = null;
    stateHasActions = null;
    hasLoop = false;
    canReachOnMore = null;
    hasNfa = null;
    mixed = null;
    initialState = null;
    curKind = 0;
    hasSkipActions = false;
    hasMoreActions = false;
    hasTokenActions = false;
    hasSpecial = false;
    hasSkip = false;
    hasMore = false;
    curRE = null;
  }

}
