------------------------------------------------------------------------------
--                                                                          --
--                             Libadalang Tools                             --
--                                                                          --
--                      Copyright (C) 2012-2024, AdaCore                    --
--                                                                          --
-- Libadalang Tools  is free software; you can redistribute it and/or modi- --
-- fy  it  under  terms of the  GNU General Public License  as published by --
-- the Free Software Foundation;  either version 3, or (at your option) any --
-- later version. This software  is distributed in the hope that it will be --
-- useful but  WITHOUT  ANY  WARRANTY; without even the implied warranty of --
-- MERCHANTABILITY  or  FITNESS  FOR A PARTICULAR PURPOSE.                  --
--                                                                          --
-- As a special  exception  under  Section 7  of  GPL  version 3,  you are  --
-- granted additional  permissions described in the  GCC  Runtime  Library  --
-- Exception, version 3.1, as published by the Free Software Foundation.    --
--                                                                          --
-- You should have received a copy of the GNU General Public License and a  --
-- copy of the GCC Runtime Library Exception along with this program;  see  --
-- the files COPYING3 and COPYING.RUNTIME respectively.  If not, see        --
-- <http://www.gnu.org/licenses/>.                                          --
------------------------------------------------------------------------------

with Ada.Text_IO;

with Pp.Buffers;

with Utils.Char_Vectors;
with Utils.Symbols;
with Utils.Vectors;

private with Utils.Var_Length_Ints;

package Pp.Scanner is

   --  This package provides a simple lexical scanner for Ada tokens. There are
   --  some unusual things about this scanner:
   --
   --     We do not ignore comments; a comment is considered to be a token.
   --
   --     We do not ignore line breaks. Blanks between tokens are always
   --     ignored. Other space characters (tabs, NO-BREAK SPACE, etc.) are
   --     removed in Buffers.

   package Syms renames Utils.Symbols;

   type Opt_Token_Kind is
     (Nil,
      Illegal_Character,
      Ident,
      Character_Literal,
      String_Lit,
      Numeric_Literal,
      Preprocessor_Directive, -- line starting with "#"
      Spaces, -- A sequence of one or more space characters.
      Pp_Off_Comment,
      --  A whole-line comment that matches the --pp-off string
      Pp_On_Comment,
      --  A whole-line comment that matches the --pp-on string
      Special_Comment,
      --  A "special" comment; that is, one that should not be formatted in any
      --  way. Special comments are not fillable.
      Fillable_Comment,
      --  A fillable whole-line comment; that is, one that should be
      --  filled if filling is turned on.
      Other_Whole_Line_Comment,
   --  A comment that appears by itself on a line. Multiple comments that may
   --  be filled as a "paragraph" are combined into a single Whole_Line_Comment
   --  token.
      End_Of_Line_Comment,
   --  A comment that appears at the end of a line, after some other
   --  program text.

      --  All the rest are in Same_Text_Kind (see body):

      Start_Of_Input,
      End_Of_Input,
      Enabled_LB_Token, Disabled_LB_Token,
      Tab_Token,
      False_End_Of_Line, True_End_Of_Line_LF, True_End_Of_Line_CR,
      True_End_Of_Line_CRLF,
      --  False_End_Of_Line and True_End_Of_Line are used only in Src_Tokns;
      --  the scanner produces these when scanning the source.
      --  A False_End_Of_Line follows a comment, and is merely a placeholder
      --  for a line break that will be processed later. All others are
      --  True_End_Of_Line.
      --  Enabled_LB_Token and Disabled_LB_Token are used in New_Tokns,
      --  representing line breaks in the generated output; these contain an
      --  index into the lines break table. When a disabled line break is
      --  enabled, the Enabled flag in the lines break table is set, but the
      --  token kind is still Disabled_LB_Token. A later pass will replace it
      --  with an Enabled_LB_Token. Thus, a Disabled_LB_Token can temporarily
      --  correspond to an enabled line break.

      '!', '#', '$', '?', '[', '\', ']', '^', '`', '{', '}', '~', '_',
      --  These are not in Ada

      '-', ''', '&', '(', ')', '+', ',', ':', ';', '|', '@',
      '<', '>', '.', '*', '=', '/',
      Arrow, --  =>
      Dot_Dot, --  ..
      Exp_Op, --  **
      Not_Equal, --  /=
      Greater_Or_Equal, --  >=
      Right_Label_Bracket, --  >>
      Less_Or_Equal, --  <=
      Left_Label_Bracket, --  <<
      Box, --  <>
      Colon_Equal, --  :=

      --  Ada 83 reserved words

      Res_Abort,
      Res_Abs,
      Res_Accept,
      Res_Access,
      Res_And,
      Res_All,
      Res_Array,
      Res_At,
      Res_Begin,
      Res_Body,
      Res_Case,
      Res_Constant,
      Res_Declare,
      Res_Delay,
      Res_Delta,
      Res_Digits,
      Res_Do,
      Res_Else,
      Res_Elsif,
      Res_End,
      Res_Entry,
      Res_Exception,
      Res_Exit,
      Res_For,
      Res_Function,
      Res_Generic,
      Res_Goto,
      Res_If,
      Res_In,
      Res_Is,
      Res_Limited,
      Res_Loop,
      Res_Mod,
      Res_New,
      Res_Not,
      Res_Null,
      Res_Of,
      Res_Or,
      Res_Others,
      Res_Out,
      Res_Package,
      Res_Pragma,
      Res_Private,
      Res_Procedure,
      Res_Raise,
      Res_Range,
      Res_Record,
      Res_Rem,
      Res_Renames,
      Res_Return,
      Res_Reverse,
      Res_Select,
      Res_Separate,
      Res_Subtype,
      Res_Task,
      Res_Terminate,
      Res_Then,
      Res_Type,
      Res_Use,
      Res_When,
      Res_While,
      Res_With,
      Res_Xor,

      --  Ada 95 reserved words

      Res_Abstract,
      Res_Aliased,
      Res_Protected,
      Res_Until,
      Res_Requeue,
      Res_Tagged,

      --  Ada 2005 reserved words

      Res_Interface,
      Res_Overriding,
      Res_Synchronized,

      --  Ada 2012 reserved words

      Res_Some
     );

   subtype Token_Kind is Opt_Token_Kind with Predicate => Token_Kind /= Nil;

   subtype Other_Lexeme is Token_Kind range '!' .. Colon_Equal;

   subtype Reserved_Word is Token_Kind range Res_Abort .. Res_Some;
   subtype Reserved_Word_Or_Id is Token_Kind with
     Predicate => Reserved_Word_Or_Id in Ident | Reserved_Word;
   subtype Reserved_Word_83 is Token_Kind range Res_Abort .. Res_Xor;
   subtype Reserved_Word_95 is Token_Kind range Res_Abort .. Res_Tagged;
   subtype Reserved_Word_2005 is
     Token_Kind range Res_Abort .. Res_Synchronized;
   subtype Reserved_Word_2012 is Token_Kind range Res_Abort .. Res_Some;

   subtype Reserved_Word_New is Reserved_Word_2012 with
     Predicate => Reserved_Word_New not in Reserved_Word_83;
   --  These are the reserved words that were added to Ada after Ada 83. We
   --  don't know what language version we are processing, so we allow these
   --  as identifiers.

   subtype Whole_Line_Comment is Token_Kind with
     Predicate => Whole_Line_Comment in
       Pp_Off_Comment | Pp_On_Comment | Special_Comment |
       Fillable_Comment | Other_Whole_Line_Comment;

   subtype Comment_Kind is Token_Kind with
        Predicate => Comment_Kind in Whole_Line_Comment | End_Of_Line_Comment;

   subtype True_End_Of_Line is Token_Kind with
     Predicate => True_End_Of_Line in
                    True_End_Of_Line_LF
                      | True_End_Of_Line_CR
                      | True_End_Of_Line_CRLF;

   subtype EOL_Token is Token_Kind with
     Predicate => EOL_Token in False_End_Of_Line | True_End_Of_Line;

   subtype Line_Break_Token is Token_Kind with
     Predicate => Line_Break_Token in Enabled_LB_Token | Disabled_LB_Token;

   subtype Pp_Off_On_Comment is Token_Kind with
        Predicate => Pp_Off_On_Comment in Pp_Off_Comment | Pp_On_Comment;

   subtype Same_Text_Kind is Opt_Token_Kind range Start_Of_Input .. Res_Some;
   --  These are the tokens that always have the same text associated with
   --  them (case insensitively), so we don't need to store the text with
   --  each token. Instead, the token text is stored in Token_To_Symbol_Map
   --  below. Example: Every Less_Or_Equal token has the text "<=".

   subtype Stored_Text_Kind is Token_Kind with
     Predicate => Stored_Text_Kind not in Same_Text_Kind;
   --  These are the tokens that have different text.
   --  The token text is stored separately for each token.
   --  Example: Ident -- one might have text = "Foo",
   --  and another might have "Bar".

   type Source_Location is record
      Line, Col : Positive := 9999; -- 1-based line and column numbers
      First     : Positive := 9999;
      Last      : Natural := 9999;
   end record;

   --  Define type Token as a variant record. Mostly, we avoid using this type,
   --  and access tokens via type Tokn_Cursor and its accessor functions.
   --  See those accessor functions for documentation on the components
   --  of this record.

   type Token (Kind : Opt_Token_Kind := Nil) is private;

   function Text (X : Same_Text_Kind) return Syms.Symbol;
   function Text (X : Token) return Syms.Symbol;

   --  To add new token kinds:
   --
   --  Decide whether it belongs in Same_Text_Kind or Stored_Text_Kind,
   --  and add the enumeral to Opt_Token_Kind in an appropriate place.
   --
   --  Add code to Get_Tokns.Get_Tokn to parse out the token.
   --
   --  If the token needs additional information, add it to the variant part
   --  of type Token above. In addition, search the body of this package for
   --  "Octet" to find all places where the additional information needs to
   --  be encoded, decoded, and skipped over. Token_At_Cursor also needs to
   --  be modified. Currently, we only have support for integer types.

   function Image (Sloc : Source_Location) return String is
     (Image (Sloc.Line) & ":" &
      Image (Sloc.Col) & "(" &
      Image (Sloc.First) & ".." &
      Image (Sloc.Last) & ")");

   function Sloc_Image
     (Sloc : Source_Location) return String is
       (Image (Sloc.Line) & ":" & Image (Sloc.Col));

   type Source_Message is record
      --  Message attached to a particular source location.
      Sloc : Source_Location; -- the location
      Text : Char_Vectors.Char_Vector; -- the text of the message
   end record;
   type Source_Message_Array is array (Positive range <>) of Source_Message;

   package Source_Message_Vectors is new Utils.Vectors
     (Index_Type => Positive,
      Element_Type => Source_Message,
      Elements_Array => Source_Message_Array);
   subtype Source_Message_Vector is Source_Message_Vectors.Vector;

   function Message_Image
     (File_Name : String; Sloc : Source_Location) return String is
       (File_Name & ":" & Image (Sloc.Line) & ":" & Image (Sloc.Col));

   Default_Pp_Off_String : aliased constant W_Str := "--!pp off";
   Default_Pp_On_String : aliased constant W_Str := "--!pp on";

   type W_Str_Access_Constant is access constant W_Str;
   type Pp_Off_On_Delimiters_Rec is record
      Off : W_Str_Access_Constant := Default_Pp_Off_String'Access;
      On : W_Str_Access_Constant := Default_Pp_On_String'Access;
      --  Text of comments for turning pretting printing off and on, including
      --  the leading '--'. For example, if the user specified --pp-off='pp-',
      --  then Off will be "--pp-". A whole-line comment of the form "--pp-"
      --  will disable pretty printing.
      --  We do not want these comments to be fillable.
   end record;

   -------------------------------------
   -- Support for -pp-off and --pp-on --
   -------------------------------------

   Pp_Off_On_Delimiters : Scanner.Pp_Off_On_Delimiters_Rec;

   ----------------

   type Tokn_Vec is private;
   type Tokn_Vec_Ref is access constant Tokn_Vec;
   --  Growable sequence of tokens
   type Tokn_Cursor is private;
   --  Pointer into a Tokn_Vec

   function Is_Nil (X : Tokn_Cursor) return Boolean;
   function Nil_Tokn_Cursor return Tokn_Cursor;

   type Tokn_Index is new Positive;

   function Kind (X : Tokn_Cursor) return Token_Kind;
   --  Note that this cannot return Nil

   function Sloc (X : Tokn_Cursor) return Source_Location;
   function Sloc_Line (X : Tokn_Cursor) return Positive;
   function Sloc_Col (X : Tokn_Cursor) return Positive;
   function Sloc_First (X : Tokn_Cursor) return Positive;
   function Sloc_Last (X : Tokn_Cursor) return Natural;
   function New_Sloc_First (V : Tokn_Vec) return Positive;
   --  Sloc_First of the new token that will be added next, but does not yet
   --  exist in the Tokn_Vec.
   function Next_Sloc_First (X : Tokn_Cursor) return Positive;
   function Tokn_Length (X : Tokn_Cursor) return Natural is
     (Next_Sloc_First (X) - Sloc_First (X));
   --  The text of the token is equal to the slice
   --     Input(Sloc_First..Sloc_Last),
   --  where Input is the parameter of Get_Tokns.
   --  except for comments
   --  and eol/blank_lines that are CR/LF
   --  Beware that it's UtF-8.

   function Text (X : Tokn_Cursor) return Syms.Symbol;
   --  The text of the token as it appears in the source, with these
   --  exceptions and clarifications:
   --
   --  Start_Of_Input and End_Of_Input have Text = "".
   --
   --  EOL_Token have Text equal to a single LF character, even if it is
   --  CR,LF in the input.
   --
   --  For comments, the text of the comment excluding the initial "--"
   --  and leading and trailing blanks, and followed by an extra NL. For
   --  multi-line comment "paragraphs", used for filling, NL terminates each
   --  line. The NL at the end isn't really part of the comment; the next
   --  token in the stream will be EOL_Token. The reason for the extra NL
   --  is that GNATCOLL.Paragraph_Filling expects it, so it's simpler and
   --  more efficient this way.

   function Leading_Blanks (X : Tokn_Cursor) return Natural with
     Pre => Kind (X) in Comment_Kind;
   --  The number of leading blanks, which are blanks after the initial "--"
   --  and before any nonblank characters.

   function Width (X : Tokn_Cursor) return Positive with
     Pre => Kind (X) in Whole_Line_Comment;
   --  For single-line Whole_Line_Comments, this is the width of the token,
   --  i.e. the same as Sloc.Last-Sloc.First+1, and the same as the length of
   --  Text. For multi-line comments, this is the width of the widest line.
   --  The initial "--" and any leading blanks are included, but the NL's are
   --  not.

   function Token_Output_Len (X : Token) return Positive with
     Pre => X.Kind in Comment_Kind;
   function Token_Output_Len (X : Tokn_Cursor) return Positive with
     Pre => Kind (X) in Comment_Kind;
   --  Returns the length of the comment as it will appear in the output. For
   --  fillable comment paragraphs, this uses Sloc_Col as the indentation. The
   --  comment is assumed to start at the initial "--" and go to the end of the
   --  text, so the indentation is included in all but the first line.

   procedure Clear (V : in out Tokn_Vec);
   function First (V : Tokn_Vec_Ref) return Tokn_Cursor;
   function Last (V : Tokn_Vec_Ref) return Tokn_Cursor;
   function Is_Empty (V : Tokn_Vec) return Boolean;
   function At_Last (Cur : Tokn_Cursor) return Boolean;
   function After_Last (Cur : Tokn_Cursor) return Boolean;
   function Get_Tokn_Index (Cur : Tokn_Cursor) return Tokn_Index;
   function Next (Cur : Tokn_Cursor) return Tokn_Cursor;
   procedure Next (Cur : in out Tokn_Cursor);
   function Prev (Cur : Tokn_Cursor) return Tokn_Cursor;
   procedure Prev (Cur : in out Tokn_Cursor);

   function Succ (Cur : Tokn_Cursor; Count : Tokn_Index) return Tokn_Cursor;
   function Pred (Cur : Tokn_Cursor; Count : Tokn_Index) return Tokn_Cursor;
   --  Returns the cursor Count tokens after/before Cur, stopping if we
   --  reach the end/start.

   function Tokens_Require_Space (X, Y : Tokn_Cursor) return Boolean;
   --  True if a space is needed between X and Y to keep them from running
   --  together. For example, if X is the identifier "Mumble", and Y is the
   --  reserved word "is", then without a space between, it would look like a
   --  single identifier "Mumbleis". If Y cannot follow X according to the
   --  grammar, then it doesn't matter whether this returns True or False.

   function Get_Num_Tokens (V : Tokn_Vec) return Tokn_Index is
     (Get_Tokn_Index (Last (V'Unrestricted_Access)));

   function Is_Blank_Line (X : Tokn_Cursor) return Boolean is
     ((Kind (X) in EOL_Token and then Kind (Prev (X)) in EOL_Token)
      or else
     (Kind (X) in Enabled_LB_Token
        and then Kind (Prev (X)) in Enabled_LB_Token));

   subtype Nonlexeme_Kind is Opt_Token_Kind with Predicate =>
     Nonlexeme_Kind in EOL_Token | Spaces | Comment_Kind | Tab_Token;
   subtype Lexeme_Kind is Opt_Token_Kind with Predicate =>
     Lexeme_Kind not in Nonlexeme_Kind;
   --  Tokens classified (or not) as "lexemes"

   function Next_Lexeme (Cur : Tokn_Cursor) return Tokn_Cursor;
   --  Returns the next lexeme

   function Prev_Lexeme (Cur : Tokn_Cursor) return Tokn_Cursor;
   --  Returns the previous lexeme

   procedure Append_Tokn (V : in out Tokn_Vec; X : Tokn_Cursor;
                          Org : String := "Append Tokn_Cursor");
   procedure Append_Tokn (V : in out Tokn_Vec; X : Same_Text_Kind;
                          Org : String := "Append Kind");
   procedure Append_Tokn
     (V : in out Tokn_Vec; X : Stored_Text_Kind; Tx : Syms.Symbol;
      Org : String := "Append Kind&Text");
   --  Append X onto the end of V. If an identifier matches a reserved word in
   --  the latest language version, we append a reserved word token.

   --  The Org parameters are for debugging. They are a string that is recorded
   --  with the token, and indicates the "origin" of that token. The origins
   --  can be printed by Put_Token (see Show_Origin below). For debugging, it
   --  is sometimes useful to know where a certain token came from. Not every
   --  call specifies an Org; feel free to add more-specific origins when faced
   --  with a confusing bug.

   procedure Append_Spaces
     (V : in out Tokn_Vec; Count : Natural;
      Existing_OK : Boolean := False;
      Org : String := "Append_Spaces");
   --  Append a Spaces token; Count is the number of blanks.
   --  We don't want two Spaces in a row; by default we blow
   --  up if the previously appended token was a Spaces.
   --  If Existing_OK is True then we combine the two
   --  tokens.

   procedure Append_Comment
     (V : in out Tokn_Vec; X : Tokn_Cursor; Org : String) with
     Pre => Kind (X) in Comment_Kind;
   --  Append a comment token, adjusting the length for zero indentation

   function Is_Header_Comment
     (Tkn : Token)
      return Boolean with
     Pre => Tkn.Kind in Comment_Kind;
   --  Check if Tkn (which must be a comment) ends with -- or --\NL

   function Is_Empty_Comment
     (Token : Pp.Scanner.Token)
      return Boolean with
     Pre => Token.Kind in Comment_Kind;
   --  Check if Token (which must be a comment) is an empty comment (starts
   --  with --, optionally followed by spaces and a NL).

   procedure Append_Comment_Text
     (V                       : in out Tokn_Vec;
      X                       : Tokn_Cursor;
      Tx                      : W_Str;
      Recompute_Length        : Boolean;
      Comments_Only           : Boolean;
      Leading_Blanks          : Natural;
      Indent                  : Natural := 0;
      Org                     : String) with
     Pre => Kind (X) in Comment_Kind;
   --  Append a comment token, adjusting the length for zero indentation,
   --  and using the possibly-filled text Tx. I don't think it's
   --  zero indentation!

   procedure Delete_Last (V : in out Tokn_Vec);
   function Delete_Last (V : in out Tokn_Vec) return Token;
   --  Removes the last token (and the function returns it)

   function Token_At_Cursor (X : Tokn_Cursor) return Token;
   --  Return the token as a record

   type Language is (Ada_Lang, Template_Lang);
   --  Get_Tokns is used to parse the tokens of two different languages: Ada
   --  and the template language. They are mostly the treated the same, but
   --  this is used to indicate differences.

   type Optional_EOL_Formats is (Nil, CRLF, LF) with
     Default_Value => Nil;
   subtype EOL_Formats is Optional_EOL_Formats with
     Predicate => EOL_Formats in CRLF | LF;
   --  Conventions for ends of lines

   procedure Get_Tokns
     (Input               : in out Buffers.Buffer;
      Result              : out Tokn_Vec;
      EOL_Format          : out EOL_Formats;
      Comments_Special_On : Boolean;
      Max_Tokens          : Tokn_Index := Tokn_Index'Last;
      Lang                : Language := Ada_Lang);
   --  Return in Result the sequence of tokens in the Input string. The first
   --  one is always Start_Of_Input, and the last one End_Of_Input.  EOL_Format
   --  is set to the convention used by the input file. Comments_Special_On is
   --  True if the --comments-special switch was given. Max_Tokens places a
   --  limit on the number of tokens (not counting Start_Of_Input); we quit
   --  before reaching end of input if we've gotten that many.

   function Get_Tokns
     (Input               : in out Buffers.Buffer;
      Result              : out Tokn_Vec;
      Comments_Special_On : Boolean;
      Max_Tokens          : Tokn_Index := Tokn_Index'Last;
      Lang                : Language := Ada_Lang)
     return Boolean;
   --  This is to get around the annoying restriction in Ada that you can't mix
   --  declarations and statements. It does the same thing as the procedure,
   --  and the result is to be ignored. ???It might make sense to return the
   --  Tokn_Vec instead of a dummy Boolean, but that might be too inefficient.

   procedure Move_Tokns (Target, Source : in out Tokn_Vec);
   function Move_Tokns (Target, Source : in out Tokn_Vec) return Boolean;
   --  Move Source to Target, leaving Source empty

   function Origin (X : Tokn_Cursor) return Syms.Symbol;

   function Same_Token (X, Y : Token) return Boolean;
   procedure Check_Same_Token
     (X, Y : Tokn_Cursor; Message, Name_1, Name_2 : String);
   procedure Check_Same_Tokens
     (X, Y : Tokn_Vec; Message, Name_1, Name_2 : String);
   --  Checks that X and Y are the same except for Slocs and line breaks; raise
   --  an exception if not.

   procedure Put_Token
     (Tok  : Tokn_Cursor;
      File : Ada.Text_IO.File_Type := Ada.Text_IO.Standard_Output);
   procedure Put_Tokens
     (First      : Tokn_Cursor;
      After_Last : Tokn_Cursor;
      Highlight  : Tokn_Cursor;
      File       : Ada.Text_IO.File_Type := Ada.Text_IO.Standard_Output);
   --  Put token(s) to standard output (even if Text_IO.Current_Output has been
   --  redirected). The tokens come out in compilable form, one per line, with
   --  the text of the token first, and the other information commented out.
   --  This one-token-per line code can be used for testing the scanner -- it
   --  should have identical semantics to the original Ada code. First and Last
   --  indicate a slice of Tokens, and we tolerate out-of-bounds indices.
   --  We draw a comment line before Highlight.
   procedure Put_Tokens
     (Highlight : Tokn_Cursor;
      Num_Toks  : Tokn_Index := 8;
      File      : Ada.Text_IO.File_Type := Ada.Text_IO.Standard_Output);
   --  Num_Toks is the number of tokens before and after Highlight to print
   procedure Put_Tokens
     (Tokens : Tokn_Vec;
      File   : Ada.Text_IO.File_Type := Ada.Text_IO.Standard_Output);
   procedure Put_Tokns
     (Tok  : Tokn_Cursor;
      File : Ada.Text_IO.File_Type := Ada.Text_IO.Standard_Output);
   --  This puts all the tokens in the vector that Tok points to, highlighting
   --  Tok.

   Show_Origin : Boolean := False with Export;
   --  Set to True in debugger to see Origins

   Check_Comment_Length : Boolean := True;
   --  True if Append_Tokn should check that the length of a comment is
   --  correct. It will not be correct during the Insert_Alignment phase.

private

   type Token (Kind : Opt_Token_Kind := Nil) is record
      Sloc : Source_Location;
      case Kind is
         when Line_Break_Token | Tab_Token =>
            Index : Positive;
            --  This is really of type Line_Break_Index or Tab_Index, but we
            --  don't want to depend on Pp.Formatting, where those are
            --  declared. See child package Lines for a well-typed interface.

         when Stored_Text_Kind =>
            Text : Syms.Symbol;
            case Kind is
               when Comment_Kind =>
                  Leading_Blanks : Natural;
                  case Kind is
                     when Whole_Line_Comment =>
                        Width : Positive;

                        --  The following are used to compute the Sloc of the
                        --  EOL_Token token that follows the comment.

                        Num_Lines : Positive;
                        --  Number of lines in the comment; this can be greater
                        --  than one for comments combined into a fillable
                        --  comment "paragraph".
                        Last_Line_Len : Positive;
                        --  Length of the last line of the comment
                     when others => null;
                  end case;
               when others => null;
            end case;
         when others => null;
      end case;
   end record; -- Token

   subtype Opt_Token is Token;
   --  Opt_Token is used when Kind can be Nil; otherwise Token is used

   use Utils.Var_Length_Ints;

   --  Each token in a Tokn_Vec consists of a fixed-length part, and possibly a
   --  variable-length part stored in Octets. See Var_Length_Ints for the
   --  encoding.

   type Fixed_Part is record
      Kind : Token_Kind;
      Sloc_Line : Positive;
      Sloc_Col : Positive;
      Sloc_First : Positive;
      Origin : Syms.Symbol;
   end record;

   type Fixed_Part_Array is array (Tokn_Index range <>) of Fixed_Part;

   package Fixed_Part_Vectors is new Utils.Vectors
     (Tokn_Index, Fixed_Part, Fixed_Part_Array);
   subtype Fixed_Part_Vector is Fixed_Part_Vectors.Vector;

   type Tokn_Vec is record
      New_Sloc_First : Positive := 1;
      Fixed : Fixed_Part_Vector;
      Octets : Octet_Vector;
   end record;

   type Tokn_Cursor (V : Tokn_Vec_Ref := null) is record
      Fi : Tokn_Index := 666_666; -- Pointer into V.Fixed
      Oc : Octet_Index := 666_666; -- Pointer into V.Octets
   end record;

   --  The following are private, for use by child package Lines:

   function Index (X : Tokn_Cursor) return Positive with
     Pre => Kind (X) in Line_Break_Token | Tab_Token;

   procedure Append_Tokn_With_Index
     (V     : in out Tokn_Vec;
      X     : Token_Kind;
      Index : Positive;
      Len   : Natural := 0;
      Org   : String := "Append Kind") with
     Pre => X in Line_Break_Token | Tab_Token;

end Pp.Scanner;
