Starshatter_Open
Open source Starshatter engine
 All Classes Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
Token.h
Go to the documentation of this file.
1 /* Project STARSHATTER
2  John DiCamillo
3  Copyright © 1997-2001. All Rights Reserved.
4 
5  SUBSYSTEM: Parser
6  FILE: Token.h
7  AUTHOR: John DiCamillo
8 
9 
10  OVERVIEW
11  ========
12  Scanner class definition
13 */
14 
15 #ifndef Token_h
16 #define Token_h
17 
18 #include "Text.h"
19 #include "Dictionary.h"
20 
21 #pragma warning( disable : 4237)
22 
23 // +-------------------------------------------------------------------+
24 
25 class Reader;
26 class Token;
27 class Scanner;
28 
29 // +-------------------------------------------------------------------+
30 
31 class Token
32 {
33  friend class Scanner;
34 
35 public:
36  // keywords must be alphanumeric identifiers or symbolic identifiers
37  enum Types { Undefined, Keyword, AlphaIdent, SymbolicIdent, Comment,
38  IntLiteral, FloatLiteral, StringLiteral, CharLiteral,
39  Dot, Comma, Colon, Semicolon,
40  LParen, RParen, LBracket, RBracket, LBrace, RBrace,
41  EOT, LastTokenType };
42 
43  enum Alias { CompoundSeparator = Dot,
44  ItemSeparator = Comma,
45  StatementTerminator = Semicolon,
46  TypeIndicator = Colon,
47  Lambda = LastTokenType + 1 };
48 
49  Token();
50  Token(const Token& rhs);
51  Token(int t);
52  Token(const char* s, int t, int k=0, int l=0, int c=0);
53  Token(const Text& s, int t, int k=0, int l=0, int c=0);
54  ~Token();
55 
56  Token& operator = (const Token& rhs);
57 
58  bool match(const Token& ref) const;
59 
60  Text symbol() const;
61  int type() const { return mType; }
62  int key() const { return mKey; }
63  int line() const { return mLine; }
64  int column() const { return mColumn; }
65 
66  Text typestr() const;
67 
68  static Text describe(const Text& tok);
69  static void addKey(const Text& k, int v);
70  static void addKeys(Dictionary<int>& keys);
71  static bool findKey(const Text& k, int& v);
72  static void comments(const Text& begin, const Text& end);
73  static void altComments(const Text& begin, const Text& end);
74  static void hideComments(bool hide = true) { hidecom = hide; }
75 
76  static char comBeg(unsigned int i) { return combeg[i]; }
77  static char comEnd(unsigned int i) { return comend[i]; }
78  static char altBeg(unsigned int i) { return altbeg[i]; }
79  static char altEnd(unsigned int i) { return altend[i]; }
80 
81  static void close();
82 
83 protected:
84  int mLength;
85  union {
86  char mSymbol[8];
87  char* mFullSymbol;
88  };
89  int mType;
90  int mKey;
91  int mLine;
92  int mColumn;
93 
94  static bool hidecom;
95  static char combeg[3];
96  static char comend[3];
97  static char altbeg[3];
98  static char altend[3];
99 
100  static Dictionary<int> keymap;
101 };
102 
103 // +-------------------------------------------------------------------+
104 
105 class Scanner
106 {
107 public:
108  Scanner(Reader* r = 0);
109  Scanner(const Text& s);
110  Scanner(const Scanner& rhs);
111  virtual ~Scanner();
112 
113  Scanner& operator = (const Scanner& rhs);
114 
115  void Load(const Text& s);
116 
117  enum Need { Demand, Request };
118  virtual Token Get(Need n = Demand);
119 
120  void PutBack() { index = old_index; line = old_line; }
121  int GetCursor() { return index; }
122  int GetLine() { return line; }
123  void Reset(int c, int l) { index = old_index = c; line = old_line = l; }
124  Token Best() const { return best; }
125 
126 protected:
127  virtual int GetNumeric();
128  virtual bool IsSymbolic(char c);
129  virtual bool IsAlpha(char c);
130 
131  Reader* reader;
132  char* str;
133 
134  const char* p;
135  const char* eos;
136 
137  size_t index;
138  size_t old_index;
139  Token best;
140  size_t length;
141  size_t line;
142  size_t old_line;
143  size_t lineStart;
144 };
145 
146 #endif // TOKEN_H