From 1a6f1241eb85c82d4fddf7b61a867a1bb828992e Mon Sep 17 00:00:00 2001 From: Aki Date: Wed, 20 Mar 2024 00:43:53 +0100 Subject: Tweaked DefinitionEx include paths; also Parser_ss.h name is finally fixed --- DefinitionEx/CMakeLists.txt | 2 +- DefinitionEx/include/Parser_ss.h | 43 --- DefinitionEx/include/Term.h | 167 ----------- DefinitionEx/include/Token.h | 143 ---------- DefinitionEx/include/starshatter/definition.h | 11 + .../include/starshatter/definition/Parser.h | 43 +++ DefinitionEx/include/starshatter/definition/Term.h | 167 +++++++++++ .../include/starshatter/definition/Token.h | 143 ++++++++++ DefinitionEx/src/Parser.cpp | 316 +++++++++++++++++++++ DefinitionEx/src/Parser_ss.cpp | 316 --------------------- DefinitionEx/src/Term.cpp | 2 +- DefinitionEx/src/Token.cpp | 2 +- DefinitionEx/test/parse_utils_like.cpp | 4 +- 13 files changed, 684 insertions(+), 675 deletions(-) delete mode 100644 DefinitionEx/include/Parser_ss.h delete mode 100644 DefinitionEx/include/Term.h delete mode 100644 DefinitionEx/include/Token.h create mode 100644 DefinitionEx/include/starshatter/definition.h create mode 100644 DefinitionEx/include/starshatter/definition/Parser.h create mode 100644 DefinitionEx/include/starshatter/definition/Term.h create mode 100644 DefinitionEx/include/starshatter/definition/Token.h create mode 100644 DefinitionEx/src/Parser.cpp delete mode 100644 DefinitionEx/src/Parser_ss.cpp (limited to 'DefinitionEx') diff --git a/DefinitionEx/CMakeLists.txt b/DefinitionEx/CMakeLists.txt index f098670..469f72a 100644 --- a/DefinitionEx/CMakeLists.txt +++ b/DefinitionEx/CMakeLists.txt @@ -2,7 +2,7 @@ project(DefinitionEx) add_library( ${PROJECT_NAME} STATIC - src/Parser_ss.cpp + src/Parser.cpp src/Term.cpp src/Token.cpp ) diff --git a/DefinitionEx/include/Parser_ss.h b/DefinitionEx/include/Parser_ss.h deleted file mode 100644 index 50659b9..0000000 --- a/DefinitionEx/include/Parser_ss.h +++ /dev/null @@ -1,43 +0,0 @@ -/* Starshatter: The Open Source Project - Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors - Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors - Copyright (c) 1997-2006, Destroyer Studios LLC. - - AUTHOR: John DiCamillo - - - OVERVIEW - ======== - Declaration of the generic Parser class -*/ - -#pragma once - -#include -#include - -#include "Term.h" - -// +-------------------------------------------------------------------+ - -class Scanner; - -// +-------------------------------------------------------------------+ - -class Parser -{ -public: - Parser(); - Parser(starshatter::foundation::Reader r); - ~Parser(); - - Term* ParseTerm(); - Term* ParseTermBase(); - Term* ParseTermRest(Term* base); - TermList* ParseTermList(int for_struct); - TermArray* ParseArray(); - TermStruct* ParseStruct(); - -private: - Scanner* lexer; -}; diff --git a/DefinitionEx/include/Term.h b/DefinitionEx/include/Term.h deleted file mode 100644 index 1bc30b6..0000000 --- a/DefinitionEx/include/Term.h +++ /dev/null @@ -1,167 +0,0 @@ -/* Starshatter: The Open Source Project - Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors - Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors - Copyright (c) 1997-2006, Destroyer Studios LLC. - - AUTHOR: John DiCamillo - - - OVERVIEW - ======== - Declaration of the Abstract Syntax Tree classes -*/ - -#pragma once - -#include -#include - -// +-------------------------------------------------------------------+ - -class Term; -class TermBool; -class TermNumber; -class TermText; -class TermArray; -class TermDef; -class TermStruct; - -// +-------------------------------------------------------------------+ - -class Term -{ -public: - static const char* TYPENAME() { return "Term"; } - - Term() { } - virtual ~Term() { } - - virtual int operator==(const Term& rhs) const { return 0; } - - virtual void print(int level=10) { } - - // conversion tests - virtual Term* touch() { return this; } - virtual TermBool* isBool() { return 0; } - virtual TermNumber* isNumber() { return 0; } - virtual TermText* isText() { return 0; } - virtual TermArray* isArray() { return 0; } - virtual TermDef* isDef() { return 0; } - virtual TermStruct* isStruct() { return 0; } -}; - -Term* error(const char*, const char* = 0); - -// +-------------------------------------------------------------------+ - -typedef List TermList; -typedef ListIter TermListIter; - -// +-------------------------------------------------------------------+ - -class TermBool : public Term -{ -public: - static const char* TYPENAME() { return "TermBool"; } - - TermBool(bool v) : val(v) { } - - virtual void print(int level=10); - virtual TermBool* isBool() { return this; } - bool value() const { return val; } - -private: - bool val; -}; - -// +-------------------------------------------------------------------+ - -class TermNumber : public Term -{ -public: - static const char* TYPENAME() { return "TermNumber"; } - - TermNumber(double v) : val(v) { } - - virtual void print(int level=10); - virtual TermNumber* isNumber() { return this; } - double value() const { return val; } - -private: - double val; -}; - -// +-------------------------------------------------------------------+ - -class TermText : public Term -{ -public: - static const char* TYPENAME() { return "TermText"; } - - TermText(const Text& v) : val(v) { } - - virtual void print(int level=10); - virtual TermText* isText() { return this; } - Text value() const { return val; } - -private: - Text val; -}; - -// +-------------------------------------------------------------------+ - -class TermArray : public Term -{ -public: - static const char* TYPENAME() { return "TermArray"; } - - TermArray(TermList* elist); - virtual ~TermArray(); - - virtual void print(int level=10); - virtual TermArray* isArray() { return this; } - TermList* elements() { return elems; } - -private: - TermList* elems; -}; - -// +-------------------------------------------------------------------+ - -class TermStruct : public Term -{ -public: - static const char* TYPENAME() { return "TermStruct"; } - - TermStruct(TermList* elist); - virtual ~TermStruct(); - - virtual void print(int level=10); - - virtual TermStruct* isStruct() { return this; } - TermList* elements() { return elems; } - -private: - TermList* elems; -}; - -// +-------------------------------------------------------------------+ - -class TermDef : public Term -{ -public: - static const char* TYPENAME() { return "TermDef"; } - - TermDef(TermText* n, Term* v) : mname(n), mval(v) { } - virtual ~TermDef(); - - virtual void print(int level=10); - virtual TermDef* isDef() { return this; } - - virtual TermText* name() { return mname; } - virtual Term* term() { return mval; } - -private: - TermText* mname; - Term* mval; -}; diff --git a/DefinitionEx/include/Token.h b/DefinitionEx/include/Token.h deleted file mode 100644 index 3dd8eb9..0000000 --- a/DefinitionEx/include/Token.h +++ /dev/null @@ -1,143 +0,0 @@ -/* Starshatter: The Open Source Project - Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors - Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors - Copyright (c) 1997-2006, Destroyer Studios LLC. - - AUTHOR: John DiCamillo - - - OVERVIEW - ======== - Scanner class definition -*/ - -#pragma once - -#include -#include -#include - -#pragma warning( disable : 4237) - -// +-------------------------------------------------------------------+ - -class Token; -class Scanner; - -// +-------------------------------------------------------------------+ - -class Token -{ - friend class Scanner; - -public: - // keywords must be alphanumeric identifiers or symbolic identifiers - enum Types { Undefined, Keyword, AlphaIdent, SymbolicIdent, Comment, - IntLiteral, FloatLiteral, StringLiteral, CharLiteral, - Dot, Comma, Colon, Semicolon, - LParen, RParen, LBracket, RBracket, LBrace, RBrace, - EOT, LastTokenType }; - - enum Alias { CompoundSeparator = Dot, - ItemSeparator = Comma, - StatementTerminator = Semicolon, - TypeIndicator = Colon, - Lambda = LastTokenType + 1 }; - - Token(); - Token(const Token& rhs); - Token(int t); - Token(const char* s, int t, int k=0, int l=0, int c=0); - Token(const Text& s, int t, int k=0, int l=0, int c=0); - ~Token(); - - Token& operator = (const Token& rhs); - - bool match(const Token& ref) const; - - Text symbol() const; - int type() const { return mType; } - int key() const { return mKey; } - int line() const { return mLine; } - int column() const { return mColumn; } - - Text typestr() const; - - static Text describe(const Text& tok); - static void addKey(const Text& k, int v); - static void addKeys(Dictionary& keys); - static bool findKey(const Text& k, int& v); - static void comments(const Text& begin, const Text& end); - static void altComments(const Text& begin, const Text& end); - static void hideComments(bool hide = true) { hidecom = hide; } - - static char comBeg(unsigned int i) { return combeg[i]; } - static char comEnd(unsigned int i) { return comend[i]; } - static char altBeg(unsigned int i) { return altbeg[i]; } - static char altEnd(unsigned int i) { return altend[i]; } - - static void close(); - -protected: - int mLength; - union { - char mSymbol[8]; - char* mFullSymbol; - }; - int mType; - int mKey; - int mLine; - int mColumn; - - static bool hidecom; - static char combeg[3]; - static char comend[3]; - static char altbeg[3]; - static char altend[3]; - - static Dictionary keymap; -}; - -// +-------------------------------------------------------------------+ - -class Scanner -{ -public: - Scanner(); - Scanner(starshatter::foundation::Reader r); - Scanner(const Text& s); - Scanner(const Scanner& rhs); - virtual ~Scanner(); - - Scanner& operator = (const Scanner& rhs); - - void Load(const Text& s); - - enum Need { Demand, Request }; - virtual Token Get(Need n = Demand); - - void PutBack() { index = old_index; line = old_line; } - int GetCursor() { return index; } - int GetLine() { return line; } - void Reset(int c, int l) { index = old_index = c; line = old_line = l; } - Token Best() const { return best; } - -protected: - virtual int GetNumeric(); - virtual bool IsSymbolic(char c); - virtual bool IsAlpha(char c); - - starshatter::foundation::Reader reader; - char* str; - - const char* p; - const char* eos; - - size_t index; - size_t old_index; - Token best; - size_t length; - size_t line; - size_t old_line; - size_t lineStart; -}; diff --git a/DefinitionEx/include/starshatter/definition.h b/DefinitionEx/include/starshatter/definition.h new file mode 100644 index 0000000..785261c --- /dev/null +++ b/DefinitionEx/include/starshatter/definition.h @@ -0,0 +1,11 @@ +/* Starshatter: The Open Source Project + Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors + Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors + Copyright (c) 1997-2006, Destroyer Studios LLC. +*/ + +#pragma once + +#include "definition/Parser.h" +#include "definition/Term.h" +#include "definition/Token.h" diff --git a/DefinitionEx/include/starshatter/definition/Parser.h b/DefinitionEx/include/starshatter/definition/Parser.h new file mode 100644 index 0000000..50659b9 --- /dev/null +++ b/DefinitionEx/include/starshatter/definition/Parser.h @@ -0,0 +1,43 @@ +/* Starshatter: The Open Source Project + Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors + Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors + Copyright (c) 1997-2006, Destroyer Studios LLC. + + AUTHOR: John DiCamillo + + + OVERVIEW + ======== + Declaration of the generic Parser class +*/ + +#pragma once + +#include +#include + +#include "Term.h" + +// +-------------------------------------------------------------------+ + +class Scanner; + +// +-------------------------------------------------------------------+ + +class Parser +{ +public: + Parser(); + Parser(starshatter::foundation::Reader r); + ~Parser(); + + Term* ParseTerm(); + Term* ParseTermBase(); + Term* ParseTermRest(Term* base); + TermList* ParseTermList(int for_struct); + TermArray* ParseArray(); + TermStruct* ParseStruct(); + +private: + Scanner* lexer; +}; diff --git a/DefinitionEx/include/starshatter/definition/Term.h b/DefinitionEx/include/starshatter/definition/Term.h new file mode 100644 index 0000000..1bc30b6 --- /dev/null +++ b/DefinitionEx/include/starshatter/definition/Term.h @@ -0,0 +1,167 @@ +/* Starshatter: The Open Source Project + Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors + Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors + Copyright (c) 1997-2006, Destroyer Studios LLC. + + AUTHOR: John DiCamillo + + + OVERVIEW + ======== + Declaration of the Abstract Syntax Tree classes +*/ + +#pragma once + +#include +#include + +// +-------------------------------------------------------------------+ + +class Term; +class TermBool; +class TermNumber; +class TermText; +class TermArray; +class TermDef; +class TermStruct; + +// +-------------------------------------------------------------------+ + +class Term +{ +public: + static const char* TYPENAME() { return "Term"; } + + Term() { } + virtual ~Term() { } + + virtual int operator==(const Term& rhs) const { return 0; } + + virtual void print(int level=10) { } + + // conversion tests + virtual Term* touch() { return this; } + virtual TermBool* isBool() { return 0; } + virtual TermNumber* isNumber() { return 0; } + virtual TermText* isText() { return 0; } + virtual TermArray* isArray() { return 0; } + virtual TermDef* isDef() { return 0; } + virtual TermStruct* isStruct() { return 0; } +}; + +Term* error(const char*, const char* = 0); + +// +-------------------------------------------------------------------+ + +typedef List TermList; +typedef ListIter TermListIter; + +// +-------------------------------------------------------------------+ + +class TermBool : public Term +{ +public: + static const char* TYPENAME() { return "TermBool"; } + + TermBool(bool v) : val(v) { } + + virtual void print(int level=10); + virtual TermBool* isBool() { return this; } + bool value() const { return val; } + +private: + bool val; +}; + +// +-------------------------------------------------------------------+ + +class TermNumber : public Term +{ +public: + static const char* TYPENAME() { return "TermNumber"; } + + TermNumber(double v) : val(v) { } + + virtual void print(int level=10); + virtual TermNumber* isNumber() { return this; } + double value() const { return val; } + +private: + double val; +}; + +// +-------------------------------------------------------------------+ + +class TermText : public Term +{ +public: + static const char* TYPENAME() { return "TermText"; } + + TermText(const Text& v) : val(v) { } + + virtual void print(int level=10); + virtual TermText* isText() { return this; } + Text value() const { return val; } + +private: + Text val; +}; + +// +-------------------------------------------------------------------+ + +class TermArray : public Term +{ +public: + static const char* TYPENAME() { return "TermArray"; } + + TermArray(TermList* elist); + virtual ~TermArray(); + + virtual void print(int level=10); + virtual TermArray* isArray() { return this; } + TermList* elements() { return elems; } + +private: + TermList* elems; +}; + +// +-------------------------------------------------------------------+ + +class TermStruct : public Term +{ +public: + static const char* TYPENAME() { return "TermStruct"; } + + TermStruct(TermList* elist); + virtual ~TermStruct(); + + virtual void print(int level=10); + + virtual TermStruct* isStruct() { return this; } + TermList* elements() { return elems; } + +private: + TermList* elems; +}; + +// +-------------------------------------------------------------------+ + +class TermDef : public Term +{ +public: + static const char* TYPENAME() { return "TermDef"; } + + TermDef(TermText* n, Term* v) : mname(n), mval(v) { } + virtual ~TermDef(); + + virtual void print(int level=10); + virtual TermDef* isDef() { return this; } + + virtual TermText* name() { return mname; } + virtual Term* term() { return mval; } + +private: + TermText* mname; + Term* mval; +}; diff --git a/DefinitionEx/include/starshatter/definition/Token.h b/DefinitionEx/include/starshatter/definition/Token.h new file mode 100644 index 0000000..3dd8eb9 --- /dev/null +++ b/DefinitionEx/include/starshatter/definition/Token.h @@ -0,0 +1,143 @@ +/* Starshatter: The Open Source Project + Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors + Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors + Copyright (c) 1997-2006, Destroyer Studios LLC. + + AUTHOR: John DiCamillo + + + OVERVIEW + ======== + Scanner class definition +*/ + +#pragma once + +#include +#include +#include + +#pragma warning( disable : 4237) + +// +-------------------------------------------------------------------+ + +class Token; +class Scanner; + +// +-------------------------------------------------------------------+ + +class Token +{ + friend class Scanner; + +public: + // keywords must be alphanumeric identifiers or symbolic identifiers + enum Types { Undefined, Keyword, AlphaIdent, SymbolicIdent, Comment, + IntLiteral, FloatLiteral, StringLiteral, CharLiteral, + Dot, Comma, Colon, Semicolon, + LParen, RParen, LBracket, RBracket, LBrace, RBrace, + EOT, LastTokenType }; + + enum Alias { CompoundSeparator = Dot, + ItemSeparator = Comma, + StatementTerminator = Semicolon, + TypeIndicator = Colon, + Lambda = LastTokenType + 1 }; + + Token(); + Token(const Token& rhs); + Token(int t); + Token(const char* s, int t, int k=0, int l=0, int c=0); + Token(const Text& s, int t, int k=0, int l=0, int c=0); + ~Token(); + + Token& operator = (const Token& rhs); + + bool match(const Token& ref) const; + + Text symbol() const; + int type() const { return mType; } + int key() const { return mKey; } + int line() const { return mLine; } + int column() const { return mColumn; } + + Text typestr() const; + + static Text describe(const Text& tok); + static void addKey(const Text& k, int v); + static void addKeys(Dictionary& keys); + static bool findKey(const Text& k, int& v); + static void comments(const Text& begin, const Text& end); + static void altComments(const Text& begin, const Text& end); + static void hideComments(bool hide = true) { hidecom = hide; } + + static char comBeg(unsigned int i) { return combeg[i]; } + static char comEnd(unsigned int i) { return comend[i]; } + static char altBeg(unsigned int i) { return altbeg[i]; } + static char altEnd(unsigned int i) { return altend[i]; } + + static void close(); + +protected: + int mLength; + union { + char mSymbol[8]; + char* mFullSymbol; + }; + int mType; + int mKey; + int mLine; + int mColumn; + + static bool hidecom; + static char combeg[3]; + static char comend[3]; + static char altbeg[3]; + static char altend[3]; + + static Dictionary keymap; +}; + +// +-------------------------------------------------------------------+ + +class Scanner +{ +public: + Scanner(); + Scanner(starshatter::foundation::Reader r); + Scanner(const Text& s); + Scanner(const Scanner& rhs); + virtual ~Scanner(); + + Scanner& operator = (const Scanner& rhs); + + void Load(const Text& s); + + enum Need { Demand, Request }; + virtual Token Get(Need n = Demand); + + void PutBack() { index = old_index; line = old_line; } + int GetCursor() { return index; } + int GetLine() { return line; } + void Reset(int c, int l) { index = old_index = c; line = old_line = l; } + Token Best() const { return best; } + +protected: + virtual int GetNumeric(); + virtual bool IsSymbolic(char c); + virtual bool IsAlpha(char c); + + starshatter::foundation::Reader reader; + char* str; + + const char* p; + const char* eos; + + size_t index; + size_t old_index; + Token best; + size_t length; + size_t line; + size_t old_line; + size_t lineStart; +}; diff --git a/DefinitionEx/src/Parser.cpp b/DefinitionEx/src/Parser.cpp new file mode 100644 index 0000000..2453819 --- /dev/null +++ b/DefinitionEx/src/Parser.cpp @@ -0,0 +1,316 @@ +/* Starshatter: The Open Source Project + Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors + Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors + Copyright (c) 1997-2006, Destroyer Studios LLC. + + AUTHOR: John DiCamillo + + + OVERVIEW + ======== + Implementation of the generic Parser class +*/ + +#include + +#include +#include + +#include + +#include +#include + +#include +#include + +enum KEYS { KEY_TRUE, KEY_FALSE, KEY_DEF, KEY_MINUS }; + +static int dump_tokens = 0; + +// +-------------------------------------------------------------------+ + +Term* error(const char* msg, const Token& token) +{ + static char buf[1024]; + snprintf(buf, 1024, " near '%s' in line %d", (const char*) token.symbol(), token.line()); + return error(msg, buf); +} + +// +-------------------------------------------------------------------+ + +Parser::Parser() : + lexer {nullptr} +{ + Token::addKey("true", KEY_TRUE); + Token::addKey("false", KEY_FALSE); + Token::addKey(":", KEY_DEF); + Token::addKey("-", KEY_MINUS); +} + + +Parser::Parser(starshatter::foundation::Reader r) : + lexer {new Scanner(std::move(r))} +{ + Token::addKey("true", KEY_TRUE); + Token::addKey("false", KEY_FALSE); + Token::addKey(":", KEY_DEF); + Token::addKey("-", KEY_MINUS); +} + +Parser::~Parser() +{ + if (lexer) + delete lexer; + //Token::close(); +} + +Term* +Parser::ParseTerm() +{ + Term* t = ParseTermBase(); + if (t == 0) return t; + + Term* t2 = ParseTermRest(t); + + return t2; +} + +Term* +Parser::ParseTermRest(Term* base) +{ + Token t = lexer->Get(); + + switch (t.type()) { + default: + lexer->PutBack(); + return base; + + case Token::StringLiteral: { + // concatenate adjacent string literal tokens: + TermText* text = base->isText(); + if (text) { + TermText* base2 = new TermText(text->value() + t.symbol()(1, t.symbol().length()-2)); + delete base; + return ParseTermRest(base2); + } + else { + lexer->PutBack(); + } + } + break; + + case Token::Keyword: + switch (t.key()) { + case KEY_DEF: + if (base->isText()) + return new TermDef(base->isText(), ParseTerm()); + else + return error("(Parse) illegal lhs in def", t); + + default: + lexer->PutBack(); + return base; + } + break; + } + + return base; +} + +static int xtol(const char* p) +{ + int n = 0; + + while (*p) { + char digit = *p++; + n *= 16; + + if (digit >= '0' && digit <= '9') + n += digit - '0'; + + else if (digit >= 'a' && digit <= 'f') + n += digit - 'a' + 10; + + else if (digit >= 'A' && digit <= 'F') + n += digit - 'A' + 10; + } + + return n; +} + +Term* +Parser::ParseTermBase() +{ + Token t = lexer->Get(); + int n = 0; + double d = 0.0; + + switch (t.type()) { + case Token::IntLiteral: { + if (dump_tokens) + Print("%s", t.symbol().data()); + + char nstr[256], *p = nstr; + for (int i = 0; i < (int) t.symbol().length(); i++) + if (t.symbol()[i] != '_') + *p++ = t.symbol()[i]; + *p++ = '\0'; + + // handle hex notation: + if (nstr[1] == 'x') + n = xtol(nstr+2); + + else + n = atol(nstr); + + return new TermNumber(n); + } + + case Token::FloatLiteral: { + if (dump_tokens) + Print("%s", t.symbol().data()); + + char nstr[256], *p = nstr; + for (int i = 0; i < (int) t.symbol().length(); i++) + if (t.symbol()[i] != '_') + *p++ = t.symbol()[i]; + *p++ = '\0'; + + d = atof(nstr); + return new TermNumber(d); + } + + case Token::StringLiteral: + if (dump_tokens) + Print("%s", t.symbol().data()); + + return new TermText(t.symbol()(1, t.symbol().length()-2)); + + case Token::AlphaIdent: + if (dump_tokens) + Print("%s", t.symbol().data()); + + return new TermText(t.symbol()); + + case Token::Keyword: + if (dump_tokens) + Print("%s", t.symbol().data()); + + switch (t.key()) { + case KEY_FALSE: return new TermBool(0); + case KEY_TRUE: return new TermBool(1); + + case KEY_MINUS: { + Token next = lexer->Get(); + if (next.type() == Token::IntLiteral) { + if (dump_tokens) + Print("%s", next.symbol().data()); + + char nstr[256], *p = nstr; + for (int i = 0; i < (int) next.symbol().length(); i++) + if (next.symbol()[i] != '_') + *p++ = next.symbol()[i]; + *p++ = '\0'; + + n = -1 * atol(nstr); + return new TermNumber(n); + } + else if (next.type() == Token::FloatLiteral) { + if (dump_tokens) + Print("%s", next.symbol().data()); + + char nstr[256], *p = nstr; + for (int i = 0; i < (int) next.symbol().length(); i++) + if (next.symbol()[i] != '_') + *p++ = next.symbol()[i]; + *p++ = '\0'; + + d = -1.0 * atof(nstr); + return new TermNumber(d); + } + else { + lexer->PutBack(); + return error("(Parse) illegal token '-': number expected", next); + } + } + break; + + default: + lexer->PutBack(); + return 0; + } + + case Token::LParen: return ParseArray(); + + case Token::LBrace: return ParseStruct(); + + case Token::CharLiteral: + return error("(Parse) illegal token ", t); + + default: + lexer->PutBack(); + return 0; + } +} + +TermArray* +Parser::ParseArray() +{ + TermList* elems = ParseTermList(0); + Token end = lexer->Get(); + + if (end.type() != Token::RParen) + return (TermArray*) error("(Parse) ')' missing in array-decl", end); + + return new TermArray(elems); +} + +TermStruct* +Parser::ParseStruct() +{ + TermList* elems = ParseTermList(1); + Token end = lexer->Get(); + + if (end.type() != Token::RBrace) + return (TermStruct*) error("(Parse) '}' missing in struct", end); + + return new TermStruct(elems); +} + +TermList* +Parser::ParseTermList(int for_struct) +{ + TermList* tlist = new TermList; + + Term* term = ParseTerm(); + while (term) { + if (for_struct && !term->isDef()) { + return (TermList*) error("(Parse) non-definition term in struct"); + } + else if (!for_struct && term->isDef()) { + return (TermList*) error("(Parse) illegal definition in array"); + } + + tlist->append(term); + Token t = lexer->Get(); + + /*** OLD WAY: COMMA SEPARATORS REQUIRED *** + if (t.type() != Token::Comma) { + lexer->PutBack(); + term = 0; + } + else + term = ParseTerm(); + /*******************************************/ + + // NEW WAY: COMMA SEPARATORS OPTIONAL: + if (t.type() != Token::Comma) { + lexer->PutBack(); + } + + term = ParseTerm(); + } + + return tlist; +} diff --git a/DefinitionEx/src/Parser_ss.cpp b/DefinitionEx/src/Parser_ss.cpp deleted file mode 100644 index 1b80d48..0000000 --- a/DefinitionEx/src/Parser_ss.cpp +++ /dev/null @@ -1,316 +0,0 @@ -/* Starshatter: The Open Source Project - Copyright (c) 2021-2024, Starshatter: The Open Source Project Contributors - Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors - Copyright (c) 1997-2006, Destroyer Studios LLC. - - AUTHOR: John DiCamillo - - - OVERVIEW - ======== - Implementation of the generic Parser class -*/ - -#include - -#include -#include - -#include - -#include -#include - -#include -#include - -enum KEYS { KEY_TRUE, KEY_FALSE, KEY_DEF, KEY_MINUS }; - -static int dump_tokens = 0; - -// +-------------------------------------------------------------------+ - -Term* error(const char* msg, const Token& token) -{ - static char buf[1024]; - snprintf(buf, 1024, " near '%s' in line %d", (const char*) token.symbol(), token.line()); - return error(msg, buf); -} - -// +-------------------------------------------------------------------+ - -Parser::Parser() : - lexer {nullptr} -{ - Token::addKey("true", KEY_TRUE); - Token::addKey("false", KEY_FALSE); - Token::addKey(":", KEY_DEF); - Token::addKey("-", KEY_MINUS); -} - - -Parser::Parser(starshatter::foundation::Reader r) : - lexer {new Scanner(std::move(r))} -{ - Token::addKey("true", KEY_TRUE); - Token::addKey("false", KEY_FALSE); - Token::addKey(":", KEY_DEF); - Token::addKey("-", KEY_MINUS); -} - -Parser::~Parser() -{ - if (lexer) - delete lexer; - //Token::close(); -} - -Term* -Parser::ParseTerm() -{ - Term* t = ParseTermBase(); - if (t == 0) return t; - - Term* t2 = ParseTermRest(t); - - return t2; -} - -Term* -Parser::ParseTermRest(Term* base) -{ - Token t = lexer->Get(); - - switch (t.type()) { - default: - lexer->PutBack(); - return base; - - case Token::StringLiteral: { - // concatenate adjacent string literal tokens: - TermText* text = base->isText(); - if (text) { - TermText* base2 = new TermText(text->value() + t.symbol()(1, t.symbol().length()-2)); - delete base; - return ParseTermRest(base2); - } - else { - lexer->PutBack(); - } - } - break; - - case Token::Keyword: - switch (t.key()) { - case KEY_DEF: - if (base->isText()) - return new TermDef(base->isText(), ParseTerm()); - else - return error("(Parse) illegal lhs in def", t); - - default: - lexer->PutBack(); - return base; - } - break; - } - - return base; -} - -static int xtol(const char* p) -{ - int n = 0; - - while (*p) { - char digit = *p++; - n *= 16; - - if (digit >= '0' && digit <= '9') - n += digit - '0'; - - else if (digit >= 'a' && digit <= 'f') - n += digit - 'a' + 10; - - else if (digit >= 'A' && digit <= 'F') - n += digit - 'A' + 10; - } - - return n; -} - -Term* -Parser::ParseTermBase() -{ - Token t = lexer->Get(); - int n = 0; - double d = 0.0; - - switch (t.type()) { - case Token::IntLiteral: { - if (dump_tokens) - Print("%s", t.symbol().data()); - - char nstr[256], *p = nstr; - for (int i = 0; i < (int) t.symbol().length(); i++) - if (t.symbol()[i] != '_') - *p++ = t.symbol()[i]; - *p++ = '\0'; - - // handle hex notation: - if (nstr[1] == 'x') - n = xtol(nstr+2); - - else - n = atol(nstr); - - return new TermNumber(n); - } - - case Token::FloatLiteral: { - if (dump_tokens) - Print("%s", t.symbol().data()); - - char nstr[256], *p = nstr; - for (int i = 0; i < (int) t.symbol().length(); i++) - if (t.symbol()[i] != '_') - *p++ = t.symbol()[i]; - *p++ = '\0'; - - d = atof(nstr); - return new TermNumber(d); - } - - case Token::StringLiteral: - if (dump_tokens) - Print("%s", t.symbol().data()); - - return new TermText(t.symbol()(1, t.symbol().length()-2)); - - case Token::AlphaIdent: - if (dump_tokens) - Print("%s", t.symbol().data()); - - return new TermText(t.symbol()); - - case Token::Keyword: - if (dump_tokens) - Print("%s", t.symbol().data()); - - switch (t.key()) { - case KEY_FALSE: return new TermBool(0); - case KEY_TRUE: return new TermBool(1); - - case KEY_MINUS: { - Token next = lexer->Get(); - if (next.type() == Token::IntLiteral) { - if (dump_tokens) - Print("%s", next.symbol().data()); - - char nstr[256], *p = nstr; - for (int i = 0; i < (int) next.symbol().length(); i++) - if (next.symbol()[i] != '_') - *p++ = next.symbol()[i]; - *p++ = '\0'; - - n = -1 * atol(nstr); - return new TermNumber(n); - } - else if (next.type() == Token::FloatLiteral) { - if (dump_tokens) - Print("%s", next.symbol().data()); - - char nstr[256], *p = nstr; - for (int i = 0; i < (int) next.symbol().length(); i++) - if (next.symbol()[i] != '_') - *p++ = next.symbol()[i]; - *p++ = '\0'; - - d = -1.0 * atof(nstr); - return new TermNumber(d); - } - else { - lexer->PutBack(); - return error("(Parse) illegal token '-': number expected", next); - } - } - break; - - default: - lexer->PutBack(); - return 0; - } - - case Token::LParen: return ParseArray(); - - case Token::LBrace: return ParseStruct(); - - case Token::CharLiteral: - return error("(Parse) illegal token ", t); - - default: - lexer->PutBack(); - return 0; - } -} - -TermArray* -Parser::ParseArray() -{ - TermList* elems = ParseTermList(0); - Token end = lexer->Get(); - - if (end.type() != Token::RParen) - return (TermArray*) error("(Parse) ')' missing in array-decl", end); - - return new TermArray(elems); -} - -TermStruct* -Parser::ParseStruct() -{ - TermList* elems = ParseTermList(1); - Token end = lexer->Get(); - - if (end.type() != Token::RBrace) - return (TermStruct*) error("(Parse) '}' missing in struct", end); - - return new TermStruct(elems); -} - -TermList* -Parser::ParseTermList(int for_struct) -{ - TermList* tlist = new TermList; - - Term* term = ParseTerm(); - while (term) { - if (for_struct && !term->isDef()) { - return (TermList*) error("(Parse) non-definition term in struct"); - } - else if (!for_struct && term->isDef()) { - return (TermList*) error("(Parse) illegal definition in array"); - } - - tlist->append(term); - Token t = lexer->Get(); - - /*** OLD WAY: COMMA SEPARATORS REQUIRED *** - if (t.type() != Token::Comma) { - lexer->PutBack(); - term = 0; - } - else - term = ParseTerm(); - /*******************************************/ - - // NEW WAY: COMMA SEPARATORS OPTIONAL: - if (t.type() != Token::Comma) { - lexer->PutBack(); - } - - term = ParseTerm(); - } - - return tlist; -} diff --git a/DefinitionEx/src/Term.cpp b/DefinitionEx/src/Term.cpp index acfdcb8..789066d 100644 --- a/DefinitionEx/src/Term.cpp +++ b/DefinitionEx/src/Term.cpp @@ -11,7 +11,7 @@ Implementation of the Term class */ -#include +#include #include diff --git a/DefinitionEx/src/Token.cpp b/DefinitionEx/src/Token.cpp index 3f516de..04f5708 100644 --- a/DefinitionEx/src/Token.cpp +++ b/DefinitionEx/src/Token.cpp @@ -11,7 +11,7 @@ Scanner class implementation */ -#include +#include #include diff --git a/DefinitionEx/test/parse_utils_like.cpp b/DefinitionEx/test/parse_utils_like.cpp index fc943f7..eb87a07 100644 --- a/DefinitionEx/test/parse_utils_like.cpp +++ b/DefinitionEx/test/parse_utils_like.cpp @@ -1,8 +1,6 @@ #include -#include -#include -#include +#include TEST(DefinitionEx, ParseUtilsLike) -- cgit v1.1