Definition Parser now consumes from the new Reader

Next steps are to move ParseUtils functionality to DefinitinoEx, sort
things into the namespaces in this library, and then prepare it to
handle overloading like nlohmann_json does. On the other side, it's
time for starshatter::data.

8 files changed, 234 insertions, 120 deletions

diff --git a/DefinitionEx/CMakeLists.txt b/DefinitionEx/CMakeLists.txt
index fe4e819..f098670 100644
--- a/DefinitionEx/CMakeLists.txt
+++ b/DefinitionEx/CMakeLists.txt
@@ -1,15 +1,15 @@
 project(DefinitionEx)
+
 add_library(
-	DefinitionEx STATIC
-	Parser_ss.cpp
-	Term.cpp
-	Token.cpp
-	)
-target_include_directories(
-	DefinitionEx
-	PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
-	)
-target_link_libraries(
-	DefinitionEx
-	PUBLIC FoundationEx
+	${PROJECT_NAME} STATIC
+	src/Parser_ss.cpp
+	src/Term.cpp
+	src/Token.cpp
 	)
+target_include_directories(${PROJECT_NAME} PUBLIC include)
+target_link_libraries(${PROJECT_NAME} PUBLIC FoundationEx)
+
+add_executable(${PROJECT_NAME}_test test/parse_utils_like.cpp)
+target_link_libraries(${PROJECT_NAME}_test PRIVATE ${PROJECT_NAME} GTest::gtest_main)
+generate_emulator(${PROJECT_NAME}_test)
+gtest_discover_tests(${PROJECT_NAME}_test DISCOVERY_TIMEOUT 60)
diff --git a/DefinitionEx/Parser_ss.h b/DefinitionEx/include/Parser_ss.h
index 5019f61..50659b9 100644
--- a/DefinitionEx/Parser_ss.h
+++ b/DefinitionEx/include/Parser_ss.h
@@ -3,23 +3,23 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Declaration of the generic Parser class
+    OVERVIEW
+    ========
+    Declaration of the generic Parser class
 */
 
-#ifndef Parser_h
-#define Parser_h
+#pragma once
+
+#include <starshatter/foundation/reader.h>
+#include <Text.h>
 
-#include "Text.h"
 #include "Term.h"
 
 // +-------------------------------------------------------------------+
 
-class Reader;
 class Scanner;
 
 // +-------------------------------------------------------------------+
@@ -27,7 +27,8 @@ class Scanner;
 class Parser
 {
 public:
-    Parser(Reader* r = 0);
+    Parser();
+    Parser(starshatter::foundation::Reader r);
     ~Parser();
 
     Term*          ParseTerm();
@@ -38,9 +39,5 @@ public:
     TermStruct*    ParseStruct();
 
 private:
-    Reader*        reader;
-    Scanner*       lexer;
+    Scanner* lexer;
 };
-
-
-#endif  // Parser_h
diff --git a/DefinitionEx/Term.h b/DefinitionEx/include/Term.h
index f7e2b86..1bc30b6 100644
--- a/DefinitionEx/Term.h
+++ b/DefinitionEx/include/Term.h
@@ -3,19 +3,18 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Declaration of the Abstract Syntax Tree classes
+    OVERVIEW
+    ========
+    Declaration of the Abstract Syntax Tree classes
 */
 
-#ifndef Term_h
-#define Term_h
+#pragma once
 
-#include "Text.h"
-#include "List.h"
+#include <Text.h>
+#include <List.h>
 
 // +-------------------------------------------------------------------+
 
@@ -32,7 +31,7 @@ class TermStruct;
 class Term
 {
 public:
-    static const char* TYPENAME() { return "Term"; }
+     static const char* TYPENAME() { return "Term"; }
 
      Term()            { }
      virtual ~Term()   { }
@@ -51,7 +50,7 @@ public:
      virtual TermStruct*        isStruct()  { return 0; }
 };
 
-Term* error(char*, char* = 0);
+Term* error(const char*, const char* = 0);
 
 // +-------------------------------------------------------------------+
 
@@ -166,6 +165,3 @@ private:
     TermText*   mname;
     Term*       mval;
 };
-
-
-#endif  // Term_h
diff --git a/DefinitionEx/Token.h b/DefinitionEx/include/Token.h
index 59a3414..3dd8eb9 100644
--- a/DefinitionEx/Token.h
+++ b/DefinitionEx/include/Token.h
@@ -3,25 +3,24 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Scanner class definition
+    OVERVIEW
+    ========
+    Scanner class definition
 */
 
-#ifndef Token_h
-#define Token_h
+#pragma once
 
-#include "Text.h"
-#include "Dictionary.h"
+#include <Dictionary.h>
+#include <starshatter/foundation/reader.h>
+#include <Text.h>
 
 #pragma warning( disable : 4237)
 
 // +-------------------------------------------------------------------+
 
-class Reader;
 class Token;
 class Scanner;
 
@@ -104,7 +103,8 @@ protected:
 class Scanner
 {
 public:
-    Scanner(Reader* r = 0);
+	Scanner();
+    Scanner(starshatter::foundation::Reader r);
     Scanner(const Text&  s);
     Scanner(const Scanner& rhs);
     virtual ~Scanner();
@@ -127,8 +127,8 @@ protected:
     virtual bool   IsSymbolic(char c);
     virtual bool   IsAlpha(char c);
 
-    Reader*     reader;
-    char*       str;
+    starshatter::foundation::Reader reader;
+    char* str;
 
     const char* p;
     const char* eos;
@@ -141,6 +141,3 @@ protected:
     size_t      old_line;
     size_t      lineStart;
 };
-
-
-#endif  // Token_h
diff --git a/DefinitionEx/Parser_ss.cpp b/DefinitionEx/src/Parser_ss.cpp
index d02b7ee..1b80d48 100644
--- a/DefinitionEx/Parser_ss.cpp
+++ b/DefinitionEx/src/Parser_ss.cpp
@@ -3,29 +3,34 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Implementation of the generic Parser class
+    OVERVIEW
+    ========
+    Implementation of the generic Parser class
 */
 
-#include "Reader.h"
-#include "Token.h"
-#include "Parser_ss.h"
-#include "Term.h"
-#include "Utils.h"
+#include <Parser_ss.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <utility>
+
+#include <starshatter/foundation/reader.h>
+#include <Utils.h>
+
+#include <Term.h>
+#include <Token.h>
+
 enum KEYS { KEY_TRUE, KEY_FALSE, KEY_DEF, KEY_MINUS };
 
 static int dump_tokens = 0;
 
 // +-------------------------------------------------------------------+
 
-Term* error(char* msg, const Token& token)
+Term* error(const char* msg, const Token& token)
 {
     static char buf[1024];
     snprintf(buf, 1024, " near '%s' in line %d", (const char*) token.symbol(), token.line());
@@ -34,11 +39,19 @@ Term* error(char* msg, const Token& token)
 
 // +-------------------------------------------------------------------+
 
-Parser::Parser(Reader* r)
+Parser::Parser() :
+	lexer {nullptr}
 {
-    reader = r;
-    lexer  = new Scanner(reader);
+    Token::addKey("true",   KEY_TRUE);
+    Token::addKey("false",  KEY_FALSE);
+    Token::addKey(":",      KEY_DEF);
+    Token::addKey("-",      KEY_MINUS);
+}
 
+
+Parser::Parser(starshatter::foundation::Reader r) :
+	lexer {new Scanner(std::move(r))}
+{
     Token::addKey("true",   KEY_TRUE);
     Token::addKey("false",  KEY_FALSE);
     Token::addKey(":",      KEY_DEF);
@@ -47,8 +60,8 @@ Parser::Parser(Reader* r)
 
 Parser::~Parser()
 {
-    delete lexer;
-    delete reader;
+    if (lexer)
+        delete lexer;
     //Token::close();
 }
 
diff --git a/DefinitionEx/Term.cpp b/DefinitionEx/src/Term.cpp
index c27eb4e..acfdcb8 100644
--- a/DefinitionEx/Term.cpp
+++ b/DefinitionEx/src/Term.cpp
@@ -3,22 +3,22 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Implementation of the Term class
+    OVERVIEW
+    ========
+    Implementation of the Term class
 */
 
+#include <Term.h>
 
-#include "Term.h"
-#include "Utils.h"
+#include <Utils.h>
 
 // +-------------------------------------------------------------------+
 
 Term*
-error(char* s1, char* s2)
+error(const char* s1, const char* s2)
 {
     Print("ERROR: ");
     if (s1) Print(s1);
diff --git a/DefinitionEx/Token.cpp b/DefinitionEx/src/Token.cpp
index 2d060b9..3f516de 100644
--- a/DefinitionEx/Token.cpp
+++ b/DefinitionEx/src/Token.cpp
@@ -3,20 +3,24 @@
     Copyright (c) 2011-2012, Starshatter OpenSource Distribution Contributors
     Copyright (c) 1997-2006, Destroyer Studios LLC.
 
-     AUTHOR:       John DiCamillo
+    AUTHOR:       John DiCamillo
 
 
-     OVERVIEW
-     ========
-     Scanner class implementation
+    OVERVIEW
+    ========
+    Scanner class implementation
 */
 
-#include "Token.h"
-#include "Reader.h"
-#include "Text.h"
+#include <Token.h>
 
 #include <ctype.h>
 
+#include <cstddef>
+#include <utility>
+
+#include <starshatter/foundation/reader.h>
+#include <Text.h>
+
 // +-------------------------------------------------------------------+
 
 bool        Token::hidecom   = true;
@@ -285,50 +289,51 @@ Token::describe(const Text& tok)
 
 // +-------------------------------------------------------------------+
 
-Scanner::Scanner(Reader* r)
-    : reader(r), str(0), index(0), old_index(0),
-      length(0), line(0), old_line(0), lineStart(0)
-{ }
-
-Scanner::Scanner(const Scanner& rhs)
-    : index(rhs.index), old_index(rhs.old_index), length(rhs.length),
-      reader(rhs.reader),
-      line(rhs.line), old_line(0), lineStart(rhs.lineStart)
+Scanner::Scanner() :
+	reader {},
+	str {nullptr},
+	index {0},
+	old_index {0},
+	length {0},
+	line {0},
+	old_line {0},
+	lineStart {0}
 {
-    str = new char [strlen(rhs.str) + 1];
-    strcpy(str, rhs.str);
 }
 
-Scanner::Scanner(const Text& s)
-    : reader(0), index(0), old_index(0), length(s.length()), line(0),
-      old_line(0), lineStart(0)
-{
-    str = new char [s.length() + 1];
-    strcpy(str, s.data());
-}
 
-Scanner::~Scanner()
+Scanner::Scanner(starshatter::foundation::Reader r) :
+	reader {std::move(r)},
+	str {nullptr},
+	index {0},
+	old_index {0},
+	length {0},
+	line {0},
+	old_line {0},
+	lineStart {0}
 {
-    delete [] str;
 }
 
-// +-------------------------------------------------------------------+
 
-Scanner&
-Scanner::operator = (const Scanner& rhs)
+Scanner::Scanner(const Text& s) :
+	reader {},
+	str {nullptr},
+	index {0},
+	old_index {0},
+	length {static_cast<std::size_t>(s.length())},
+	line {0},
+	old_line {0},
+	lineStart {0}
 {
-    delete [] str;
-    str = new char [strlen(rhs.str) + 1];
-    strcpy(str, rhs.str);
+    str = new char [s.length() + 1];
+    strcpy(str, s.data());
+}
 
-    index     = rhs.index;
-    old_index = rhs.old_index;
-    length    = rhs.length;
-    line      = rhs.line;
-    old_line  = rhs.old_line;
-    lineStart = rhs.lineStart;
 
-    return *this;
+Scanner::~Scanner()
+{
+    if (str)
+        delete [] str;
 }
 
 // +-------------------------------------------------------------------+
@@ -362,8 +367,8 @@ Scanner::Get(Need need)
     p   = str + index;
 
     if (p >= eos) {
-        if (need == Demand && reader) {
-            Load(reader->more());
+        if (need == Demand && reader.valid()) {
+            Load(reader.more());
             if (length > 0)
                 return Get(need);
         }
@@ -379,8 +384,8 @@ Scanner::Get(Need need)
     }
 
     if (p >= eos) {
-        if (need == Demand && reader) {
-            Load(reader->more());
+        if (need == Demand && reader.valid()) {
+            Load(reader.more());
             if (length > 0)
                 return Get(need);
         }
diff --git a/DefinitionEx/test/parse_utils_like.cpp b/DefinitionEx/test/parse_utils_like.cpp
new file mode 100644
index 0000000..fc943f7
--- /dev/null
+++ b/DefinitionEx/test/parse_utils_like.cpp
@@ -0,0 +1,106 @@
+#include <gtest/gtest.h>
+
+#include <Parser_ss.h>
+#include <Term.h>
+#include <Token.h>
+
+
+TEST(DefinitionEx, ParseUtilsLike)
+{
+	std::string content {
+		"TEST\n\n"
+		"number: 3\n"
+		"bool: true\n"
+		"short: hello\n"
+		"longer: \"hello there\"\n"
+		"array: (0.98, -1.0, 0.35)\n"
+		"struct: {\n"
+		"	name: \"trailing comma\",\n"
+		"}\n"
+	};
+	Parser parser({content.data(), content.size()});
+	Term* term = parser.ParseTerm();
+	ASSERT_NE(nullptr, term);
+	{
+		TermText* text = term->isText();
+		ASSERT_NE(nullptr, text);
+		EXPECT_STREQ("TEST", text->value());
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("number", def->name()->value());
+		TermNumber* num = def->term()->isNumber();
+		ASSERT_NE(nullptr, num);
+		EXPECT_EQ(3, static_cast<int>(num->value()));
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("bool", def->name()->value());
+		TermBool* boolean = def->term()->isBool();
+		ASSERT_NE(nullptr, boolean);
+		EXPECT_TRUE(boolean->value());
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("short", def->name()->value());
+		TermText* text = def->term()->isText();
+		ASSERT_NE(nullptr, text);
+		EXPECT_STREQ("hello", text->value());
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("longer", def->name()->value());
+		TermText* text = def->term()->isText();
+		ASSERT_NE(nullptr, text);
+		EXPECT_STREQ("hello there", text->value());
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("array", def->name()->value());
+		TermArray* array = def->term()->isArray();
+		ASSERT_NE(nullptr, array);
+		ASSERT_EQ(3, array->elements()->size());
+		EXPECT_FLOAT_EQ(0.98, array->elements()->at(0)->isNumber()->value());  // what if not a number?
+		EXPECT_FLOAT_EQ(-1.0, array->elements()->at(1)->isNumber()->value());
+		EXPECT_FLOAT_EQ(0.35, array->elements()->at(2)->isNumber()->value());
+	}
+	{
+		delete term;
+		term = parser.ParseTerm();
+		ASSERT_NE(nullptr, term);
+		TermDef* def = term->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("struct", def->name()->value());
+		ASSERT_NE(nullptr, def->term());
+		TermStruct* strukt = def->term()->isStruct();
+		ASSERT_NE(nullptr, strukt);
+		ASSERT_EQ(1, strukt->elements()->size());
+		def = strukt->elements()->at(0)->isDef();
+		ASSERT_NE(nullptr, def);
+		EXPECT_STREQ("name", def->name()->value());
+		TermText* text = def->term()->isText();
+		ASSERT_NE(nullptr, text);
+		EXPECT_STREQ("trailing comma", text->value());
+	}
+	Token::close();
+}