Tokenizer Okuması kolaylaştırıldı

core/Tokenizer.cpp

@@ -21,66 +21,46 @@ class Token {
 
 class StringToken : public Token {
     public:
-        StringToken() { this->type = "string"; }
+        StringToken(){
+            this->type = "string";
+        };
         std::string context;
         int size = 0;
-        void log()
-        {
-            std::cout << "Token String{" << this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " Context{"<< this->context << "} Size="<< this->context.size() <<"\n";
-        }
 };
 class NumberToken : public Token {
     public:
-        NumberToken() { this->type = "number"; }
+        NumberToken(){
+            this->type = "number";
+        }
         bool isFloat = false;
         bool hasEpsilon = false;
         int base = 10;
-        void log()
-        {
-            std::cout << "NumberToken "<< (this->isFloat ? "Float" : "Integer") <<"{" << this->token << "} HasExponent="<< (this->hasEpsilon ? "Yes" : "No") << " Base=" << this->base << " Start=" <<  this->start  << " End=" <<  this->end << "\n";
-        }
 };
-// class BoolToken : public Token {
-//     public:
-//         BoolToken() { this->type = "boolean"; }
-//         void log()
-//         {
-//             std::cout << "BoolToken Value{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
-//         }
-// };
 class OperatorToken : public Token {
     public:
-        OperatorToken() { this->type = "operator"; }
-        void log()
-        {
-            std::cout << "OperatorToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
+        OperatorToken(){
+            this->type = "operator";
         }
 };
 class DelimiterToken : public Token {
     public:
-        DelimiterToken() { this->type = "delimiter"; }
-        void log()
-        {
-            std::cout << "DelimiterToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
+        DelimiterToken(){
+            this->type = "delimiter";
         }
 };
 class KeywordToken : public Token {
     public:
-        KeywordToken() { this->type = "keyword"; }
-        void log()
-        {
-            std::cout << "KeywordToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
+        KeywordToken(){
+            this->type = "keyword";
         }
 };
 class IdentifierToken : public Token {
     public:
-        IdentifierToken() { this->type = "identifier"; }
+        IdentifierToken(){
+            this->type = "identifier";
+        }
         std::string context;
         int size = 0;
-        void log()
-        {
-            std::cout << "IdentifierToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
-        }
 };
 
 
@@ -159,7 +139,16 @@ const constexpr std::string_view keywords[] = {
 class Tokenizer {
 public:
     Lexer hmx;
-    std::vector<Token> scan(std::string input)
+    std::vector<Token> scan(std::string input);
+    Token scope();
+    IdentifierToken readIndetifier();
+    StringToken readString();
+    void skipOneLineComment();
+    void skipMultiLineComment();
+};
+
+
+std::vector<Token> Tokenizer::scan(std::string input)
 {
     std::vector<Token> tokens;
     this->hmx.setText(input);
@@ -174,7 +163,7 @@ public:
     }
     return tokens;
 }
-    Token scope()
+Token Tokenizer::scope()
 {
     this->hmx.skipWhiteSpace();
 
@@ -252,7 +241,7 @@ public:
 
     return this->readIndetifier();
 }
-    IdentifierToken readIndetifier()
+IdentifierToken Tokenizer::readIndetifier()
 {
     this->hmx.beginPosition();
     IdentifierToken idenditifierToken;
@@ -313,7 +302,7 @@ public:
     this->hmx.acceptPosition();
     return idenditifierToken;
 }
-    StringToken readString()
+StringToken Tokenizer::readString()
 {
     this->hmx.beginPosition();
     StringToken stringToken;
@@ -359,7 +348,7 @@ public:
     this->hmx.acceptPosition();
     return stringToken;
 }
-    void skipOneLineComment()
+void Tokenizer::skipOneLineComment()
 {
     std::cout << "SkipLineComment\n";
     while(this->hmx.isEnd() == false)
@@ -374,7 +363,7 @@ public:
         }
     }
 }
-    void skipMultiLineComment()
+void Tokenizer::skipMultiLineComment()
 {
     std::cout << "SkipBlockComment\n";
     while(this->hmx.isEnd() == false)
@@ -388,6 +377,5 @@ public:
         }
     }
 }
-};
 
 #endif

core/Tools.cpp

@@ -0,0 +1,14 @@
+#include <string>
+#ifndef Tools
+#define Tools
+
+
+std::string padRight(std::string str, size_t totalLen) {
+    if (str.size() < totalLen) {
+        str.append(totalLen - str.size(), ' ');
+    }
+    return str;
+}
+
+
+#endif