Tokenizer Okuması kolaylaştırıldı

core/Tokenizer.cpp

@@ -21,66 +21,46 @@ class Token {
 
 class StringToken : public Token {
     public:
-        StringToken() { this->type = "string"; }
+        StringToken(){
+            this->type = "string";
+        };
         std::string context;
         int size = 0;
-        void log()
-        {
-            std::cout << "Token String{" << this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " Context{"<< this->context << "} Size="<< this->context.size() <<"\n";
-        }
 };
 class NumberToken : public Token {
     public:
-        NumberToken() { this->type = "number"; }
+        NumberToken(){
+            this->type = "number";
+        }
         bool isFloat = false;
         bool hasEpsilon = false;
         int base = 10;
-        void log()
-        {
-            std::cout << "NumberToken "<< (this->isFloat ? "Float" : "Integer") <<"{" << this->token << "} HasExponent="<< (this->hasEpsilon ? "Yes" : "No") << " Base=" << this->base << " Start=" <<  this->start  << " End=" <<  this->end << "\n";
-        }
 };
-// class BoolToken : public Token {
-//     public:
-//         BoolToken() { this->type = "boolean"; }
-//         void log()
-//         {
-//             std::cout << "BoolToken Value{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
-//         }
-// };
 class OperatorToken : public Token {
     public:
-        OperatorToken() { this->type = "operator"; }
+        OperatorToken(){
-        void log()
+            this->type = "operator";
-        {
-            std::cout << "OperatorToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
         }
 };
 class DelimiterToken : public Token {
     public:
-        DelimiterToken() { this->type = "delimiter"; }
+        DelimiterToken(){
-        void log()
+            this->type = "delimiter";
-        {
-            std::cout << "DelimiterToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
         }
 };
 class KeywordToken : public Token {
     public:
-        KeywordToken() { this->type = "keyword"; }
+        KeywordToken(){
-        void log()
+            this->type = "keyword";
-        {
-            std::cout << "KeywordToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
         }
 };
 class IdentifierToken : public Token {
     public:
-        IdentifierToken() { this->type = "identifier"; }
+        IdentifierToken(){
+            this->type = "identifier";
+        }
         std::string context;
         int size = 0;
-        void log()
-        {
-            std::cout << "IdentifierToken Context{"<<this->token<<"} Start=" <<  this->start  << " End=" <<  this->end << " \n";
-        }
 };
 
 
@@ -159,8 +139,17 @@ const constexpr std::string_view keywords[] = {
 class Tokenizer {
 public:
     Lexer hmx;
-    std::vector<Token> scan(std::string input)
+    std::vector<Token> scan(std::string input);
-    {
+    Token scope();
+    IdentifierToken readIndetifier();
+    StringToken readString();
+    void skipOneLineComment();
+    void skipMultiLineComment();
+};
+
+
+std::vector<Token> Tokenizer::scan(std::string input)
+{
     std::vector<Token> tokens;
     this->hmx.setText(input);
     while(1)
@@ -173,9 +162,9 @@ public:
         }
     }
     return tokens;
-    }
+}
-    Token scope()
+Token Tokenizer::scope()
-    {
+{
     this->hmx.skipWhiteSpace();
 
     // Yorum satırları
@@ -251,9 +240,9 @@ public:
     }
 
     return this->readIndetifier();
-    }
+}
-    IdentifierToken readIndetifier()
+IdentifierToken Tokenizer::readIndetifier()
-    {
+{
     this->hmx.beginPosition();
     IdentifierToken idenditifierToken;
     idenditifierToken.start = this->hmx.getOffset();
@@ -312,9 +301,9 @@ public:
     idenditifierToken.size = idenditifierToken.context.size();
     this->hmx.acceptPosition();
     return idenditifierToken;
-    }
+}
-    StringToken readString()
+StringToken Tokenizer::readString()
-    {
+{
     this->hmx.beginPosition();
     StringToken stringToken;
     bool started = false;
@@ -358,9 +347,9 @@ public:
     stringToken.size = stringToken.context.size();
     this->hmx.acceptPosition();
     return stringToken;
-    }
+}
-    void skipOneLineComment()
+void Tokenizer::skipOneLineComment()
-    {
+{
     std::cout << "SkipLineComment\n";
     while(this->hmx.isEnd() == false)
     {
@@ -373,9 +362,9 @@ public:
             this->hmx.nextChar();
         }
     }
-    }
+}
-    void skipMultiLineComment()
+void Tokenizer::skipMultiLineComment()
-    {
+{
     std::cout << "SkipBlockComment\n";
     while(this->hmx.isEnd() == false)
     {
@@ -387,7 +376,6 @@ public:
             this->hmx.nextChar();
         }
     }
-    }
+}
-};
 
 #endif

core/Tools.cpp

@@ -0,0 +1,14 @@
+#include <string>
+#ifndef Tools
+#define Tools
+
+
+std::string padRight(std::string str, size_t totalLen) {
+    if (str.size() < totalLen) {
+        str.append(totalLen - str.size(), ' ');
+    }
+    return str;
+}
+
+
+#endif