lexer

.gitignore

@@ -0,0 +1,120 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+### macOS template
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+.idea
+

go.mod

@@ -0,0 +1,3 @@
+module github/runnignwater/monkey
+
+go 1.16

lexer/lexer.go

@@ -0,0 +1,146 @@
+package lexer
+
+import "github/runnignwater/monkey/token"
+
+/**
+ * @Author: simon
+ * @Author: ynwdlxm@163.com
+ * @Date: 2022/10/2 上午11:07
+ * @Desc: ASCII 码
+ */
+
+type Lexer struct {
+    input        string
+    position     int  // current position in input (points to current char)
+    readPosition int  // current reading position in input (after current char)
+    ch           byte // current char under examination
+}
+
+func New(input string) *Lexer {
+    l := &Lexer{input: input}
+    l.readChar()
+    return l
+}
+
+func (l *Lexer) readChar() {
+    if l.readPosition >= len(l.input) {
+        l.ch = 0
+    } else {
+        l.ch = l.input[l.readPosition]
+    }
+    l.position = l.readPosition
+    l.readPosition += 1
+}
+
+/**
+ * Only peek ahead in the input
+ * and not move around in it
+ */
+func (l *Lexer) peekChar() byte {
+    if l.readPosition >= len(l.input) {
+        return 0
+    } else {
+        return l.input[l.readPosition]
+    }
+}
+
+func (l *Lexer) readIdentifier() string {
+    position := l.position
+    for isLetter(l.ch) {
+        l.readChar()
+    }
+    return l.input[position:l.position]
+}
+
+func (l *Lexer) readNumber() string {
+    position := l.position
+    for isDigit(l.ch) {
+        l.readChar()
+    }
+    return l.input[position:l.position]
+}
+
+func (l *Lexer) NextToken() token.Token {
+    var tok token.Token
+
+    l.skipWhitespace()
+
+    switch l.ch {
+    case '=':
+        if l.peekChar() == '=' {
+            ch := l.ch
+            l.readChar()
+            tok = token.Token{Type: token.EQ, Literal: string(ch) + string(l.ch)}
+        } else {
+            tok = newToken(token.ASSIGN, l.ch)
+        }
+    case ';':
+        tok = newToken(token.SEMICOLON, l.ch)
+    case '(':
+        tok = newToken(token.LPAREN, l.ch)
+    case ')':
+        tok = newToken(token.RPAREN, l.ch)
+    case ',':
+        tok = newToken(token.COMMA, l.ch)
+    case '+':
+        tok = newToken(token.PLUS, l.ch)
+    case '-':
+        tok = newToken(token.MINUS, l.ch)
+    case '!':
+        if l.peekChar() == '=' {
+            ch := l.ch
+            l.readChar()
+            tok = token.Token{Type: token.NOT_EQ, Literal: string(ch) + string(l.ch)}
+        } else {
+            tok = newToken(token.BANG, l.ch)
+        }
+    case '*':
+        tok = newToken(token.ASTERISK, l.ch)
+    case '/':
+        tok = newToken(token.SLASH, l.ch)
+    case '<':
+        tok = newToken(token.LT, l.ch)
+    case '>':
+        tok = newToken(token.GT, l.ch)
+    case '{':
+        tok = newToken(token.LBRACE, l.ch)
+    case '}':
+        tok = newToken(token.RBRACE, l.ch)
+    case 0:
+        tok.Type = token.EOF
+        tok.Literal = ""
+    default:
+        if isLetter(l.ch) {
+            tok.Literal = l.readIdentifier()
+            tok.Type = token.LookupIdent(tok.Literal)
+        } else if isDigit(l.ch) {
+            tok.Type = token.INT
+            tok.Literal = l.readNumber()
+        } else {
+            tok = newToken(token.ILLEGAL, l.ch)
+        }
+
+        return tok
+    }
+
+    l.readChar()
+    return tok
+}
+
+func (l *Lexer) skipWhitespace() {
+    for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+        l.readChar()
+    }
+}
+
+func isDigit(ch byte) bool {
+    return '0' <= ch && ch <= '9'
+}
+
+func isLetter(ch byte) bool {
+    return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+func newToken(tokenType token.TypeToken, ch byte) token.Token {
+    return token.Token{Type: tokenType, Literal: string(ch)}
+}

lexer/lexer_test.go

@@ -0,0 +1,123 @@
+package lexer
+
+import (
+    "github/runnignwater/monkey/token"
+    "testing"
+)
+
+/**
+ * @Author: simon
+ * @Author: ynwdlxm@163.com
+ * @Date: 2022/10/2 上午10:58
+ * @Desc:
+ */
+func TestNextToken(t *testing.T) {
+    input := `let five = 5;
+        let ten = 10;
+        let add = fn(x, y) { x + y;
+        };
+        let result = add(five, ten); 
+        !-/*5
+        < 10 > 5;
+        
+        if (5 < 10) {
+            return true;
+        } else {
+            return false;
+        }
+        
+        10 == 10;
+        10 != 9;
+        `
+    tests := []struct {
+        expectedType    token.TypeToken
+        expectedLiteral string
+    }{
+        {token.LET, "let"},
+        {token.IDENT, "five"},
+        {token.ASSIGN, "="},
+        {token.INT, "5"},
+        {token.SEMICOLON, ";"},
+        {token.LET, "let"},
+        {token.IDENT, "ten"},
+        {token.ASSIGN, "="},
+        {token.INT, "10"},
+        {token.SEMICOLON, ";"},
+        {token.LET, "let"},
+        {token.IDENT, "add"},
+        {token.ASSIGN, "="},
+        {token.FUNCTION, "fn"},
+        {token.LPAREN, "("},
+        {token.IDENT, "x"},
+        {token.COMMA, ","},
+        {token.IDENT, "y"},
+        {token.RPAREN, ")"},
+        {token.LBRACE, "{"},
+        {token.IDENT, "x"},
+        {token.PLUS, "+"},
+        {token.IDENT, "y"},
+        {token.SEMICOLON, ";"},
+        {token.RBRACE, "}"},
+        {token.SEMICOLON, ";"},
+        {token.LET, "let"},
+        {token.IDENT, "result"},
+        {token.ASSIGN, "="},
+        {token.IDENT, "add"},
+        {token.LPAREN, "("},
+        {token.IDENT, "five"},
+        {token.COMMA, ","},
+        {token.IDENT, "ten"},
+        {token.RPAREN, ")"},
+        {token.SEMICOLON, ";"},
+        {token.BANG, "!"},
+        {token.MINUS, "-"},
+        {token.SLASH, "/"},
+        {token.ASTERISK, "*"},
+        {token.INT, "5"},
+        {token.LT, "<"},
+        {token.INT, "10"},
+        {token.GT, ">"},
+        {token.INT, "5"},
+        {token.SEMICOLON, ";"},
+
+        {token.IF, "if"},
+        {token.LPAREN, "("},
+        {token.INT, "5"},
+        {token.LT, "<"},
+        {token.INT, "10"},
+        {token.RPAREN, ")"},
+        {token.LBRACE, "{"},
+        {token.RETURN, "return"},
+        {token.TRUE, "true"},
+        {token.SEMICOLON, ";"},
+        {token.RBRACE, "}"},
+        {token.ELSE, "else"},
+        {token.LBRACE, "{"},
+        {token.RETURN, "return"},
+        {token.FALSE, "false"},
+        {token.SEMICOLON, ";"},
+        {token.RBRACE, "}"},
+        {token.INT, "10"},
+        {token.EQ, "=="},
+        {token.INT, "10"},
+        {token.SEMICOLON, ";"},
+        {token.INT, "10"},
+        {token.NOT_EQ, "!="},
+        {token.INT, "9"},
+        {token.SEMICOLON, ";"},
+        {token.EOF, ""},
+    }
+
+    l := New(input)
+
+    for i, tt := range tests {
+        tok := l.NextToken()
+
+        if tok.Type != tt.expectedType {
+            t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type)
+        }
+        if tok.Literal != tt.expectedLiteral {
+            t.Fatalf("tests[%d] - literal wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal)
+        }
+    }
+}

main.go

@@ -0,0 +1,8 @@
+package main
+
+/**
+ * @Author: simon
+ * @Author: ynwdlxm@163.com
+ * @Date: 2022/10/2 上午10:55
+ * @Desc:
+ */

token/token.go

@@ -0,0 +1,71 @@
+package token
+
+/**
+ * @Author: simon
+ * @Author: ynwdlxm@163.com
+ * @Date: 2022/10/2 上午10:44
+ * @Desc:
+ */
+
+const (
+    ILLEGAL = "ILLEGAL"
+    EOF     = "EOF"
+
+    // Identifiers + literals
+    IDENT = "IDENT" // add, foobar, x, y ...
+    INT   = "INT"   // 123456
+
+    // Operators
+    ASSIGN   = "="
+    PLUS     = "+"
+    MINUS    = "-"
+    BANG     = "!"
+    ASTERISK = "*"
+    SLASH    = "/"
+    LT       = "<"
+    GT       = ">"
+    EQ       = "=="
+    NOT_EQ   = "!="
+
+    // Delimiters
+    COMMA     = ","
+    SEMICOLON = ";"
+
+    LPAREN = "("
+    RPAREN = ")"
+    LBRACE = "{"
+    RBRACE = "}"
+
+    // Keywords
+    FUNCTION = "FUNCTION"
+    LET      = "LET"
+    TRUE     = "TRUE"
+    FALSE    = "FALSE"
+    IF       = "IF"
+    ELSE     = "ELSE"
+    RETURN   = "RETURN"
+)
+
+var keyword = map[string]TypeToken{
+    "fn":     FUNCTION,
+    "let":    LET,
+    "true":   TRUE,
+    "false":  FALSE,
+    "if":     IF,
+    "else":   ELSE,
+    "return": RETURN,
+}
+
+func LookupIdent(ident string) TypeToken {
+    if tok, ok := keyword[ident]; ok {
+        return tok
+    }
+    return IDENT
+}
+
+type TypeToken string
+
+type Token struct {
+    Type    TypeToken
+    Literal string
+}