Note: This page does not support English, using the default language version
Monkey语言: 分为表达式和语句, 下面给出一些语法.
// Integers & arithmetic expressions...let version = 1 + (50 / 2) - (8 * 3);
// ... and stringslet name = "The Monkey programming language";
// ... booleanslet isMonkeyFastNow = true;
// ... arrays & hash mapslet people = [{"name": "Anna", "age": 24}, {"name": "Bob", "age": 99}];当然现在的进度还没有实现这么复杂的语句.
词法分析就是输出一系列的token先给出token 的定义
pub const Token = struct { Type: TokenType, Literal: []const u8, // 原始切片,用于报错信息的打印};对于Monkey有如下的一些token
pub const TokenType = union(enum(u8)) { illegal, eof, ident, int: i64, // 少一次转换
// 运算符 assign, plus, minus, bang, asterisk, slash, lt, gt, eq, not_eq,
// 分隔符 comma, semicolon, lparen, rparen, lbrace, rbrace,
// 关键字 function, let, true, false, @"if", @"else", @"return",};这里重点讲一下为什么int我用i64, 因为想避免str -> int 的一个开销, 为什么用枚举就比较好理解
使用状态机的思想, 测试用例来自原书
先思考测试用例来决定我们要编写怎么样的代码(直接抄的)
test "nextToken_From go" { const gpa = std.testing.allocator; const input = \\let five = 5; \\let ten = 10; \\let add = fn(x, y) { \\x + y; \\}; \\let result = add(five, ten); \\!-/*5; \\5 < 10 > 5; \\if (5 < 10) { \\return true; \\} else { \\return false; \\} \\10 == 10; \\10 != 9; ; var l: Lexer = .init(input); const tokenList = try l.nextToken(gpa); defer gpa.free(tokenList); const Expected = struct { expected_type: std.meta.Tag(token.TokenType), expected_literal: []const u8, };
const tests = [_]Expected{ .{ .expected_type = .let, .expected_literal = "let" }, .{ .expected_type = .ident, .expected_literal = "five" }, .{ .expected_type = .assign, .expected_literal = "=" }, .{ .expected_type = .int, .expected_literal = "5" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .let, .expected_literal = "let" }, .{ .expected_type = .ident, .expected_literal = "ten" }, .{ .expected_type = .assign, .expected_literal = "=" }, .{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .let, .expected_literal = "let" }, .{ .expected_type = .ident, .expected_literal = "add" }, .{ .expected_type = .assign, .expected_literal = "=" }, .{ .expected_type = .function, .expected_literal = "fn" }, .{ .expected_type = .lparen, .expected_literal = "(" }, .{ .expected_type = .ident, .expected_literal = "x" }, .{ .expected_type = .comma, .expected_literal = "," }, .{ .expected_type = .ident, .expected_literal = "y" }, .{ .expected_type = .rparen, .expected_literal = ")" }, .{ .expected_type = .lbrace, .expected_literal = "{" }, .{ .expected_type = .ident, .expected_literal = "x" }, .{ .expected_type = .plus, .expected_literal = "+" }, .{ .expected_type = .ident, .expected_literal = "y" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .rbrace, .expected_literal = "}" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .let, .expected_literal = "let" }, .{ .expected_type = .ident, .expected_literal = "result" }, .{ .expected_type = .assign, .expected_literal = "=" }, .{ .expected_type = .ident, .expected_literal = "add" }, .{ .expected_type = .lparen, .expected_literal = "(" }, .{ .expected_type = .ident, .expected_literal = "five" }, .{ .expected_type = .comma, .expected_literal = "," }, .{ .expected_type = .ident, .expected_literal = "ten" }, .{ .expected_type = .rparen, .expected_literal = ")" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .bang, .expected_literal = "!" }, .{ .expected_type = .minus, .expected_literal = "-" }, .{ .expected_type = .slash, .expected_literal = "/" }, .{ .expected_type = .asterisk, .expected_literal = "*" }, .{ .expected_type = .int, .expected_literal = "5" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .int, .expected_literal = "5" }, .{ .expected_type = .lt, .expected_literal = "<" }, .{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .gt, .expected_literal = ">" }, .{ .expected_type = .int, .expected_literal = "5" }, .{ .expected_type = .semicolon, .expected_literal = ";" },
.{ .expected_type = .@"if", .expected_literal = "if" }, .{ .expected_type = .lparen, .expected_literal = "(" }, .{ .expected_type = .int, .expected_literal = "5" }, .{ .expected_type = .lt, .expected_literal = "<" }, .{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .rparen, .expected_literal = ")" }, .{ .expected_type = .lbrace, .expected_literal = "{" }, .{ .expected_type = .@"return", .expected_literal = "return" }, .{ .expected_type = .true, .expected_literal = "true" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .rbrace, .expected_literal = "}" }, .{ .expected_type = .@"else", .expected_literal = "else" }, .{ .expected_type = .lbrace, .expected_literal = "{" }, .{ .expected_type = .@"return", .expected_literal = "return" }, .{ .expected_type = .false, .expected_literal = "false" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .rbrace, .expected_literal = "}" },
.{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .eq, .expected_literal = "==" }, .{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .semicolon, .expected_literal = ";" }, .{ .expected_type = .int, .expected_literal = "10" }, .{ .expected_type = .not_eq, .expected_literal = "!=" }, .{ .expected_type = .int, .expected_literal = "9" }, .{ .expected_type = .semicolon, .expected_literal = ";" },
.{ .expected_type = .eof, .expected_literal = "" }, }; // for (tests, 0..) |expected, i| { const actual = tokenList[i];
try std.testing.expectEqual(expected.expected_type, std.meta.activeTag(actual.Type));
try std.testing.expectEqualStrings(expected.expected_literal, actual.Literal); }
try std.testing.expectEqual(tests.len, tokenList.len);}接下来进行分词
const token = @import("./token.zig");const std = @import("std");
pub const Lexer = struct { const State = enum { start, int, indentifer }; atPosition: u8, rdPosition: u8, ch: ?u8, source: []const u8, pub fn init(source: []const u8) Lexer { var l: Lexer = .{ .source = source, .atPosition = 0, .rdPosition = 0, .ch = 0 }; l.readChar(); return l; } pub fn nextToken(l: *Lexer, gpa: std.mem.Allocator) ![]token.Token { var list: std.ArrayList(token.Token) = .empty; state: switch (State.start) { .start => { if (l.ch == null) { try list.append(gpa, .{ .Type = .eof, .Literal = "" }); break :state; } while (std.ascii.isWhitespace(l.ch.?)) { l.readChar(); } const start = l.atPosition; if (std.ascii.isDigit(l.ch.?)) { continue :state .int; } if (std.ascii.isAlphabetic(l.ch.?) or (l.ch.? == '_')) { continue :state .indentifer; } if (l.ch) |ch| { var token_type: token.TokenType = switch (ch) { '+' => .plus, '-' => .minus, '*' => .asterisk, '/' => .slash, '<' => .lt, '>' => .gt, ',' => .comma, ';' => .semicolon, '(' => .lparen, ')' => .rparen, '{' => .lbrace, '}' => .rbrace, else => .illegal, }; if (ch == '=') { if (l.peekChar() == '=') { l.readChar(); token_type = .eq; } else { token_type = .assign; } } else if (ch == '!') { if (l.peekChar() == '=') { l.readChar(); token_type = .not_eq; } else { token_type = .bang; } } try list.append(gpa, .{ .Type = token_type, .Literal = l.source[start..l.rdPosition] }); l.readChar(); continue :state .start; } }, .int => { const start = l.atPosition; // while (l.ch) |chr| { // if (!std.ascii.isDigit(chr)) break; // l.readChar(); // } while (l.ch != null and std.ascii.isDigit(l.ch.?)) { l.readChar(); } const str = l.source[start..l.atPosition]; const num = try std.fmt.parseInt(i64, str, 10); try list.append(gpa, .{ .Type = .{ .int = num }, .Literal = str }); continue :state .start; }, .indentifer => { const start = l.atPosition; while (l.ch != null and (std.ascii.isDigit(l.ch.?) or std.ascii.isAlphabetic(l.ch.?) or l.ch.? == '_')) { l.readChar(); }
const str = l.source[start..l.atPosition]; const tokentype = l.lookIndent(str); try list.append(gpa, .{ .Type = tokentype, .Literal = str }); continue :state .start; }, } return list.toOwnedSlice(gpa); }
pub fn readChar(l: *Lexer) void { l.atPosition = l.rdPosition; if (l.rdPosition >= l.source.len) { l.ch = null; return; } else { l.ch = l.source[l.rdPosition]; } l.rdPosition += 1; }
pub fn peekChar(l: *Lexer) u8 { if (l.rdPosition >= l.source.len) { return 0; } return l.source[l.rdPosition]; }
pub fn lookIndent(l: *Lexer, ident: []const u8) token.TokenType { _ = l; if (token.keywords.get(ident)) |token_type| { return token_type; } return .ident; }};这里没什么好讲的(bushi