summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lexer/build.gradle.kts19
-rw-r--r--lexer/src/main/java/lv/enes/orang/lexer/Lexer.java197
-rw-r--r--lexer/src/main/java/lv/enes/orang/lexer/Token.java62
-rw-r--r--lexer/src/main/java/module-info.java5
-rw-r--r--orang/build.gradle.kts1
-rw-r--r--orang/src/main/java/lv/enes/orang/Codepoint.java28
-rw-r--r--orang/src/main/java/lv/enes/orang/Lexer.java174
-rw-r--r--orang/src/main/java/lv/enes/orang/Main.java1
-rw-r--r--orang/src/main/java/lv/enes/orang/Token.java15
-rw-r--r--orang/src/main/java/lv/enes/orang/TokenType.java91
-rw-r--r--orang/src/main/java/module-info.java1
-rw-r--r--parser/build.gradle.kts22
-rw-r--r--parser/src/main/java/lv/enes/orang/parser/Parser.java (renamed from orang/src/main/java/lv/enes/orang/Parser.java)129
-rw-r--r--parser/src/main/java/lv/enes/orang/parser/ParserException.java (renamed from orang/src/main/java/lv/enes/orang/ParserException.java)2
-rw-r--r--parser/src/main/java/module-info.java8
-rw-r--r--settings.gradle.kts2
-rw-r--r--utils/src/main/java/lv/enes/orang/utils/Codepoint.java8
-rw-r--r--utils/src/main/java/lv/enes/orang/utils/PeekableStream.java (renamed from orang/src/main/java/lv/enes/orang/PeekableStream.java)2
18 files changed, 413 insertions, 354 deletions
diff --git a/lexer/build.gradle.kts b/lexer/build.gradle.kts
new file mode 100644
index 0000000..7fe8777
--- /dev/null
+++ b/lexer/build.gradle.kts
@@ -0,0 +1,19 @@
1plugins {
2 java
3}
4
5dependencies {
6 implementation(project(":utils"))
7}
8
9java {
10 sourceCompatibility = JavaVersion.VERSION_22
11 targetCompatibility = JavaVersion.VERSION_22
12 toolchain {
13 languageVersion = JavaLanguageVersion.of(22)
14 }
15}
16
17tasks.withType<JavaCompile> {
18 options.compilerArgs.add("--enable-preview")
19} \ No newline at end of file
diff --git a/lexer/src/main/java/lv/enes/orang/lexer/Lexer.java b/lexer/src/main/java/lv/enes/orang/lexer/Lexer.java
new file mode 100644
index 0000000..8fec98e
--- /dev/null
+++ b/lexer/src/main/java/lv/enes/orang/lexer/Lexer.java
@@ -0,0 +1,197 @@
1package lv.enes.orang.lexer;
2
3import lv.enes.orang.utils.Codepoint;
4import lv.enes.orang.utils.PeekableStream;
5
6import java.io.*;
7import java.util.Iterator;
8import java.util.function.BiFunction;
9import java.util.function.Predicate;
10import java.util.stream.IntStream;
11import java.util.stream.Stream;
12
13public class Lexer implements Iterator<Token> {
14 public static boolean isIdentInitial(Codepoint cp) {
15 return Character.isLetter(cp.cp()) || cp.cp() == '_';
16 }
17
18 public static boolean isIdentFinal(Codepoint cp) {
19 return isIdentInitial(cp) || Character.isDigit(cp.cp());
20 }
21
22 public static boolean isNewline(Codepoint cp) {
23 return cp.cp() == '\n';
24 }
25
26 public static boolean isNumeral(Codepoint cp) {
27 return Character.isDigit(cp.cp());
28 }
29
30 public static boolean isWhitespace(Codepoint cp) {
31 return Character.isWhitespace(cp.cp());
32 }
33
34 private final PeekableStream<Codepoint> input;
35
36 public Lexer(InputStream input) {
37 this(new InputStreamReader(input));
38 }
39
40 public Lexer(Reader input) {
41 var cpStream = new BufferedReader(input)
42 .lines()
43 .flatMapToInt(str -> IntStream.concat(str.codePoints(), IntStream.of('\n')))
44 .mapToObj(Codepoint::new);
45 var theEof = Stream.of(new Codepoint(-1));
46 this.input = new PeekableStream<>(Stream.concat(cpStream, theEof).iterator());
47 }
48
49 public Lexer(String input) {
50 this(new StringReader(input));
51 }
52
53 private boolean hasNext = true;
54
55 @Override
56 public Token next() {
57 var tok = nextToken();
58 if (tok.type() == Token.Type.EOF) {
59 hasNext = false;
60 }
61 return tok;
62 }
63
64 @Override
65 public boolean hasNext() {
66 return hasNext;
67 }
68
69 private Token nextToken() {
70 skipWhitespace();
71 return switch (input.peek().cp()) {
72 case -1 -> new Token(Token.Type.EOF, "");
73
74 case '*' -> new Token(Token.Type.ASTERISK, input.next());
75 case '!' -> new Token(Token.Type.BANG, input.next());
76 case '[' -> new Token(Token.Type.BRACKET_LEFT, input.next());
77 case ']' -> new Token(Token.Type.BRACKET_RIGHT, input.next());
78 case ',' -> new Token(Token.Type.COMMA, input.next());
79 case '=' -> new Token(Token.Type.EQUAL, input.next());
80 case '>' -> {
81 var first = input.next();
82 if (input.peek().cp() == '=') {
83 yield new Token(Token.Type.GREATER_EQUAL, first, input.next());
84 } else {
85 yield new Token(Token.Type.GREATER, first);
86 }
87 }
88 case '<' -> {
89 var first = input.next();
90 if (input.peek().cp() == '=') {
91 yield new Token(Token.Type.LESS_EQUAL, first, input.next());
92 } else {
93 yield new Token(Token.Type.LESS, first);
94 }
95 }
96 case '-' -> {
97 var first = input.next();
98 if (input.peek().cp() == '>') {
99 yield new Token(Token.Type.MINUS_GREATER, first, input.next());
100 } else {
101 yield new Token(Token.Type.MINUS, first);
102 }
103 }
104 case '(' -> new Token(Token.Type.PAREN_LEFT, input.next());
105 case ')' -> new Token(Token.Type.PAREN_RIGHT, input.next());
106 case '+' -> new Token(Token.Type.PLUS, input.next());
107 case '?' -> {
108 var first = input.next();
109 if (input.peek().cp() == '=') {
110 yield new Token(Token.Type.QUESTION_EQUAL, first, input.next());
111 } else {
112 yield new Token(Token.Type.ILLEGAL, first, input.next());
113 }
114 }
115 case ';' -> new Token(Token.Type.SEMICOLON, input.next());
116 case '/' -> {
117 var first = input.next();
118 if (input.peek().cp() == '=') {
119 yield new Token(Token.Type.SLASH_EQUAL, first, input.next());
120 } else {
121 yield new Token(Token.Type.SLASH, first);
122 }
123 }
124
125 case '"' -> new Token(Token.Type.STRING, readString());
126
127 default -> {
128 if (isIdentInitial(input.peek())) {
129 var ident = readIdentifier();
130 var type = switch (ident) {
131 case "and" -> Token.Type.AND;
132 case "def" -> Token.Type.DEF;
133 case "do" -> Token.Type.DO;
134 case "else" -> Token.Type.ELSE;
135 case "end" -> Token.Type.END;
136 case "false" -> Token.Type.FALSE;
137 case "fn" -> Token.Type.FN;
138 case "if" -> Token.Type.IF;
139 case "in" -> Token.Type.IN;
140 case "let" -> Token.Type.LET;
141 case "then" -> Token.Type.THEN;
142 case "true" -> Token.Type.TRUE;
143 default -> Token.Type.IDENTIFIER;
144 };
145 yield new Token(type, ident);
146 } else if (isNumeral(input.peek())) {
147 yield new Token(Token.Type.INTEGER, readInteger());
148 } else {
149 yield new Token(Token.Type.ILLEGAL, input.next());
150 }
151 }
152 };
153 }
154
155 private <T> T foldWhile(Predicate<Codepoint> pred, T initial, BiFunction<T, Codepoint, T> combine) {
156 var res = initial;
157 var ch = input.peek();
158 while (pred.test(ch)) {
159 res = combine.apply(res, input.next());
160 ch = input.peek();
161 }
162 return res;
163 }
164
165 private String readWhile(Predicate<Codepoint> pred) {
166 return foldWhile(pred, new StringBuilder(), StringBuilder::append).toString();
167 }
168
169 private void skipWhile(Predicate<Codepoint> pred) {
170 foldWhile(pred, Object.class, (x, _) -> x);
171 }
172
173 private String readIdentifier() {
174 return readWhile(Lexer::isIdentFinal);
175 }
176
177 private String readInteger() {
178 return readWhile(Lexer::isNumeral);
179 }
180
181 private String readString() {
182 input.next();
183 var literal = readWhile(cp -> cp.cp() != '"');
184 input.next();
185 return literal;
186 }
187
188 private void skipWhitespace() {
189 while (true) {
190 skipWhile(Lexer::isWhitespace);
191 if (input.peek().cp() != '#') {
192 return;
193 }
194 skipWhile(cp -> !isNewline(cp));
195 }
196 }
197}
diff --git a/lexer/src/main/java/lv/enes/orang/lexer/Token.java b/lexer/src/main/java/lv/enes/orang/lexer/Token.java
new file mode 100644
index 0000000..59626c7
--- /dev/null
+++ b/lexer/src/main/java/lv/enes/orang/lexer/Token.java
@@ -0,0 +1,62 @@
1package lv.enes.orang.lexer;
2
3import lv.enes.orang.utils.Codepoint;
4
5public record Token(Type type, String literal) {
6 public Token(Type type, Codepoint... cps) {
7 this(type, codepointsToString(cps));
8 }
9
10 private static String codepointsToString(Codepoint... cps) {
11 var sb = new StringBuilder(cps.length);
12 for (var cp : cps) {
13 sb.append(cp);
14 }
15 return sb.toString();
16 }
17
18 public enum Type {
19 ILLEGAL,
20 EOF,
21
22 // Literals
23 IDENTIFIER,
24 INTEGER,
25 STRING,
26
27 // Keywords
28 AND,
29 DEF,
30 DO,
31 ELSE,
32 END,
33 FALSE,
34 FN,
35 IF,
36 IN,
37 LET,
38 THEN,
39 TRUE,
40
41 // Special chars
42 ASTERISK,
43 BANG,
44 BRACKET_LEFT,
45 BRACKET_RIGHT,
46 COMMA,
47 EQUAL,
48 GREATER,
49 GREATER_EQUAL,
50 LESS,
51 LESS_EQUAL,
52 MINUS,
53 MINUS_GREATER,
54 PAREN_LEFT,
55 PAREN_RIGHT,
56 PLUS,
57 QUESTION_EQUAL,
58 SEMICOLON,
59 SLASH,
60 SLASH_EQUAL,
61 }
62}
diff --git a/lexer/src/main/java/module-info.java b/lexer/src/main/java/module-info.java
new file mode 100644
index 0000000..a57a694
--- /dev/null
+++ b/lexer/src/main/java/module-info.java
@@ -0,0 +1,5 @@
1module lv.enes.orang.lexer {
2 exports lv.enes.orang.lexer;
3
4 requires lv.enes.orang.utils;
5} \ No newline at end of file
diff --git a/orang/build.gradle.kts b/orang/build.gradle.kts
index f70609e..ef78ded 100644
--- a/orang/build.gradle.kts
+++ b/orang/build.gradle.kts
@@ -24,6 +24,7 @@ dependencies {
24 24
25 implementation(project(":ast")) 25 implementation(project(":ast"))
26 implementation(project(":core")) 26 implementation(project(":core"))
27 implementation(project(":parser"))
27 implementation(project(":utils")) 28 implementation(project(":utils"))
28} 29}
29 30
diff --git a/orang/src/main/java/lv/enes/orang/Codepoint.java b/orang/src/main/java/lv/enes/orang/Codepoint.java
deleted file mode 100644
index 7157062..0000000
--- a/orang/src/main/java/lv/enes/orang/Codepoint.java
+++ /dev/null
@@ -1,28 +0,0 @@
1package lv.enes.orang;
2
3public record Codepoint(int cp) {
4 @Override
5 public String toString() {
6 return Character.toString(cp);
7 }
8
9 public boolean isIdentInitial() {
10 return Character.isLetter(cp) || cp == '_';
11 }
12
13 public boolean isIdentFinal() {
14 return isIdentInitial() || Character.isDigit(cp);
15 }
16
17 public boolean isNewline() {
18 return cp == '\n';
19 }
20
21 public boolean isNumeral() {
22 return Character.isDigit(cp);
23 }
24
25 public boolean isWhitespace() {
26 return Character.isWhitespace(cp);
27 }
28}
diff --git a/orang/src/main/java/lv/enes/orang/Lexer.java b/orang/src/main/java/lv/enes/orang/Lexer.java
deleted file mode 100644
index d4e1533..0000000
--- a/orang/src/main/java/lv/enes/orang/Lexer.java
+++ /dev/null
@@ -1,174 +0,0 @@
1package lv.enes.orang;
2
3import java.io.*;
4import java.util.Iterator;
5import java.util.function.BiFunction;
6import java.util.function.Predicate;
7import java.util.stream.IntStream;
8import java.util.stream.Stream;
9
10public class Lexer implements Iterator<Token> {
11 private final PeekableStream<Codepoint> input;
12
13 public Lexer(InputStream input) {
14 this(new InputStreamReader(input));
15 }
16
17 public Lexer(Reader input) {
18 var cpStream = new BufferedReader(input)
19 .lines()
20 .flatMapToInt(str -> IntStream.concat(str.codePoints(), IntStream.of('\n')))
21 .mapToObj(Codepoint::new);
22 var theEof = Stream.of(new Codepoint(-1));
23 this.input = new PeekableStream<>(Stream.concat(cpStream, theEof).iterator());
24 }
25
26 public Lexer(String input) {
27 this(new StringReader(input));
28 }
29
30 private boolean hasNext = true;
31
32 @Override
33 public Token next() {
34 var tok = nextToken();
35 if (tok.type() == TokenType.EOF) {
36 hasNext = false;
37 }
38 return tok;
39 }
40
41 @Override
42 public boolean hasNext() {
43 return hasNext;
44 }
45
46 private Token nextToken() {
47 skipWhitespace();
48 return switch (input.peek().cp()) {
49 case -1 -> new Token(TokenType.EOF, "");
50
51 case '*' -> new Token(TokenType.ASTERISK, input.next());
52 case '!' -> new Token(TokenType.BANG, input.next());
53 case '[' -> new Token(TokenType.BRACKET_LEFT, input.next());
54 case ']' -> new Token(TokenType.BRACKET_RIGHT, input.next());
55 case ',' -> new Token(TokenType.COMMA, input.next());
56 case '=' -> new Token(TokenType.EQUAL, input.next());
57 case '>' -> {
58 var first = input.next();
59 if (input.peek().cp() == '=') {
60 yield new Token(TokenType.GREATER_EQUAL, first, input.next());
61 } else {
62 yield new Token(TokenType.GREATER, first);
63 }
64 }
65 case '<' -> {
66 var first = input.next();
67 if (input.peek().cp() == '=') {
68 yield new Token(TokenType.LESS_EQUAL, first, input.next());
69 } else {
70 yield new Token(TokenType.LESS, first);
71 }
72 }
73 case '-' -> {
74 var first = input.next();
75 if (input.peek().cp() == '>') {
76 yield new Token(TokenType.MINUS_GREATER, first, input.next());
77 } else {
78 yield new Token(TokenType.MINUS, first);
79 }
80 }
81 case '(' -> new Token(TokenType.PAREN_LEFT, input.next());
82 case ')' -> new Token(TokenType.PAREN_RIGHT, input.next());
83 case '+' -> new Token(TokenType.PLUS, input.next());
84 case '?' -> {
85 var first = input.next();
86 if (input.peek().cp() == '=') {
87 yield new Token(TokenType.QUESTION_EQUAL, first, input.next());
88 } else {
89 yield new Token(TokenType.ILLEGAL, first, input.next());
90 }
91 }
92 case ';' -> new Token(TokenType.SEMICOLON, input.next());
93 case '/' -> {
94 var first = input.next();
95 if (input.peek().cp() == '=') {
96 yield new Token(TokenType.SLASH_EQUAL, first, input.next());
97 } else {
98 yield new Token(TokenType.SLASH, first);
99 }
100 }
101
102 case '"' -> new Token(TokenType.STRING, readString());
103
104 default -> {
105 if (input.peek().isIdentInitial()) {
106 var ident = readIdentifier();
107 var type = switch (ident) {
108 case "and" -> TokenType.AND;
109 case "def" -> TokenType.DEF;
110 case "do" -> TokenType.DO;
111 case "else" -> TokenType.ELSE;
112 case "end" -> TokenType.END;
113 case "false" -> TokenType.FALSE;
114 case "fn" -> TokenType.FN;
115 case "if" -> TokenType.IF;
116 case "in" -> TokenType.IN;
117 case "let" -> TokenType.LET;
118 case "then" -> TokenType.THEN;
119 case "true" -> TokenType.TRUE;
120 default -> TokenType.IDENTIFIER;
121 };
122 yield new Token(type, ident);
123 } else if (input.peek().isNumeral()) {
124 yield new Token(TokenType.INTEGER, readInteger());
125 } else {
126 yield new Token(TokenType.ILLEGAL, input.next());
127 }
128 }
129 };
130 }
131
132 private <T> T foldWhile(Predicate<Codepoint> pred, T initial, BiFunction<T, Codepoint, T> combine) {
133 var res = initial;
134 var ch = input.peek();
135 while (pred.test(ch)) {
136 res = combine.apply(res, input.next());
137 ch = input.peek();
138 }
139 return res;
140 }
141
142 private String readWhile(Predicate<Codepoint> pred) {
143 return foldWhile(pred, new StringBuilder(), StringBuilder::append).toString();
144 }
145
146 private void skipWhile(Predicate<Codepoint> pred) {
147 foldWhile(pred, Object.class, (x, _) -> x);
148 }
149
150 private String readIdentifier() {
151 return readWhile(Codepoint::isIdentFinal);
152 }
153
154 private String readInteger() {
155 return readWhile(Codepoint::isNumeral);
156 }
157
158 private String readString() {
159 input.next();
160 var literal = readWhile(cp -> cp.cp() != '"');
161 input.next();
162 return literal;
163 }
164
165 private void skipWhitespace() {
166 while (true) {
167 skipWhile(Codepoint::isWhitespace);
168 if (input.peek().cp() != '#') {
169 return;
170 }
171 skipWhile(cp -> !cp.isNewline());
172 }
173 }
174}
diff --git a/orang/src/main/java/lv/enes/orang/Main.java b/orang/src/main/java/lv/enes/orang/Main.java
index 7ca14a1..eb4bfcc 100644
--- a/orang/src/main/java/lv/enes/orang/Main.java
+++ b/orang/src/main/java/lv/enes/orang/Main.java
@@ -1,6 +1,7 @@
1package lv.enes.orang; 1package lv.enes.orang;
2 2
3import lv.enes.orang.core.OrangException; 3import lv.enes.orang.core.OrangException;
4import lv.enes.orang.parser.Parser;
4 5
5import java.io.FileReader; 6import java.io.FileReader;
6import java.io.IOException; 7import java.io.IOException;
diff --git a/orang/src/main/java/lv/enes/orang/Token.java b/orang/src/main/java/lv/enes/orang/Token.java
deleted file mode 100644
index 4456b8f..0000000
--- a/orang/src/main/java/lv/enes/orang/Token.java
+++ /dev/null
@@ -1,15 +0,0 @@
1package lv.enes.orang;
2
3public record Token(TokenType type, String literal) {
4 public Token(TokenType type, Codepoint... cps) {
5 this(type, codepointsToString(cps));
6 }
7
8 private static String codepointsToString(Codepoint... cps) {
9 var sb = new StringBuilder(cps.length);
10 for (var cp : cps) {
11 sb.append(cp);
12 }
13 return sb.toString();
14 }
15}
diff --git a/orang/src/main/java/lv/enes/orang/TokenType.java b/orang/src/main/java/lv/enes/orang/TokenType.java
deleted file mode 100644
index 960435e..0000000
--- a/orang/src/main/java/lv/enes/orang/TokenType.java
+++ /dev/null
@@ -1,91 +0,0 @@
1package lv.enes.orang;
2
3import lv.enes.orang.ast.BinaryExpression;
4import lv.enes.orang.ast.UnaryExpression;
5
6public enum TokenType {
7 ILLEGAL,
8 EOF,
9
10 // Literals
11 IDENTIFIER,
12 INTEGER,
13 STRING,
14
15 // Keywords
16 AND,
17 DEF,
18 DO,
19 ELSE,
20 END,
21 FALSE,
22 FN,
23 IF,
24 IN,
25 LET,
26 THEN,
27 TRUE,
28
29 // Special chars
30 ASTERISK,
31 BANG,
32 BRACKET_LEFT,
33 BRACKET_RIGHT,
34 COMMA,
35 EQUAL,
36 GREATER,
37 GREATER_EQUAL,
38 LESS,
39 LESS_EQUAL,
40 MINUS,
41 MINUS_GREATER,
42 PAREN_LEFT,
43 PAREN_RIGHT,
44 PLUS,
45 QUESTION_EQUAL,
46 SEMICOLON,
47 SLASH,
48 SLASH_EQUAL,
49
50 ;
51
52 public boolean isBinaryOp() {
53 return switch (this) {
54 case ASTERISK, SLASH, PLUS, MINUS, QUESTION_EQUAL, SLASH_EQUAL, GREATER, GREATER_EQUAL, LESS, LESS_EQUAL
55 -> true;
56 default -> false;
57 };
58 }
59
60 public BinaryExpression.Operator toBinaryOp() {
61 return switch (this) {
62 case ASTERISK -> BinaryExpression.Operator.MULTIPLY;
63 case SLASH -> BinaryExpression.Operator.DIVIDE;
64 case PLUS -> BinaryExpression.Operator.ADD;
65 case MINUS -> BinaryExpression.Operator.SUBTRACT;
66 case QUESTION_EQUAL -> BinaryExpression.Operator.EQUALS;
67 case SLASH_EQUAL -> BinaryExpression.Operator.NOT_EQUALS;
68 case GREATER -> BinaryExpression.Operator.GT;
69 case GREATER_EQUAL -> BinaryExpression.Operator.GTE;
70 case LESS -> BinaryExpression.Operator.LT;
71 case LESS_EQUAL -> BinaryExpression.Operator.LTE;
72 default -> throw new IllegalStateException("Token " + this + " is not a binary operator");
73 };
74 }
75
76 public boolean isUnaryOp() {
77 return switch (this) {
78 case PLUS, MINUS, BANG -> true;
79 default -> false;
80 };
81 }
82
83 public UnaryExpression.Operator toUnaryOp() {
84 return switch (this) {
85 case PLUS -> UnaryExpression.Operator.PLUS;
86 case MINUS -> UnaryExpression.Operator.NEGATE;
87 case BANG -> UnaryExpression.Operator.NOT;
88 default -> throw new IllegalStateException("Token " + this + " is not a unary operator");
89 };
90 }
91}
diff --git a/orang/src/main/java/module-info.java b/orang/src/main/java/module-info.java
index 060657f..353d0dd 100644
--- a/orang/src/main/java/module-info.java
+++ b/orang/src/main/java/module-info.java
@@ -4,6 +4,7 @@ module lv.enes.orang {
4 4
5 requires lv.enes.orang.ast; 5 requires lv.enes.orang.ast;
6 requires lv.enes.orang.core; 6 requires lv.enes.orang.core;
7 requires lv.enes.orang.parser;
7 requires lv.enes.orang.utils; 8 requires lv.enes.orang.utils;
8 9
9 requires static jakarta.annotation; 10 requires static jakarta.annotation;
diff --git a/parser/build.gradle.kts b/parser/build.gradle.kts
new file mode 100644
index 0000000..9934d9d
--- /dev/null
+++ b/parser/build.gradle.kts
@@ -0,0 +1,22 @@
1plugins {
2 java
3}
4
5dependencies {
6 implementation(project(":ast"))
7 implementation(project(":core"))
8 implementation(project(":lexer"))
9 implementation(project(":utils"))
10}
11
12java {
13 sourceCompatibility = JavaVersion.VERSION_22
14 targetCompatibility = JavaVersion.VERSION_22
15 toolchain {
16 languageVersion = JavaLanguageVersion.of(22)
17 }
18}
19
20tasks.withType<JavaCompile> {
21 options.compilerArgs.add("--enable-preview")
22} \ No newline at end of file
diff --git a/orang/src/main/java/lv/enes/orang/Parser.java b/parser/src/main/java/lv/enes/orang/parser/Parser.java
index 77abe24..6c86e85 100644
--- a/orang/src/main/java/lv/enes/orang/Parser.java
+++ b/parser/src/main/java/lv/enes/orang/parser/Parser.java
@@ -1,9 +1,12 @@
1package lv.enes.orang; 1package lv.enes.orang.parser;
2 2
3import lv.enes.orang.ast.*; 3import lv.enes.orang.ast.*;
4import lv.enes.orang.ast.IfElseExpression; 4import lv.enes.orang.ast.IfElseExpression;
5import lv.enes.orang.ast.Statement; 5import lv.enes.orang.ast.Statement;
6import lv.enes.orang.lexer.Lexer;
7import lv.enes.orang.lexer.Token;
6import lv.enes.orang.utils.NonEmptyList; 8import lv.enes.orang.utils.NonEmptyList;
9import lv.enes.orang.utils.PeekableStream;
7 10
8import java.io.InputStream; 11import java.io.InputStream;
9import java.io.Reader; 12import java.io.Reader;
@@ -49,13 +52,53 @@ public class Parser {
49 52
50 public Program parseProgram() throws ParserException { 53 public Program parseProgram() throws ParserException {
51 var statements = new ArrayList<Statement>(); 54 var statements = new ArrayList<Statement>();
52 while (!maybeConsumeToken(TokenType.EOF)) { 55 while (!maybeConsumeToken(Token.Type.EOF)) {
53 statements.add(parseStatement()); 56 statements.add(parseStatement());
54 maybeConsumeToken(TokenType.SEMICOLON); 57 maybeConsumeToken(Token.Type.SEMICOLON);
55 } 58 }
56 return new Program(Collections.unmodifiableList(statements)); 59 return new Program(Collections.unmodifiableList(statements));
57 } 60 }
58 61
62 private static boolean isBinaryOp(Token token) {
63 return switch (token.type()) {
64 case ASTERISK, SLASH, PLUS, MINUS, QUESTION_EQUAL, SLASH_EQUAL, GREATER, GREATER_EQUAL, LESS, LESS_EQUAL
65 -> true;
66 default -> false;
67 };
68 }
69
70 private static BinaryExpression.Operator toBinaryOp(Token token) {
71 return switch (token.type()) {
72 case ASTERISK -> BinaryExpression.Operator.MULTIPLY;
73 case SLASH -> BinaryExpression.Operator.DIVIDE;
74 case PLUS -> BinaryExpression.Operator.ADD;
75 case MINUS -> BinaryExpression.Operator.SUBTRACT;
76 case QUESTION_EQUAL -> BinaryExpression.Operator.EQUALS;
77 case SLASH_EQUAL -> BinaryExpression.Operator.NOT_EQUALS;
78 case GREATER -> BinaryExpression.Operator.GT;
79 case GREATER_EQUAL -> BinaryExpression.Operator.GTE;
80 case LESS -> BinaryExpression.Operator.LT;
81 case LESS_EQUAL -> BinaryExpression.Operator.LTE;
82 default -> throw new IllegalStateException(STR."Token \{token.type()} is not a binary operator");
83 };
84 }
85
86 public static boolean isUnaryOp(Token token) {
87 return switch (token.type()) {
88 case PLUS, MINUS, BANG -> true;
89 default -> false;
90 };
91 }
92
93 public static UnaryExpression.Operator toUnaryOp(Token token) {
94 return switch (token.type()) {
95 case PLUS -> UnaryExpression.Operator.PLUS;
96 case MINUS -> UnaryExpression.Operator.NEGATE;
97 case BANG -> UnaryExpression.Operator.NOT;
98 default -> throw new IllegalStateException(STR."Token \{token.type()} is not a unary operator");
99 };
100 }
101
59 private Token consume(Predicate<Token> pred, String msg) throws ParserException { 102 private Token consume(Predicate<Token> pred, String msg) throws ParserException {
60 var tok = input.next(); 103 var tok = input.next();
61 if (!pred.test(tok)) { 104 if (!pred.test(tok)) {
@@ -64,11 +107,11 @@ public class Parser {
64 return tok; 107 return tok;
65 } 108 }
66 109
67 private Token consumeToken(TokenType type) throws ParserException { 110 private Token consumeToken(Token.Type type) throws ParserException {
68 return consume(tok -> tok.type() == type, STR."Expected \{type}"); 111 return consume(tok -> tok.type() == type, STR."Expected \{type}");
69 } 112 }
70 113
71 private boolean maybeConsumeToken(TokenType type) { 114 private boolean maybeConsumeToken(Token.Type type) {
72 if (input.peek().type() == type) { 115 if (input.peek().type() == type) {
73 input.next(); 116 input.next();
74 return true; 117 return true;
@@ -77,16 +120,16 @@ public class Parser {
77 } 120 }
78 121
79 private ArrayExpression parseArray() throws ParserException { 122 private ArrayExpression parseArray() throws ParserException {
80 consumeToken(TokenType.BRACKET_LEFT); 123 consumeToken(Token.Type.BRACKET_LEFT);
81 if (maybeConsumeToken(TokenType.BRACKET_RIGHT)) { 124 if (maybeConsumeToken(Token.Type.BRACKET_RIGHT)) {
82 return new ArrayExpression(List.of()); 125 return new ArrayExpression(List.of());
83 } 126 }
84 127
85 var items = new ArrayList<Expression>(); 128 var items = new ArrayList<Expression>();
86 do { 129 do {
87 items.add(parseExpression()); 130 items.add(parseExpression());
88 } while (maybeConsumeToken(TokenType.COMMA)); 131 } while (maybeConsumeToken(Token.Type.COMMA));
89 consumeToken(TokenType.BRACKET_RIGHT); 132 consumeToken(Token.Type.BRACKET_RIGHT);
90 133
91 return new ArrayExpression(Collections.unmodifiableList(items)); 134 return new ArrayExpression(Collections.unmodifiableList(items));
92 } 135 }
@@ -94,11 +137,11 @@ public class Parser {
94 private List<ArgSpec> parseArgSpecs() throws ParserException { 137 private List<ArgSpec> parseArgSpecs() throws ParserException {
95 var argSpecs = new ArrayList<ArgSpec>(); 138 var argSpecs = new ArrayList<ArgSpec>();
96 while (true) { 139 while (true) {
97 if (input.peek().type() == TokenType.IDENTIFIER) { 140 if (input.peek().type() == Token.Type.IDENTIFIER) {
98 argSpecs.add(ArgSpec.named(input.next().literal())); 141 argSpecs.add(ArgSpec.named(input.next().literal()));
99 } else if (input.peek().type() == TokenType.PAREN_LEFT) { 142 } else if (input.peek().type() == Token.Type.PAREN_LEFT) {
100 consumeToken(TokenType.PAREN_LEFT); 143 consumeToken(Token.Type.PAREN_LEFT);
101 consumeToken(TokenType.PAREN_RIGHT); 144 consumeToken(Token.Type.PAREN_RIGHT);
102 argSpecs.add(ArgSpec.nothing()); 145 argSpecs.add(ArgSpec.nothing());
103 } else { 146 } else {
104 break; 147 break;
@@ -109,20 +152,20 @@ public class Parser {
109 152
110 private Expression parseBinaryExpression() throws ParserException { 153 private Expression parseBinaryExpression() throws ParserException {
111 var lhs = parseCallExpression(); 154 var lhs = parseCallExpression();
112 if (!input.peek().type().isBinaryOp()) { 155 if (!isBinaryOp(input.peek())) {
113 return lhs; 156 return lhs;
114 } 157 }
115 158
116 return parseBinaryExpressionRhs(lhs, input.next().type().toBinaryOp()); 159 return parseBinaryExpressionRhs(lhs, toBinaryOp(input.next()));
117 } 160 }
118 161
119 private Expression parseBinaryExpressionRhs(Expression lhs, BinaryExpression.Operator op) throws ParserException { 162 private Expression parseBinaryExpressionRhs(Expression lhs, BinaryExpression.Operator op) throws ParserException {
120 var rhs = parseCallExpression(); 163 var rhs = parseCallExpression();
121 if (!input.peek().type().isBinaryOp()) { 164 if (!isBinaryOp(input.peek())) {
122 return new BinaryExpression(op, lhs, rhs); 165 return new BinaryExpression(op, lhs, rhs);
123 } 166 }
124 167
125 var op2 = input.next().type().toBinaryOp(); 168 var op2 = toBinaryOp(input.next());
126 if (op2.bindsStrongerThan(op)) { 169 if (op2.bindsStrongerThan(op)) {
127 return new BinaryExpression(op, lhs, parseBinaryExpressionRhs(rhs, op2)); 170 return new BinaryExpression(op, lhs, parseBinaryExpressionRhs(rhs, op2));
128 } else { 171 } else {
@@ -131,8 +174,8 @@ public class Parser {
131 } 174 }
132 175
133 private BooleanLiteral parseBoolean() throws ParserException { 176 private BooleanLiteral parseBoolean() throws ParserException {
134 var t = consume(tok -> tok.type() == TokenType.FALSE || tok.type() == TokenType.TRUE, "Expected TRUE or FALSE"); 177 var t = consume(tok -> tok.type() == Token.Type.FALSE || tok.type() == Token.Type.TRUE, "Expected TRUE or FALSE");
135 return new BooleanLiteral(t.type() == TokenType.TRUE); 178 return new BooleanLiteral(t.type() == Token.Type.TRUE);
136 } 179 }
137 180
138 private Expression parseCallExpression() throws ParserException { 181 private Expression parseCallExpression() throws ParserException {
@@ -145,9 +188,9 @@ public class Parser {
145 } 188 }
146 189
147 private Definition parseDefinition() throws ParserException { 190 private Definition parseDefinition() throws ParserException {
148 consumeToken(TokenType.DEF); 191 consumeToken(Token.Type.DEF);
149 var defSpec = parseDefSpec(); 192 var defSpec = parseDefSpec();
150 consumeToken(TokenType.EQUAL); 193 consumeToken(Token.Type.EQUAL);
151 var value = parseExpression(); 194 var value = parseExpression();
152 if (defSpec.args().isEmpty()) { 195 if (defSpec.args().isEmpty()) {
153 return new Definition(defSpec.name(), value); 196 return new Definition(defSpec.name(), value);
@@ -157,59 +200,59 @@ public class Parser {
157 } 200 }
158 201
159 private DefSpec parseDefSpec() throws ParserException { 202 private DefSpec parseDefSpec() throws ParserException {
160 var name = consumeToken(TokenType.IDENTIFIER).literal(); 203 var name = consumeToken(Token.Type.IDENTIFIER).literal();
161 var argSpecs = parseArgSpecs(); 204 var argSpecs = parseArgSpecs();
162 return new DefSpec(name, argSpecs); 205 return new DefSpec(name, argSpecs);
163 } 206 }
164 207
165 private DoExpression parseDoExpression() throws ParserException { 208 private DoExpression parseDoExpression() throws ParserException {
166 consumeToken(TokenType.DO); 209 consumeToken(Token.Type.DO);
167 var exprs = new ArrayList<Expression>(); 210 var exprs = new ArrayList<Expression>();
168 do { 211 do {
169 exprs.add(parseExpression()); 212 exprs.add(parseExpression());
170 } while (maybeConsumeToken(TokenType.SEMICOLON)); 213 } while (maybeConsumeToken(Token.Type.SEMICOLON));
171 consumeToken(TokenType.END); 214 consumeToken(Token.Type.END);
172 return new DoExpression(Collections.unmodifiableList(exprs)); 215 return new DoExpression(Collections.unmodifiableList(exprs));
173 } 216 }
174 217
175 private Expression parseExpression() throws ParserException { 218 private Expression parseExpression() throws ParserException {
176 if (input.peek().type().isUnaryOp()) { 219 if (isUnaryOp(input.peek())) {
177 return parseUnaryExpression(); 220 return parseUnaryExpression();
178 } 221 }
179 return parseBinaryExpression(); 222 return parseBinaryExpression();
180 } 223 }
181 224
182 private FnExpression parseFnExpression() throws ParserException { 225 private FnExpression parseFnExpression() throws ParserException {
183 consumeToken(TokenType.FN); 226 consumeToken(Token.Type.FN);
184 var argSpecs = parseArgSpecs(); 227 var argSpecs = parseArgSpecs();
185 if (argSpecs.isEmpty()) { 228 if (argSpecs.isEmpty()) {
186 throw new ParserException("Function definition with no arguments"); 229 throw new ParserException("Function definition with no arguments");
187 } 230 }
188 var body = maybeConsumeToken(TokenType.MINUS_GREATER) ? parseExpression() : parseDoExpression(); 231 var body = maybeConsumeToken(Token.Type.MINUS_GREATER) ? parseExpression() : parseDoExpression();
189 return new FnExpression(new NonEmptyList<>(argSpecs), body); 232 return new FnExpression(new NonEmptyList<>(argSpecs), body);
190 } 233 }
191 234
192 private IfElseExpression parseIfElseExpression() throws ParserException { 235 private IfElseExpression parseIfElseExpression() throws ParserException {
193 consumeToken(TokenType.IF); 236 consumeToken(Token.Type.IF);
194 var cond = parseExpression(); 237 var cond = parseExpression();
195 consumeToken(TokenType.THEN); 238 consumeToken(Token.Type.THEN);
196 var trueBranch = parseExpression(); 239 var trueBranch = parseExpression();
197 consumeToken(TokenType.ELSE); 240 consumeToken(Token.Type.ELSE);
198 var falseBranch = parseExpression(); 241 var falseBranch = parseExpression();
199 return new IfElseExpression(cond, trueBranch, falseBranch); 242 return new IfElseExpression(cond, trueBranch, falseBranch);
200 } 243 }
201 244
202 private IntLiteral parseInteger() throws ParserException { 245 private IntLiteral parseInteger() throws ParserException {
203 var tok = consumeToken(TokenType.INTEGER); 246 var tok = consumeToken(Token.Type.INTEGER);
204 return new IntLiteral(Integer.parseInt(tok.literal())); 247 return new IntLiteral(Integer.parseInt(tok.literal()));
205 } 248 }
206 249
207 private LetInExpression parseLetInExpression() throws ParserException { 250 private LetInExpression parseLetInExpression() throws ParserException {
208 consumeToken(TokenType.LET); 251 consumeToken(Token.Type.LET);
209 var bindings = new ArrayList<LetInExpression.Binding>(); 252 var bindings = new ArrayList<LetInExpression.Binding>();
210 do { 253 do {
211 var defSpec = parseDefSpec(); 254 var defSpec = parseDefSpec();
212 consumeToken(TokenType.EQUAL); 255 consumeToken(Token.Type.EQUAL);
213 var value = parseExpression(); 256 var value = parseExpression();
214 if (defSpec.args().isEmpty()) { 257 if (defSpec.args().isEmpty()) {
215 bindings.add(new LetInExpression.Binding(defSpec.name(), value)); 258 bindings.add(new LetInExpression.Binding(defSpec.name(), value));
@@ -217,8 +260,8 @@ public class Parser {
217 var fn = new FnExpression(new NonEmptyList<>(defSpec.args()), value); 260 var fn = new FnExpression(new NonEmptyList<>(defSpec.args()), value);
218 bindings.add(new LetInExpression.Binding(defSpec.name(), fn)); 261 bindings.add(new LetInExpression.Binding(defSpec.name(), fn));
219 } 262 }
220 } while (maybeConsumeToken(TokenType.AND)); 263 } while (maybeConsumeToken(Token.Type.AND));
221 consumeToken(TokenType.IN); 264 consumeToken(Token.Type.IN);
222 var body = parseExpression(); 265 var body = parseExpression();
223 return new LetInExpression(Collections.unmodifiableList(bindings), body); 266 return new LetInExpression(Collections.unmodifiableList(bindings), body);
224 } 267 }
@@ -226,12 +269,12 @@ public class Parser {
226 private Expression parseSimpleExpression() throws ParserException { 269 private Expression parseSimpleExpression() throws ParserException {
227 return switch (input.peek().type()) { 270 return switch (input.peek().type()) {
228 case PAREN_LEFT -> { 271 case PAREN_LEFT -> {
229 consumeToken(TokenType.PAREN_LEFT); 272 consumeToken(Token.Type.PAREN_LEFT);
230 if (maybeConsumeToken(TokenType.PAREN_RIGHT)) { 273 if (maybeConsumeToken(Token.Type.PAREN_RIGHT)) {
231 yield VoidExpression.INSTANCE; 274 yield VoidExpression.INSTANCE;
232 } 275 }
233 var expr = parseExpression(); 276 var expr = parseExpression();
234 consumeToken(TokenType.PAREN_RIGHT); 277 consumeToken(Token.Type.PAREN_RIGHT);
235 yield expr; 278 yield expr;
236 } 279 }
237 case TRUE, FALSE -> parseBoolean(); 280 case TRUE, FALSE -> parseBoolean();
@@ -247,7 +290,7 @@ public class Parser {
247 }; 290 };
248 } 291 }
249 292
250 private boolean couldStartSimpleExpression(TokenType type) { 293 private boolean couldStartSimpleExpression(Token.Type type) {
251 return switch (type) { 294 return switch (type) {
252 case PAREN_LEFT, TRUE, FALSE, INTEGER, IDENTIFIER, STRING, BRACKET_LEFT, IF, LET, FN, DO -> true; 295 case PAREN_LEFT, TRUE, FALSE, INTEGER, IDENTIFIER, STRING, BRACKET_LEFT, IF, LET, FN, DO -> true;
253 default -> false; 296 default -> false;
@@ -255,7 +298,7 @@ public class Parser {
255 } 298 }
256 299
257 private Statement parseStatement() throws ParserException { 300 private Statement parseStatement() throws ParserException {
258 if (input.peek().type() == TokenType.DEF) { 301 if (input.peek().type() == Token.Type.DEF) {
259 return parseDefinition(); 302 return parseDefinition();
260 } else { 303 } else {
261 return new ExpressionStatement(parseExpression()); 304 return new ExpressionStatement(parseExpression());
@@ -286,8 +329,8 @@ public class Parser {
286 } 329 }
287 330
288 private Expression parseUnaryExpression() throws ParserException { 331 private Expression parseUnaryExpression() throws ParserException {
289 if (input.peek().type().isUnaryOp()) { 332 if (isUnaryOp(input.peek())) {
290 var op = input.next().type().toUnaryOp(); 333 var op = toUnaryOp(input.next());
291 return new UnaryExpression(op, parseUnaryExpression()); 334 return new UnaryExpression(op, parseUnaryExpression());
292 } else { 335 } else {
293 return parseSimpleExpression(); 336 return parseSimpleExpression();
diff --git a/orang/src/main/java/lv/enes/orang/ParserException.java b/parser/src/main/java/lv/enes/orang/parser/ParserException.java
index bd65e7a..632ce15 100644
--- a/orang/src/main/java/lv/enes/orang/ParserException.java
+++ b/parser/src/main/java/lv/enes/orang/parser/ParserException.java
@@ -1,4 +1,4 @@
1package lv.enes.orang; 1package lv.enes.orang.parser;
2 2
3import lv.enes.orang.core.OrangException; 3import lv.enes.orang.core.OrangException;
4 4
diff --git a/parser/src/main/java/module-info.java b/parser/src/main/java/module-info.java
new file mode 100644
index 0000000..43c2dc5
--- /dev/null
+++ b/parser/src/main/java/module-info.java
@@ -0,0 +1,8 @@
1module lv.enes.orang.parser {
2 exports lv.enes.orang.parser;
3
4 requires lv.enes.orang.ast;
5 requires lv.enes.orang.core;
6 requires lv.enes.orang.lexer;
7 requires lv.enes.orang.utils;
8} \ No newline at end of file
diff --git a/settings.gradle.kts b/settings.gradle.kts
index fc692fa..391d33b 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -1,3 +1,3 @@
1rootProject.name = "orang" 1rootProject.name = "orang"
2 2
3include("ast", "core", "orang", "utils") \ No newline at end of file 3include("ast", "core", "lexer", "orang", "parser", "utils") \ No newline at end of file
diff --git a/utils/src/main/java/lv/enes/orang/utils/Codepoint.java b/utils/src/main/java/lv/enes/orang/utils/Codepoint.java
new file mode 100644
index 0000000..a981c5e
--- /dev/null
+++ b/utils/src/main/java/lv/enes/orang/utils/Codepoint.java
@@ -0,0 +1,8 @@
1package lv.enes.orang.utils;
2
3public record Codepoint(int cp) {
4 @Override
5 public String toString() {
6 return Character.toString(cp);
7 }
8}
diff --git a/orang/src/main/java/lv/enes/orang/PeekableStream.java b/utils/src/main/java/lv/enes/orang/utils/PeekableStream.java
index b77bab1..7607a50 100644
--- a/orang/src/main/java/lv/enes/orang/PeekableStream.java
+++ b/utils/src/main/java/lv/enes/orang/utils/PeekableStream.java
@@ -1,4 +1,4 @@
1package lv.enes.orang; 1package lv.enes.orang.utils;
2 2
3import java.util.ArrayDeque; 3import java.util.ArrayDeque;
4import java.util.Deque; 4import java.util.Deque;
HIC SVNT DRACONES