equal
deleted
inserted
replaced
1 // A simple lexer inspired by work of Sulzmann & Lu |
1 // A simple tokeniser based on the Sulzmann & Lu algorithm |
2 //================================================== |
2 //========================================================= |
3 |
3 // |
4 |
4 // call with |
5 object Lexer { |
5 // |
|
6 // scala tokenise.scala fib.while |
|
7 // |
|
8 // scala tokenise.scala loops.while |
|
9 |
|
10 object Tokenise { |
6 |
11 |
7 import scala.language.implicitConversions |
12 import scala.language.implicitConversions |
8 import scala.language.reflectiveCalls |
13 import scala.language.reflectiveCalls |
9 |
14 |
10 // regular expressions including records |
15 // regular expressions including records |
215 ("str" $ STRING) | |
220 ("str" $ STRING) | |
216 ("p" $ (LPAREN | RPAREN)) | |
221 ("p" $ (LPAREN | RPAREN)) | |
217 ("w" $ WHITESPACE)).% |
222 ("w" $ WHITESPACE)).% |
218 |
223 |
219 |
224 |
220 // escapes strings and prints them out as "", "\n" and so on |
|
221 def esc(raw: String): String = { |
|
222 import scala.reflect.runtime.universe._ |
|
223 Literal(Constant(raw)).toString |
|
224 } |
|
225 |
|
226 def escape(tks: List[(String, String)]) = |
|
227 tks.map{ case (s1, s2) => (s1, esc(s2))} |
|
228 |
|
229 val prog2 = """ |
|
230 write "Fib"; |
|
231 read n; |
|
232 minus1 := 0; |
|
233 minus2 := 1; |
|
234 while n > 0 do { |
|
235 temp := minus2; |
|
236 minus2 := minus1 + minus2; |
|
237 minus1 := temp; |
|
238 n := n - 1 |
|
239 }; |
|
240 write "Result"; |
|
241 write minus2 |
|
242 """ |
|
243 |
|
244 val prog3 = """ |
|
245 start := 1000; |
|
246 x := start; |
|
247 y := start; |
|
248 z := start; |
|
249 while 0 < x do { |
|
250 while 0 < y do { |
|
251 while 0 < z do { |
|
252 z := z - 1 |
|
253 }; |
|
254 z := start; |
|
255 y := y - 1 |
|
256 }; |
|
257 y := start; |
|
258 x := x - 1 |
|
259 } |
|
260 """ |
|
261 |
225 |
262 // Generating tokens for the WHILE language |
226 // Generating tokens for the WHILE language |
|
227 // and serialising them into a .tks file |
263 |
228 |
264 import java.io._ |
229 import java.io._ |
265 |
230 |
266 abstract class Token extends Serializable |
231 abstract class Token extends Serializable |
267 case object T_SEMI extends Token |
232 case object T_SEMI extends Token |
293 out.writeObject(data) |
258 out.writeObject(data) |
294 out.close |
259 out.close |
295 } |
260 } |
296 |
261 |
297 def main(args: Array[String]) = { |
262 def main(args: Array[String]) = { |
298 serialise("/tmp/nflx", tokenise(prog3)) |
263 val fname = args(0) |
299 } |
264 val file = io.Source.fromFile(fname).mkString |
300 |
265 val tks = fname.stripSuffix(".while") ++ ".tks" |
301 |
266 serialise(tks, tokenise(file)) |
302 } |
267 } |
|
268 |
|
269 |
|
270 } |