|
1 // This produces more meaningful tokens and |
|
2 // also filters out comments and whitespaces |
|
3 // |
|
4 // call with |
|
5 // |
|
6 // amm token.sc |
|
7 // |
|
8 |
|
9 // load the lexer |
|
10 import $file.lexer |
|
11 import lexer._ |
|
12 |
|
13 |
|
14 // The tokens for the WHILE language |
|
15 |
|
16 abstract class Token |
|
17 case object T_SEMI extends Token |
|
18 case object T_LPAREN extends Token |
|
19 case object T_RPAREN extends Token |
|
20 case class T_ID(s: String) extends Token |
|
21 case class T_OP(s: String) extends Token |
|
22 case class T_NUM(n: Int) extends Token |
|
23 case class T_KWD(s: String) extends Token |
|
24 case class T_STR(s: String) extends Token |
|
25 |
|
26 val token : PartialFunction[(String, String), Token] = { |
|
27 case ("s", _) => T_SEMI |
|
28 case ("p", "{") => T_LPAREN |
|
29 case ("p", "}") => T_RPAREN |
|
30 case ("i", s) => T_ID(s) |
|
31 case ("o", s) => T_OP(s) |
|
32 case ("n", s) => T_NUM(s.toInt) |
|
33 case ("k", s) => T_KWD(s) |
|
34 case ("str", s) => T_STR(s) |
|
35 } |
|
36 |
|
37 // by using collect we filter out all unwanted tokens |
|
38 def tokenise(s: String) : List[Token] = |
|
39 lexing_simp(WHILE_REGS, s).collect(token) |
|
40 |
|
41 |
|
42 |
|
43 |
|
44 @doc("Tokens for fib and loops programs.") |
|
45 @main |
|
46 def main() = { |
|
47 println("Fib program") |
|
48 println(tokenise(prog2)) |
|
49 println("Loops program") |
|
50 println(tokenise(prog3)) |
|
51 |
|
52 for (i <- 0 to 20 by 5) { |
|
53 println(f"$i%2.0f: ${time(tokenise(prog3 * i))._2}") |
|
54 } |
|
55 |
|
56 } |