author | Christian Urban <urbanc@in.tum.de> |
Sun, 28 Jul 2019 14:24:46 +0100 | |
changeset 624 | 8d0af38389bc |
parent 617 | f7de0915fff2 |
permissions | -rw-r--r-- |
624 | 1 |
// A Simple Tokenizer according to Sulzmann & Lu |
581 | 2 |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
3 |
import scala.language.implicitConversions |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
4 |
import scala.language.reflectiveCalls |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
5 |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
6 |
abstract class Rexp |
426
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
7 |
case object ZERO extends Rexp |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
8 |
case object ONE extends Rexp |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
9 |
case class CHAR(c: Char) extends Rexp |
579 | 10 |
case class ALT(r1: Rexp, r2: Rexp) extends Rexp |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
11 |
case class SEQ(r1: Rexp, r2: Rexp) extends Rexp |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
12 |
case class STAR(r: Rexp) extends Rexp |
579 | 13 |
case class RECD(x: String, r: Rexp) extends Rexp |
581 | 14 |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
15 |
abstract class Val |
354
86b2aeae3e98
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
352
diff
changeset
|
16 |
case object Empty extends Val |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
17 |
case class Chr(c: Char) extends Val |
520 | 18 |
case class Sequ(v1: Val, v2: Val) extends Val |
579 | 19 |
case class Left(v: Val) extends Val |
20 |
case class Right(v: Val) extends Val |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
21 |
case class Stars(vs: List[Val]) extends Val |
579 | 22 |
case class Rec(x: String, v: Val) extends Val |
23 |
||
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
24 |
// some convenience for typing in regular expressions |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
25 |
def charlist2rexp(s : List[Char]): Rexp = s match { |
426
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
26 |
case Nil => ONE |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
27 |
case c::Nil => CHAR(c) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
28 |
case c::s => SEQ(CHAR(c), charlist2rexp(s)) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
29 |
} |
581 | 30 |
implicit def string2rexp(s : String) : Rexp = |
31 |
charlist2rexp(s.toList) |
|
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
32 |
|
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
33 |
implicit def RexpOps(r: Rexp) = new { |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
34 |
def | (s: Rexp) = ALT(r, s) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
35 |
def % = STAR(r) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
36 |
def ~ (s: Rexp) = SEQ(r, s) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
37 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
38 |
|
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
39 |
implicit def stringOps(s: String) = new { |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
40 |
def | (r: Rexp) = ALT(s, r) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
41 |
def | (r: String) = ALT(s, r) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
42 |
def % = STAR(s) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
43 |
def ~ (r: Rexp) = SEQ(s, r) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
44 |
def ~ (r: String) = SEQ(s, r) |
579 | 45 |
def $ (r: Rexp) = RECD(s, r) |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
46 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
47 |
|
581 | 48 |
// A test for more conveninet syntax |
49 |
val re : Rexp = ("ab" | "a") ~ ("b" | ONE) |
|
50 |
||
624 | 51 |
// the nullable function: tests whether the regular |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
52 |
// expression can recognise the empty string |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
53 |
def nullable (r: Rexp) : Boolean = r match { |
426
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
54 |
case ZERO => false |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
55 |
case ONE => true |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
56 |
case CHAR(_) => false |
579 | 57 |
case ALT(r1, r2) => nullable(r1) || nullable(r2) |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
58 |
case SEQ(r1, r2) => nullable(r1) && nullable(r2) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
59 |
case STAR(_) => true |
579 | 60 |
case RECD(_, r1) => nullable(r1) |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
61 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
62 |
|
624 | 63 |
// the derivative of a regular expression w.r.t. a character |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
64 |
def der (c: Char, r: Rexp) : Rexp = r match { |
426
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
65 |
case ZERO => ZERO |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
66 |
case ONE => ZERO |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
67 |
case CHAR(d) => if (c == d) ONE else ZERO |
579 | 68 |
case ALT(r1, r2) => ALT(der(c, r1), der(c, r2)) |
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
69 |
case SEQ(r1, r2) => |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
70 |
if (nullable(r1)) ALT(SEQ(der(c, r1), r2), der(c, r2)) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
71 |
else SEQ(der(c, r1), r2) |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
72 |
case STAR(r) => SEQ(der(c, r), STAR(r)) |
579 | 73 |
case RECD(_, r1) => der(c, r1) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
74 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
75 |
|
624 | 76 |
// the derivative w.r.t. a string (iterates der) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
77 |
def ders (s: List[Char], r: Rexp) : Rexp = s match { |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
78 |
case Nil => r |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
79 |
case c::s => ders(s, der(c, r)) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
80 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
81 |
|
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
82 |
// extracts a string from value |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
83 |
def flatten(v: Val) : String = v match { |
354
86b2aeae3e98
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
352
diff
changeset
|
84 |
case Empty => "" |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
85 |
case Chr(c) => c.toString |
579 | 86 |
case Left(v) => flatten(v) |
87 |
case Right(v) => flatten(v) |
|
88 |
case Sequ(v1, v2) => flatten(v1) + flatten(v2) |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
89 |
case Stars(vs) => vs.map(flatten).mkString |
579 | 90 |
case Rec(_, v) => flatten(v) |
91 |
} |
|
92 |
||
624 | 93 |
// extracts an environment from a value; |
581 | 94 |
// used for tokenise a string |
579 | 95 |
def env(v: Val) : List[(String, String)] = v match { |
96 |
case Empty => Nil |
|
97 |
case Chr(c) => Nil |
|
98 |
case Left(v) => env(v) |
|
99 |
case Right(v) => env(v) |
|
100 |
case Sequ(v1, v2) => env(v1) ::: env(v2) |
|
101 |
case Stars(vs) => vs.flatMap(env) |
|
102 |
case Rec(x, v) => (x, flatten(v))::env(v) |
|
103 |
} |
|
104 |
||
624 | 105 |
// The Injection Part of the Tokeniser |
106 |
||
107 |
// calculates a value for how a nullable regex |
|
108 |
// matches the empty string |
|
579 | 109 |
def mkeps(r: Rexp) : Val = r match { |
110 |
case ONE => Empty |
|
111 |
case ALT(r1, r2) => |
|
112 |
if (nullable(r1)) Left(mkeps(r1)) else Right(mkeps(r2)) |
|
113 |
case SEQ(r1, r2) => Sequ(mkeps(r1), mkeps(r2)) |
|
114 |
case STAR(r) => Stars(Nil) |
|
115 |
case RECD(x, r) => Rec(x, mkeps(r)) |
|
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
116 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
117 |
|
624 | 118 |
// injects back a character into a value |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
119 |
def inj(r: Rexp, c: Char, v: Val) : Val = (r, v) match { |
520 | 120 |
case (STAR(r), Sequ(v1, Stars(vs))) => Stars(inj(r, c, v1)::vs) |
121 |
case (SEQ(r1, r2), Sequ(v1, v2)) => Sequ(inj(r1, c, v1), v2) |
|
579 | 122 |
case (SEQ(r1, r2), Left(Sequ(v1, v2))) => Sequ(inj(r1, c, v1), v2) |
123 |
case (SEQ(r1, r2), Right(v2)) => Sequ(mkeps(r1), inj(r2, c, v2)) |
|
124 |
case (ALT(r1, r2), Left(v1)) => Left(inj(r1, c, v1)) |
|
125 |
case (ALT(r1, r2), Right(v2)) => Right(inj(r2, c, v2)) |
|
354
86b2aeae3e98
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
352
diff
changeset
|
126 |
case (CHAR(d), Empty) => Chr(c) |
579 | 127 |
case (RECD(x, r1), _) => Rec(x, inj(r1, c, v)) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
128 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
129 |
|
624 | 130 |
// the main lexing function (produces a value) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
131 |
def lex(r: Rexp, s: List[Char]) : Val = s match { |
579 | 132 |
case Nil => if (nullable(r)) mkeps(r) |
133 |
else throw new Exception("Not matched") |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
134 |
case c::cs => inj(r, c, lex(der(c, r), cs)) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
135 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
136 |
|
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
137 |
def lexing(r: Rexp, s: String) : Val = lex(r, s.toList) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
138 |
|
581 | 139 |
// a simple test for extracting an environment |
140 |
val re1 : Rexp = ("first" $ ("a" | "ab")) ~ ("second" $ ("b" | ONE)) |
|
141 |
env(lexing(re1, "ab")) |
|
450
d91a1748a9be
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
426
diff
changeset
|
142 |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
143 |
// some "rectification" functions for simplification |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
144 |
def F_ID(v: Val): Val = v |
579 | 145 |
def F_RIGHT(f: Val => Val) = (v:Val) => Right(f(v)) |
146 |
def F_LEFT(f: Val => Val) = (v:Val) => Left(f(v)) |
|
147 |
def F_ALT(f1: Val => Val, f2: Val => Val) = (v:Val) => v match { |
|
148 |
case Right(v) => Right(f2(v)) |
|
149 |
case Left(v) => Left(f1(v)) |
|
150 |
} |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
151 |
def F_SEQ(f1: Val => Val, f2: Val => Val) = (v:Val) => v match { |
520 | 152 |
case Sequ(v1, v2) => Sequ(f1(v1), f2(v2)) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
153 |
} |
354
86b2aeae3e98
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
352
diff
changeset
|
154 |
def F_SEQ_Empty1(f1: Val => Val, f2: Val => Val) = |
520 | 155 |
(v:Val) => Sequ(f1(Empty), f2(v)) |
354
86b2aeae3e98
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
352
diff
changeset
|
156 |
def F_SEQ_Empty2(f1: Val => Val, f2: Val => Val) = |
520 | 157 |
(v:Val) => Sequ(f1(v), f2(Empty)) |
579 | 158 |
def F_RECD(f: Val => Val) = (v:Val) => v match { |
159 |
case Rec(x, v) => Rec(x, f(v)) |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
160 |
} |
579 | 161 |
def F_ERROR(v: Val): Val = throw new Exception("error") |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
162 |
|
624 | 163 |
// simplification of regular expressions returns now also |
164 |
// an rectification function; no simplification under STAR |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
165 |
def simp(r: Rexp): (Rexp, Val => Val) = r match { |
579 | 166 |
case ALT(r1, r2) => { |
167 |
val (r1s, f1s) = simp(r1) |
|
168 |
val (r2s, f2s) = simp(r2) |
|
169 |
(r1s, r2s) match { |
|
170 |
case (ZERO, _) => (r2s, F_RIGHT(f2s)) |
|
171 |
case (_, ZERO) => (r1s, F_LEFT(f1s)) |
|
172 |
case _ => if (r1s == r2s) (r1s, F_LEFT(f1s)) |
|
173 |
else (ALT (r1s, r2s), F_ALT(f1s, f2s)) |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
174 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
175 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
176 |
case SEQ(r1, r2) => { |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
177 |
val (r1s, f1s) = simp(r1) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
178 |
val (r2s, f2s) = simp(r2) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
179 |
(r1s, r2s) match { |
426
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
180 |
case (ZERO, _) => (ZERO, F_ERROR) |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
181 |
case (_, ZERO) => (ZERO, F_ERROR) |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
182 |
case (ONE, _) => (r2s, F_SEQ_Empty1(f1s, f2s)) |
0debe6f41396
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
354
diff
changeset
|
183 |
case (_, ONE) => (r1s, F_SEQ_Empty2(f1s, f2s)) |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
184 |
case _ => (SEQ(r1s,r2s), F_SEQ(f1s, f2s)) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
185 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
186 |
} |
579 | 187 |
case RECD(x, r1) => { |
188 |
val (r1s, f1s) = simp(r1) |
|
189 |
(RECD(x, r1s), F_RECD(f1s)) |
|
190 |
} |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
191 |
case r => (r, F_ID) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
192 |
} |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
193 |
|
624 | 194 |
// lexing functions including simplification |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
195 |
def lex_simp(r: Rexp, s: List[Char]) : Val = s match { |
579 | 196 |
case Nil => if (nullable(r)) mkeps(r) else throw new Exception("Not matched") |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
197 |
case c::cs => { |
579 | 198 |
val (r_simp, f_simp) = simp(der(c, r)) |
199 |
inj(r, c, f_simp(lex_simp(r_simp, cs))) |
|
164
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
200 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
201 |
} |
6c1d214c39ef
added progs
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
diff
changeset
|
202 |
|
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
203 |
def lexing_simp(r: Rexp, s: String) : Val = lex_simp(r, s.toList) |
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
204 |
|
579 | 205 |
lexing_simp(("a" | "ab") ~ ("b" | ""), "ab") |
549 | 206 |
|
624 | 207 |
// The Lexing Rules for a Small While Language |
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
208 |
|
579 | 209 |
def PLUS(r: Rexp) = r ~ r.% |
549 | 210 |
|
579 | 211 |
val SYM = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z" |
212 |
val DIGIT = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" |
|
213 |
val ID = SYM ~ (SYM | DIGIT).% |
|
214 |
val NUM = PLUS(DIGIT) |
|
215 |
val KEYWORD : Rexp = "skip" | "while" | "do" | "if" | "then" | "else" | "read" | "write" | "true" | "false" |
|
216 |
val SEMI: Rexp = ";" |
|
217 |
val OP: Rexp = ":=" | "==" | "-" | "+" | "*" | "!=" | "<" | ">" | "<=" | ">=" | "%" | "/" |
|
218 |
val WHITESPACE = PLUS(" " | "\n" | "\t") |
|
219 |
val RPAREN: Rexp = ")" |
|
220 |
val LPAREN: Rexp = "(" |
|
221 |
val BEGIN: Rexp = "{" |
|
222 |
val END: Rexp = "}" |
|
223 |
val STRING: Rexp = "\"" ~ SYM.% ~ "\"" |
|
549 | 224 |
|
225 |
||
579 | 226 |
val WHILE_REGS = (("k" $ KEYWORD) | |
227 |
("i" $ ID) | |
|
228 |
("o" $ OP) | |
|
229 |
("n" $ NUM) | |
|
230 |
("s" $ SEMI) | |
|
231 |
("str" $ STRING) | |
|
232 |
("p" $ (LPAREN | RPAREN)) | |
|
233 |
("b" $ (BEGIN | END)) | |
|
234 |
("w" $ WHITESPACE)).% |
|
549 | 235 |
|
579 | 236 |
// Testing |
237 |
//============ |
|
549 | 238 |
|
579 | 239 |
def time[T](code: => T) = { |
240 |
val start = System.nanoTime() |
|
241 |
val result = code |
|
242 |
val end = System.nanoTime() |
|
243 |
println((end - start)/1.0e9) |
|
244 |
result |
|
245 |
} |
|
549 | 246 |
|
579 | 247 |
val r1 = ("a" | "ab") ~ ("bcd" | "c") |
248 |
println(lexing(r1, "abcd")) |
|
549 | 249 |
|
579 | 250 |
val r2 = ("" | "a") ~ ("ab" | "b") |
251 |
println(lexing(r2, "ab")) |
|
542 | 252 |
|
253 |
||
579 | 254 |
// Two Simple While Tests |
255 |
//======================== |
|
256 |
println("prog0 test") |
|
542 | 257 |
|
581 | 258 |
val prog0 = """read if""" |
579 | 259 |
println(env(lexing_simp(WHILE_REGS, prog0))) |
260 |
||
261 |
println("prog1 test") |
|
262 |
||
263 |
val prog1 = """read n; write (n)""" |
|
264 |
println(env(lexing_simp(WHILE_REGS, prog1))) |
|
265 |
||
266 |
||
267 |
// Bigger Test |
|
268 |
//============= |
|
269 |
||
270 |
val prog2 = """ |
|
271 |
write "fib"; |
|
272 |
read n; |
|
273 |
minus1 := 0; |
|
274 |
minus2 := 1; |
|
275 |
while n > 0 do { |
|
276 |
temp := minus2; |
|
277 |
minus2 := minus1 + minus2; |
|
278 |
minus1 := temp; |
|
279 |
n := n - 1 |
|
280 |
}; |
|
281 |
write "result"; |
|
282 |
write minus2 |
|
283 |
""" |
|
284 |
||
285 |
println("Tokens") |
|
286 |
println(env(lexing_simp(WHILE_REGS, prog2))) |
|
287 |
println(env(lexing_simp(WHILE_REGS, prog2)).filterNot{_._1 == "w"}.mkString("\n")) |
|
288 |
||
289 |
// some more timing tests with |
|
290 |
// i copies of the program |
|
291 |
||
624 | 292 |
for (i <- 0 to 20 by 10) { |
579 | 293 |
print(i.toString + ": ") |
294 |
time(lexing_simp(WHILE_REGS, prog2 * i)) |
|
542 | 295 |
} |
296 |
||
352
1e1b0fe66107
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
165
diff
changeset
|
297 |
|
617 | 298 |
val fib = """ |
299 |
write "Fib"; |
|
300 |
read n; |
|
301 |
minus1 := 0; |
|
302 |
minus2 := 1; |
|
303 |
while n > 0 do { |
|
304 |
temp := minus2; |
|
305 |
minus2 := minus1 + minus2; |
|
306 |
minus1 := temp; |
|
307 |
n := n - 1 |
|
308 |
}; |
|
309 |
write "Result"; |
|
310 |
write minus2 |
|
311 |
""" |
|
312 |
||
313 |
println(env(lexing_simp(WHILE_REGS, prog2)).filterNot{_._1 == "w"}) |