644
|
1 |
// A simple tokeniser based on the Sulzmann & Lu algorithm
|
|
2 |
//=========================================================
|
|
3 |
//
|
|
4 |
// call with
|
|
5 |
//
|
|
6 |
// scala tokenise.scala fib.while
|
|
7 |
//
|
|
8 |
// scala tokenise.scala loops.while
|
642
|
9 |
|
644
|
10 |
object Tokenise {
|
642
|
11 |
|
|
12 |
import scala.language.implicitConversions
|
|
13 |
import scala.language.reflectiveCalls
|
|
14 |
|
|
15 |
// regular expressions including records
|
|
16 |
abstract class Rexp
|
|
17 |
case object ZERO extends Rexp
|
|
18 |
case object ONE extends Rexp
|
|
19 |
case class CHAR(c: Char) extends Rexp
|
|
20 |
case class ALT(r1: Rexp, r2: Rexp) extends Rexp
|
|
21 |
case class SEQ(r1: Rexp, r2: Rexp) extends Rexp
|
|
22 |
case class STAR(r: Rexp) extends Rexp
|
|
23 |
case class RECD(x: String, r: Rexp) extends Rexp
|
|
24 |
|
|
25 |
// values
|
|
26 |
abstract class Val
|
|
27 |
case object Empty extends Val
|
|
28 |
case class Chr(c: Char) extends Val
|
|
29 |
case class Sequ(v1: Val, v2: Val) extends Val
|
|
30 |
case class Left(v: Val) extends Val
|
|
31 |
case class Right(v: Val) extends Val
|
|
32 |
case class Stars(vs: List[Val]) extends Val
|
|
33 |
case class Rec(x: String, v: Val) extends Val
|
|
34 |
|
|
35 |
// some convenience for typing in regular expressions
|
|
36 |
def charlist2rexp(s : List[Char]): Rexp = s match {
|
|
37 |
case Nil => ONE
|
|
38 |
case c::Nil => CHAR(c)
|
|
39 |
case c::s => SEQ(CHAR(c), charlist2rexp(s))
|
|
40 |
}
|
|
41 |
implicit def string2rexp(s : String) : Rexp =
|
|
42 |
charlist2rexp(s.toList)
|
|
43 |
|
|
44 |
implicit def RexpOps(r: Rexp) = new {
|
|
45 |
def | (s: Rexp) = ALT(r, s)
|
|
46 |
def % = STAR(r)
|
|
47 |
def ~ (s: Rexp) = SEQ(r, s)
|
|
48 |
}
|
|
49 |
|
|
50 |
implicit def stringOps(s: String) = new {
|
|
51 |
def | (r: Rexp) = ALT(s, r)
|
|
52 |
def | (r: String) = ALT(s, r)
|
|
53 |
def % = STAR(s)
|
|
54 |
def ~ (r: Rexp) = SEQ(s, r)
|
|
55 |
def ~ (r: String) = SEQ(s, r)
|
|
56 |
def $ (r: Rexp) = RECD(s, r)
|
|
57 |
}
|
|
58 |
|
|
59 |
def nullable(r: Rexp) : Boolean = r match {
|
|
60 |
case ZERO => false
|
|
61 |
case ONE => true
|
|
62 |
case CHAR(_) => false
|
|
63 |
case ALT(r1, r2) => nullable(r1) || nullable(r2)
|
|
64 |
case SEQ(r1, r2) => nullable(r1) && nullable(r2)
|
|
65 |
case STAR(_) => true
|
|
66 |
case RECD(_, r1) => nullable(r1)
|
|
67 |
}
|
|
68 |
|
|
69 |
def der(c: Char, r: Rexp) : Rexp = r match {
|
|
70 |
case ZERO => ZERO
|
|
71 |
case ONE => ZERO
|
|
72 |
case CHAR(d) => if (c == d) ONE else ZERO
|
|
73 |
case ALT(r1, r2) => ALT(der(c, r1), der(c, r2))
|
|
74 |
case SEQ(r1, r2) =>
|
|
75 |
if (nullable(r1)) ALT(SEQ(der(c, r1), r2), der(c, r2))
|
|
76 |
else SEQ(der(c, r1), r2)
|
|
77 |
case STAR(r) => SEQ(der(c, r), STAR(r))
|
|
78 |
case RECD(_, r1) => der(c, r1)
|
|
79 |
}
|
|
80 |
|
|
81 |
|
|
82 |
// extracts a string from value
|
|
83 |
def flatten(v: Val) : String = v match {
|
|
84 |
case Empty => ""
|
|
85 |
case Chr(c) => c.toString
|
|
86 |
case Left(v) => flatten(v)
|
|
87 |
case Right(v) => flatten(v)
|
|
88 |
case Sequ(v1, v2) => flatten(v1) + flatten(v2)
|
|
89 |
case Stars(vs) => vs.map(flatten).mkString
|
|
90 |
case Rec(_, v) => flatten(v)
|
|
91 |
}
|
|
92 |
|
|
93 |
|
|
94 |
// extracts an environment from a value;
|
|
95 |
// used for tokenise a string
|
|
96 |
def env(v: Val) : List[(String, String)] = v match {
|
|
97 |
case Empty => Nil
|
|
98 |
case Chr(c) => Nil
|
|
99 |
case Left(v) => env(v)
|
|
100 |
case Right(v) => env(v)
|
|
101 |
case Sequ(v1, v2) => env(v1) ::: env(v2)
|
|
102 |
case Stars(vs) => vs.flatMap(env)
|
|
103 |
case Rec(x, v) => (x, flatten(v))::env(v)
|
|
104 |
}
|
|
105 |
|
|
106 |
// The Injection Part of the lexer
|
|
107 |
|
|
108 |
def mkeps(r: Rexp) : Val = r match {
|
|
109 |
case ONE => Empty
|
|
110 |
case ALT(r1, r2) =>
|
|
111 |
if (nullable(r1)) Left(mkeps(r1)) else Right(mkeps(r2))
|
|
112 |
case SEQ(r1, r2) => Sequ(mkeps(r1), mkeps(r2))
|
|
113 |
case STAR(r) => Stars(Nil)
|
|
114 |
case RECD(x, r) => Rec(x, mkeps(r))
|
|
115 |
}
|
|
116 |
|
|
117 |
def inj(r: Rexp, c: Char, v: Val) : Val = (r, v) match {
|
|
118 |
case (STAR(r), Sequ(v1, Stars(vs))) => Stars(inj(r, c, v1)::vs)
|
|
119 |
case (SEQ(r1, r2), Sequ(v1, v2)) => Sequ(inj(r1, c, v1), v2)
|
|
120 |
case (SEQ(r1, r2), Left(Sequ(v1, v2))) => Sequ(inj(r1, c, v1), v2)
|
|
121 |
case (SEQ(r1, r2), Right(v2)) => Sequ(mkeps(r1), inj(r2, c, v2))
|
|
122 |
case (ALT(r1, r2), Left(v1)) => Left(inj(r1, c, v1))
|
|
123 |
case (ALT(r1, r2), Right(v2)) => Right(inj(r2, c, v2))
|
|
124 |
case (CHAR(d), Empty) => Chr(c)
|
|
125 |
case (RECD(x, r1), _) => Rec(x, inj(r1, c, v))
|
|
126 |
}
|
|
127 |
|
|
128 |
// some "rectification" functions for simplification
|
|
129 |
def F_ID(v: Val): Val = v
|
|
130 |
def F_RIGHT(f: Val => Val) = (v:Val) => Right(f(v))
|
|
131 |
def F_LEFT(f: Val => Val) = (v:Val) => Left(f(v))
|
|
132 |
def F_ALT(f1: Val => Val, f2: Val => Val) = (v:Val) => v match {
|
|
133 |
case Right(v) => Right(f2(v))
|
|
134 |
case Left(v) => Left(f1(v))
|
|
135 |
}
|
|
136 |
def F_SEQ(f1: Val => Val, f2: Val => Val) = (v:Val) => v match {
|
|
137 |
case Sequ(v1, v2) => Sequ(f1(v1), f2(v2))
|
|
138 |
}
|
|
139 |
def F_SEQ_Empty1(f1: Val => Val, f2: Val => Val) =
|
|
140 |
(v:Val) => Sequ(f1(Empty), f2(v))
|
|
141 |
def F_SEQ_Empty2(f1: Val => Val, f2: Val => Val) =
|
|
142 |
(v:Val) => Sequ(f1(v), f2(Empty))
|
|
143 |
def F_RECD(f: Val => Val) = (v:Val) => v match {
|
|
144 |
case Rec(x, v) => Rec(x, f(v))
|
|
145 |
}
|
|
146 |
def F_ERROR(v: Val): Val = throw new Exception("error")
|
|
147 |
|
|
148 |
def simp(r: Rexp): (Rexp, Val => Val) = r match {
|
|
149 |
case ALT(r1, r2) => {
|
|
150 |
val (r1s, f1s) = simp(r1)
|
|
151 |
val (r2s, f2s) = simp(r2)
|
|
152 |
(r1s, r2s) match {
|
|
153 |
case (ZERO, _) => (r2s, F_RIGHT(f2s))
|
|
154 |
case (_, ZERO) => (r1s, F_LEFT(f1s))
|
|
155 |
case _ => if (r1s == r2s) (r1s, F_LEFT(f1s))
|
|
156 |
else (ALT (r1s, r2s), F_ALT(f1s, f2s))
|
|
157 |
}
|
|
158 |
}
|
|
159 |
case SEQ(r1, r2) => {
|
|
160 |
val (r1s, f1s) = simp(r1)
|
|
161 |
val (r2s, f2s) = simp(r2)
|
|
162 |
(r1s, r2s) match {
|
|
163 |
case (ZERO, _) => (ZERO, F_ERROR)
|
|
164 |
case (_, ZERO) => (ZERO, F_ERROR)
|
|
165 |
case (ONE, _) => (r2s, F_SEQ_Empty1(f1s, f2s))
|
|
166 |
case (_, ONE) => (r1s, F_SEQ_Empty2(f1s, f2s))
|
|
167 |
case _ => (SEQ(r1s,r2s), F_SEQ(f1s, f2s))
|
|
168 |
}
|
|
169 |
}
|
|
170 |
case RECD(x, r1) => {
|
|
171 |
val (r1s, f1s) = simp(r1)
|
|
172 |
(RECD(x, r1s), F_RECD(f1s))
|
|
173 |
}
|
|
174 |
case r => (r, F_ID)
|
|
175 |
}
|
|
176 |
|
|
177 |
// lexing functions including simplification
|
|
178 |
def lex_simp(r: Rexp, s: List[Char]) : Val = s match {
|
|
179 |
case Nil => if (nullable(r)) mkeps(r) else
|
|
180 |
{ throw new Exception("lexing error") }
|
|
181 |
case c::cs => {
|
|
182 |
val (r_simp, f_simp) = simp(der(c, r))
|
|
183 |
inj(r, c, f_simp(lex_simp(r_simp, cs)))
|
|
184 |
}
|
|
185 |
}
|
|
186 |
|
|
187 |
def lexing_simp(r: Rexp, s: String) =
|
|
188 |
env(lex_simp(r, s.toList))
|
|
189 |
|
|
190 |
|
|
191 |
// The Lexing Rules for the Fun Language
|
|
192 |
|
|
193 |
def PLUS(r: Rexp) = r ~ r.%
|
|
194 |
|
|
195 |
def Range(s : List[Char]) : Rexp = s match {
|
|
196 |
case Nil => ZERO
|
|
197 |
case c::Nil => CHAR(c)
|
|
198 |
case c::s => ALT(CHAR(c), Range(s))
|
|
199 |
}
|
|
200 |
def RANGE(s: String) = Range(s.toList)
|
|
201 |
|
|
202 |
val SYM = RANGE("ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz_")
|
|
203 |
val DIGIT = RANGE("0123456789")
|
|
204 |
val ID = SYM ~ (SYM | DIGIT).%
|
|
205 |
val NUM = PLUS(DIGIT)
|
|
206 |
val KEYWORD : Rexp = "skip" | "while" | "do" | "if" | "then" | "else" | "read" | "write"
|
|
207 |
val SEMI: Rexp = ";"
|
|
208 |
val OP: Rexp = ":=" | "=" | "-" | "+" | "*" | "!=" | "<" | ">"
|
|
209 |
val WHITESPACE = PLUS(" " | "\n" | "\t")
|
|
210 |
val RPAREN: Rexp = "{"
|
|
211 |
val LPAREN: Rexp = "}"
|
|
212 |
val STRING: Rexp = "\"" ~ SYM.% ~ "\""
|
|
213 |
|
|
214 |
|
|
215 |
val WHILE_REGS = (("k" $ KEYWORD) |
|
|
216 |
("i" $ ID) |
|
|
217 |
("o" $ OP) |
|
|
218 |
("n" $ NUM) |
|
|
219 |
("s" $ SEMI) |
|
|
220 |
("str" $ STRING) |
|
|
221 |
("p" $ (LPAREN | RPAREN)) |
|
|
222 |
("w" $ WHITESPACE)).%
|
|
223 |
|
|
224 |
|
|
225 |
|
|
226 |
// Generating tokens for the WHILE language
|
644
|
227 |
// and serialising them into a .tks file
|
642
|
228 |
|
|
229 |
import java.io._
|
|
230 |
|
|
231 |
abstract class Token extends Serializable
|
|
232 |
case object T_SEMI extends Token
|
|
233 |
case object T_LPAREN extends Token
|
|
234 |
case object T_RPAREN extends Token
|
|
235 |
case class T_ID(s: String) extends Token
|
|
236 |
case class T_OP(s: String) extends Token
|
|
237 |
case class T_NUM(n: Int) extends Token
|
|
238 |
case class T_KWD(s: String) extends Token
|
|
239 |
case class T_STR(s: String) extends Token
|
|
240 |
|
|
241 |
val token : PartialFunction[(String, String), Token] = {
|
|
242 |
case ("s", _) => T_SEMI
|
|
243 |
case ("p", "{") => T_LPAREN
|
|
244 |
case ("p", "}") => T_RPAREN
|
|
245 |
case ("i", s) => T_ID(s)
|
|
246 |
case ("o", s) => T_OP(s)
|
|
247 |
case ("n", s) => T_NUM(s.toInt)
|
|
248 |
case ("k", s) => T_KWD(s)
|
|
249 |
case ("str", s) => T_STR(s)
|
|
250 |
}
|
|
251 |
|
|
252 |
def tokenise(s: String) : List[Token] =
|
|
253 |
lexing_simp(WHILE_REGS, s).collect(token)
|
|
254 |
|
|
255 |
|
|
256 |
def serialise[T](fname: String, data: T) = {
|
|
257 |
val out = new ObjectOutputStream(new FileOutputStream(fname))
|
|
258 |
out.writeObject(data)
|
|
259 |
out.close
|
|
260 |
}
|
|
261 |
|
|
262 |
def main(args: Array[String]) = {
|
644
|
263 |
val fname = args(0)
|
|
264 |
val file = io.Source.fromFile(fname).mkString
|
|
265 |
val tks = fname.stripSuffix(".while") ++ ".tks"
|
|
266 |
serialise(tks, tokenise(file))
|
642
|
267 |
}
|
|
268 |
|
|
269 |
|
|
270 |
} |