equal
deleted
inserted
replaced
1 // A tokeniser for the fun language |
1 // A tokeniser for the Fun language |
2 |
2 //================================== |
|
3 // |
|
4 // call with |
|
5 // |
|
6 // scala fun_tokens.scala fact.fun |
|
7 // |
|
8 // scala fun_tokens.scala defs.fun |
|
9 // |
|
10 // this will generate a .tks file that can be deserialised back |
|
11 // into a list of tokens |
|
12 // you can add -Xno-patmat-analysis in order to get rid of the |
|
13 // match-not-exhaustive warning |
3 |
14 |
4 object Fun_Tokens { |
15 object Fun_Tokens { |
5 |
16 |
6 import scala.language.implicitConversions |
17 import scala.language.implicitConversions |
7 import scala.language.reflectiveCalls |
18 import scala.language.reflectiveCalls |
198 val ALL = SYM | DIGIT | OP | " " | ":" | ";" | "\"" | "=" | "," | "(" | ")" |
209 val ALL = SYM | DIGIT | OP | " " | ":" | ";" | "\"" | "=" | "," | "(" | ")" |
199 val ALL2 = ALL | "\n" |
210 val ALL2 = ALL | "\n" |
200 val COMMENT = ("/*" ~ ALL2.% ~ "*/") | ("//" ~ ALL.% ~ "\n") |
211 val COMMENT = ("/*" ~ ALL2.% ~ "*/") | ("//" ~ ALL.% ~ "\n") |
201 |
212 |
202 |
213 |
203 val WHILE_REGS = (("k" $ KEYWORD) | |
214 val FUN_REGS = (("k" $ KEYWORD) | |
204 ("i" $ ID) | |
215 ("i" $ ID) | |
205 ("o" $ OP) | |
216 ("o" $ OP) | |
206 ("n" $ NUM) | |
217 ("n" $ NUM) | |
207 ("s" $ SEMI) | |
218 ("s" $ SEMI) | |
208 ("c" $ COMMA) | |
219 ("c" $ COMMA) | |
237 case ("pr", _) => T_RPAREN |
248 case ("pr", _) => T_RPAREN |
238 } |
249 } |
239 |
250 |
240 |
251 |
241 def tokenise(s: String) : List[Token] = |
252 def tokenise(s: String) : List[Token] = |
242 lexing_simp(WHILE_REGS, s).collect(token) |
253 lexing_simp(FUN_REGS, s).collect(token) |
243 |
254 |
244 def serialise[T](fname: String, data: T) = { |
255 def serialise[T](fname: String, data: T) = { |
245 val out = new ObjectOutputStream(new FileOutputStream(fname)) |
256 import scala.util.Using |
246 out.writeObject(data) |
257 Using(new ObjectOutputStream(new FileOutputStream(fname))) { |
247 out.close |
258 out => out.writeObject(data) |
248 } |
259 } |
249 |
260 } |
250 def main(args: Array[String]) = { |
261 |
|
262 def main(args: Array[String]) : Unit = { |
251 val fname = args(0) |
263 val fname = args(0) |
|
264 val tname = fname.stripSuffix(".fun") ++ ".tks" |
252 val file = io.Source.fromFile(fname).mkString |
265 val file = io.Source.fromFile(fname).mkString |
253 val tks = fname.stripSuffix(".fun") ++ ".tks" |
266 serialise(tname, tokenise(file)) |
254 serialise(tks, tokenise(file)) |
267 } |
255 } |
268 |
256 |
269 |
257 |
270 } |
258 } |
|