903
+ − 1
// Author: Zhuo Ying Jiang Li
+ − 2
// Starting code by Dr Christian Urban
+ − 3
+ − 4
// lexer
+ − 5
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 6
//
903
+ − 7
// Use this command to print the list of tokens:
+ − 8
// amm fun_token.sc <name>.fun
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 9
//
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 10
903
+ − 11
type Token = (String, String)
+ − 12
type Tokens = List[Token]
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 13
903
+ − 14
// regular expressions including records
+ − 15
abstract class Rexp
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 16
case object ZERO extends Rexp
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 17
case object ONE extends Rexp
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 18
case class CHAR(c: Char) extends Rexp
903
+ − 19
case class RANGE(chars: List[Char]) extends Rexp
+ − 20
case class ALT(r1: Rexp, r2: Rexp) extends Rexp
+ − 21
case class SEQ(r1: Rexp, r2: Rexp) extends Rexp
+ − 22
case class STAR(r: Rexp) extends Rexp
+ − 23
case class OPTIONAL(r: Rexp) extends Rexp
+ − 24
case class PLUS(r: Rexp) extends Rexp
+ − 25
case class NTIMES(r: Rexp, n: Int) extends Rexp
+ − 26
case class RECD(x: String, r: Rexp) extends Rexp // records for extracting strings or tokens
+ − 27
+ − 28
// values
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 29
abstract class Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 30
case object Empty extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 31
case class Chr(c: Char) extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 32
case class Sequ(v1: Val, v2: Val) extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 33
case class Left(v: Val) extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 34
case class Right(v: Val) extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 35
case class Stars(vs: List[Val]) extends Val
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 36
case class Rec(x: String, v: Val) extends Val
903
+ − 37
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 38
// some convenience for typing in regular expressions
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 39
def charlist2rexp(s : List[Char]): Rexp = s match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 40
case Nil => ONE
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 41
case c::Nil => CHAR(c)
903
+ − 42
case c::vs => SEQ(CHAR(c), charlist2rexp(vs))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 43
}
903
+ − 44
920
+ − 45
implicit def string2rexp(s : String) : Rexp =
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 46
charlist2rexp(s.toList)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 47
920
+ − 48
extension (r: Rexp) {
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 49
def | (s: Rexp) = ALT(r, s)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 50
def % = STAR(r)
903
+ − 51
def + = PLUS(r)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 52
def ~ (s: Rexp) = SEQ(r, s)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 53
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 54
920
+ − 55
extension (s: String) {
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 56
def | (r: Rexp) = ALT(s, r)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 57
def | (r: String) = ALT(s, r)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 58
def % = STAR(s)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 59
def ~ (r: Rexp) = SEQ(s, r)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 60
def ~ (r: String) = SEQ(s, r)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 61
def $ (r: Rexp) = RECD(s, r)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 62
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 63
920
+ − 64
903
+ − 65
def nullable(r: Rexp) : Boolean = r match {
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 66
case ZERO => false
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 67
case ONE => true
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 68
case CHAR(_) => false
903
+ − 69
case RANGE(_) => false
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 70
case ALT(r1, r2) => nullable(r1) || nullable(r2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 71
case SEQ(r1, r2) => nullable(r1) && nullable(r2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 72
case STAR(_) => true
903
+ − 73
case OPTIONAL(r1) => true
+ − 74
case PLUS(r1) => nullable(r1)
+ − 75
case NTIMES(r1, n) => if (n == 0) true else nullable(r1)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 76
case RECD(_, r1) => nullable(r1)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 77
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 78
903
+ − 79
def der(c: Char, r: Rexp) : Rexp = r match {
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 80
case ZERO => ZERO
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 81
case ONE => ZERO
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 82
case CHAR(d) => if (c == d) ONE else ZERO
903
+ − 83
case RANGE(chars) => if (chars.contains(c)) ONE else ZERO
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 84
case ALT(r1, r2) => ALT(der(c, r1), der(c, r2))
903
+ − 85
case SEQ(r1, r2) =>
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 86
if (nullable(r1)) ALT(SEQ(der(c, r1), r2), der(c, r2))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 87
else SEQ(der(c, r1), r2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 88
case STAR(r) => SEQ(der(c, r), STAR(r))
903
+ − 89
case OPTIONAL(r) => der(c, r)
+ − 90
case PLUS(r) => SEQ(der(c, r), STAR(r))
+ − 91
case NTIMES(r1, n) => if (n == 0) ZERO else SEQ(der(c, r1), NTIMES(r1, n - 1))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 92
case RECD(_, r1) => der(c, r1)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 93
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 94
903
+ − 95
// extracts a string from a value
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 96
def flatten(v: Val) : String = v match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 97
case Empty => ""
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 98
case Chr(c) => c.toString
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 99
case Left(v) => flatten(v)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 100
case Right(v) => flatten(v)
903
+ − 101
case Sequ(v1, v2) => flatten(v1) ++ flatten(v2)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 102
case Stars(vs) => vs.map(flatten).mkString
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 103
case Rec(_, v) => flatten(v)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 104
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 105
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 106
// extracts an environment from a value;
903
+ − 107
// used for tokenising a string
+ − 108
def env(v: Val) : Tokens = v match {
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 109
case Empty => Nil
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 110
case Chr(c) => Nil
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 111
case Left(v) => env(v)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 112
case Right(v) => env(v)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 113
case Sequ(v1, v2) => env(v1) ::: env(v2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 114
case Stars(vs) => vs.flatMap(env)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 115
case Rec(x, v) => (x, flatten(v))::env(v)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 116
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 117
903
+ − 118
+ − 119
// The injection and mkeps part of the lexer
+ − 120
//===========================================
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 121
920
+ − 122
// Mkeps
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 123
def mkeps(r: Rexp) : Val = r match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 124
case ONE => Empty
920
+ − 125
case ALT(r1, r2) =>
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 126
if (nullable(r1)) Left(mkeps(r1)) else Right(mkeps(r2))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 127
case SEQ(r1, r2) => Sequ(mkeps(r1), mkeps(r2))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 128
case STAR(r) => Stars(Nil)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 129
case RECD(x, r) => Rec(x, mkeps(r))
920
+ − 130
+ − 131
case PLUS(r) => Stars(List(mkeps(r))) // the first copy must match the empty string
+ − 132
case OPTIONAL(r) => if (nullable(r)) Stars(List(mkeps(r))) else Stars(Nil)
+ − 133
case NTIMES(r, i) => Stars(List.fill(i)(mkeps(r)))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 134
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 135
920
+ − 136
// Inj
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 137
def inj(r: Rexp, c: Char, v: Val) : Val = (r, v) match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 138
case (STAR(r), Sequ(v1, Stars(vs))) => Stars(inj(r, c, v1)::vs)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 139
case (SEQ(r1, r2), Sequ(v1, v2)) => Sequ(inj(r1, c, v1), v2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 140
case (SEQ(r1, r2), Left(Sequ(v1, v2))) => Sequ(inj(r1, c, v1), v2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 141
case (SEQ(r1, r2), Right(v2)) => Sequ(mkeps(r1), inj(r2, c, v2))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 142
case (ALT(r1, r2), Left(v1)) => Left(inj(r1, c, v1))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 143
case (ALT(r1, r2), Right(v2)) => Right(inj(r2, c, v2))
920
+ − 144
case (CHAR(d), Empty) => Chr(c)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 145
case (RECD(x, r1), _) => Rec(x, inj(r1, c, v))
920
+ − 146
+ − 147
case (RANGE(_), Empty) => Chr(c)
+ − 148
case (PLUS(r), Sequ(v1, Stars(vs))) => Stars(inj(r, c, v1)::vs)
+ − 149
case (OPTIONAL(r), v1) => Stars(List(inj(r, c, v1)))
+ − 150
case (NTIMES(r, n), Sequ(v1, Stars(vs))) => Stars(inj(r, c, v1)::vs)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 151
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 152
920
+ − 153
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 154
// some "rectification" functions for simplification
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 155
def F_ID(v: Val): Val = v
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 156
def F_RIGHT(f: Val => Val) = (v:Val) => Right(f(v))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 157
def F_LEFT(f: Val => Val) = (v:Val) => Left(f(v))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 158
def F_ALT(f1: Val => Val, f2: Val => Val) = (v:Val) => v match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 159
case Right(v) => Right(f2(v))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 160
case Left(v) => Left(f1(v))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 161
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 162
def F_SEQ(f1: Val => Val, f2: Val => Val) = (v:Val) => v match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 163
case Sequ(v1, v2) => Sequ(f1(v1), f2(v2))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 164
}
903
+ − 165
def F_SEQ_Empty1(f1: Val => Val, f2: Val => Val) =
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 166
(v:Val) => Sequ(f1(Empty), f2(v))
903
+ − 167
def F_SEQ_Empty2(f1: Val => Val, f2: Val => Val) =
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 168
(v:Val) => Sequ(f1(v), f2(Empty))
903
+ − 169
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 170
def F_ERROR(v: Val): Val = throw new Exception("error")
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 171
903
+ − 172
// simplification
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 173
def simp(r: Rexp): (Rexp, Val => Val) = r match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 174
case ALT(r1, r2) => {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 175
val (r1s, f1s) = simp(r1)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 176
val (r2s, f2s) = simp(r2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 177
(r1s, r2s) match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 178
case (ZERO, _) => (r2s, F_RIGHT(f2s))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 179
case (_, ZERO) => (r1s, F_LEFT(f1s))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 180
case _ => if (r1s == r2s) (r1s, F_LEFT(f1s))
903
+ − 181
else (ALT (r1s, r2s), F_ALT(f1s, f2s))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 182
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 183
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 184
case SEQ(r1, r2) => {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 185
val (r1s, f1s) = simp(r1)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 186
val (r2s, f2s) = simp(r2)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 187
(r1s, r2s) match {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 188
case (ZERO, _) => (ZERO, F_ERROR)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 189
case (_, ZERO) => (ZERO, F_ERROR)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 190
case (ONE, _) => (r2s, F_SEQ_Empty1(f1s, f2s))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 191
case (_, ONE) => (r1s, F_SEQ_Empty2(f1s, f2s))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 192
case _ => (SEQ(r1s,r2s), F_SEQ(f1s, f2s))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 193
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 194
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 195
case r => (r, F_ID)
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 196
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 197
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 198
// lexing functions including simplification
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 199
def lex_simp(r: Rexp, s: List[Char]) : Val = s match {
903
+ − 200
case Nil => if (nullable(r)) mkeps(r) else
+ − 201
{ throw new Exception("lexing error") }
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 202
case c::cs => {
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 203
val (r_simp, f_simp) = simp(der(c, r))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 204
inj(r, c, f_simp(lex_simp(r_simp, cs)))
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 205
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 206
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 207
903
+ − 208
def lexing_simp(r: Rexp, s: String) =
+ − 209
env(lex_simp(r, s.toList))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 210
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 211
903
+ − 212
// FUN language lexer
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 213
903
+ − 214
val DIGIT = RANGE("0123456789".toList)
+ − 215
val LOWERCASE = RANGE("abcdefghijklmnopqrstuvwxyz".toList)
+ − 216
val UPPERCASE = RANGE("ABCDEFGHIJKLMNOPQRSTUVWXYZ".toList)
+ − 217
val SYM = RANGE("!\"#$%&'()*+,-./:;<>=?`@[]\\^_{}|~".toList) // I referenced the CPP ASCII table https://en.cppreference.com/w/cpp/language/ascii
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 218
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 219
903
+ − 220
val KEYWORD : Rexp = "val" | "if" | "then" | "else" | "def" | "skip" // "skip" is hardcoded because hanoi.fun calls skip() without parentheses
+ − 221
val TYPE : Rexp = "Int" | "Double" | "Void"
+ − 222
val GLOBAL_ID : Rexp = UPPERCASE ~ ("_" | LOWERCASE | DIGIT | UPPERCASE).% // start with capital letter and followed by any case
+ − 223
val ID : Rexp = LOWERCASE ~ ("_" | UPPERCASE | LOWERCASE | DIGIT).% // start with lowercase
+ − 224
val SEMI : Rexp = ";"
+ − 225
val COLON : Rexp = ":"
+ − 226
val OP : Rexp = "=" | "==" | "-" | "+" | "*" | "!=" | "<" | ">" | "<=" | ">=" | "%" | "/" // no && and || operators
+ − 227
val INT : Rexp = DIGIT.+
+ − 228
val DOUBLE : Rexp = DIGIT.+ ~ "." ~ DIGIT.+ // negative numbers sign is lexed as operator, but the parser will identify negative numbers
+ − 229
val COMMA : Rexp = ","
+ − 230
val WHITESPACES: Rexp = (" " | "\n" | "\t" | "\r").+ // whitespaces are either " " or \n or \t or \r
+ − 231
val LPAREN : Rexp = RANGE("({".toList)
+ − 232
val RPAREN : Rexp = RANGE(")}".toList)
+ − 233
val CH : Rexp = "'" ~ (LOWERCASE | UPPERCASE | DIGIT | SYM | " " | "\\n" | "\\t" | "\\r") ~ "'" // \n, \t and \r should also be tokenized, any character should be, whitespaces too
+ − 234
val COMMENT : Rexp = ("//" ~ (LOWERCASE | UPPERCASE | SYM | DIGIT | RANGE(" \t\r".toList)).% ~ "\n") | ("/*" ~ (LOWERCASE | UPPERCASE | SYM | DIGIT | RANGE(" \n\t\r".toList)).% ~ "*/")
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 235
903
+ − 236
val FUN_REGS = (("keyword" $ KEYWORD) |
+ − 237
("type" $ TYPE) |
+ − 238
("global" $ GLOBAL_ID) |
+ − 239
("id" $ ID) |
+ − 240
("op" $ OP) |
+ − 241
("double" $ DOUBLE) |
+ − 242
("int" $ INT) |
+ − 243
("semi" $ SEMI) |
+ − 244
("colon" $ COLON) |
+ − 245
("comma" $ COMMA) |
+ − 246
("ch" $ CH) |
+ − 247
("par" $ (LPAREN | RPAREN)) |
+ − 248
COMMENT | WHITESPACES).%
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 249
903
+ − 250
def fun_lex(program: String) : Tokens = {
+ − 251
lexing_simp(FUN_REGS, program)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 252
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 253
903
+ − 254
def tokenise(program: String) : Tokens = {
+ − 255
lexing_simp(FUN_REGS, program)
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 256
}
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 257
903
+ − 258
import scala.io.Source._
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 259
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 260
@main
903
+ − 261
def lex(filename: String) = {
+ − 262
// read file
+ − 263
val fun_code = fromFile(filename).getLines.mkString("\n")
+ − 264
// print tokens to screen
+ − 265
println(fun_lex(fun_code).mkString("\n"))
864
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff
changeset
+ − 266
}