progs/lexer/token.sc
author Christian Urban <christian.urban@kcl.ac.uk>
Thu, 09 Dec 2021 00:07:51 +0000
changeset 862 3ea5ba4bc3b4
parent 827 67c8a6e6a305
permissions -rw-r--r--
updated
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
725
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     1
// This produces more meaningful tokens and
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     2
// also filters out comments and whitespaces
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     3
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     4
// call with
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     5
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     6
//  amm token.sc
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     7
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     8
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     9
// load the lexer
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    10
import $file.lexer
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    11
import lexer._ 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    12
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    13
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    14
// The tokens for the WHILE language
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    15
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    16
abstract class Token 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    17
case object T_SEMI extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    18
case object T_LPAREN extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    19
case object T_RPAREN extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    20
case class T_ID(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    21
case class T_OP(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    22
case class T_NUM(n: Int) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    23
case class T_KWD(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    24
case class T_STR(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    25
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    26
val token : PartialFunction[(String, String), Token] = {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    27
  case ("s", _) => T_SEMI
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    28
  case ("p", "{") => T_LPAREN
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    29
  case ("p", "}") => T_RPAREN
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    30
  case ("i", s) => T_ID(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    31
  case ("o", s) => T_OP(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    32
  case ("n", s) => T_NUM(s.toInt)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    33
  case ("k", s) => T_KWD(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    34
  case ("str", s) => T_STR(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    35
}
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    36
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    37
// by using collect we filter out all unwanted tokens
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    38
def tokenise(s: String) : List[Token] = 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    39
  lexing_simp(WHILE_REGS, s).collect(token)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    40
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    41
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    42
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    43
827
67c8a6e6a305 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    44
@arg(doc = "Tokens for fib and loops programs.")
725
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    45
@main
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    46
def main() = {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    47
  println("Fib program")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    48
  println(tokenise(prog2))
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    49
  println("Loops program")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    50
  println(tokenise(prog3))
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    51
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    52
  for (i <- 0 to 20 by 5) {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    53
    println(f"$i%2.0f: ${time(tokenise(prog3 * i))._2}")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    54
  }
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    55
729
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    56
}
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    57
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    58
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    59
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    60
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    61
// Primes program
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    62
//================
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    63
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    64
/*
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    65
end := 100;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    66
n := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    67
while (n < end) do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    68
  f := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    69
  tmp := 0;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    70
  while ((f < n / 2 + 1) && (tmp == 0)) do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    71
    if ((n / f) * f == n) then  { tmp := 1 } else { skip };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    72
    f := f + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    73
  };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    74
  if (tmp == 0) then { write(n) } else { skip };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    75
  n  := n + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    76
}
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    77
*/
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    78
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    79
// Factors program
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    80
//=================
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    81
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    82
/*
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    83
write "Input n please";
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    84
read n;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    85
write "The factors of n are";
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    86
f := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    87
while n != 1 do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    88
    while (n / f) * f == n do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    89
        write f;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    90
        n := n / f
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    91
    };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    92
    f := f + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    93
}
827
67c8a6e6a305 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    94
*/
67c8a6e6a305 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    95
67c8a6e6a305 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    96
67c8a6e6a305 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    97
// runs with amm2 and amm3