progs/lexer/token.sc
author Christian Urban <christian.urban@kcl.ac.uk>
Sun, 04 Oct 2020 22:20:25 +0100
changeset 772 3bf3f5bb067e
parent 729 b147a10be8dd
child 827 67c8a6e6a305
permissions -rw-r--r--
update
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
725
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     1
// This produces more meaningful tokens and
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     2
// also filters out comments and whitespaces
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     3
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     4
// call with
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     5
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     6
//  amm token.sc
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     7
//
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     8
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     9
// load the lexer
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    10
import $file.lexer
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    11
import lexer._ 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    12
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    13
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    14
// The tokens for the WHILE language
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    15
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    16
abstract class Token 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    17
case object T_SEMI extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    18
case object T_LPAREN extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    19
case object T_RPAREN extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    20
case class T_ID(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    21
case class T_OP(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    22
case class T_NUM(n: Int) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    23
case class T_KWD(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    24
case class T_STR(s: String) extends Token
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    25
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    26
val token : PartialFunction[(String, String), Token] = {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    27
  case ("s", _) => T_SEMI
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    28
  case ("p", "{") => T_LPAREN
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    29
  case ("p", "}") => T_RPAREN
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    30
  case ("i", s) => T_ID(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    31
  case ("o", s) => T_OP(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    32
  case ("n", s) => T_NUM(s.toInt)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    33
  case ("k", s) => T_KWD(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    34
  case ("str", s) => T_STR(s)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    35
}
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    36
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    37
// by using collect we filter out all unwanted tokens
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    38
def tokenise(s: String) : List[Token] = 
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    39
  lexing_simp(WHILE_REGS, s).collect(token)
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    40
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    41
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    42
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    43
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    44
@doc("Tokens for fib and loops programs.")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    45
@main
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    46
def main() = {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    47
  println("Fib program")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    48
  println(tokenise(prog2))
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    49
  println("Loops program")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    50
  println(tokenise(prog3))
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    51
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    52
  for (i <- 0 to 20 by 5) {
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    53
    println(f"$i%2.0f: ${time(tokenise(prog3 * i))._2}")
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    54
  }
f345e89895f5 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    55
729
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    56
}
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    57
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    58
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    59
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    60
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    61
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    62
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    63
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    64
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    65
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    66
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    67
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    68
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    69
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    70
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    71
// Primes program
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    72
//================
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    73
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    74
/*
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    75
end := 100;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    76
n := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    77
while (n < end) do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    78
  f := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    79
  tmp := 0;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    80
  while ((f < n / 2 + 1) && (tmp == 0)) do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    81
    if ((n / f) * f == n) then  { tmp := 1 } else { skip };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    82
    f := f + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    83
  };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    84
  if (tmp == 0) then { write(n) } else { skip };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    85
  n  := n + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    86
}
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    87
*/
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    88
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    89
// Factors program
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    90
//=================
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    91
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    92
/*
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    93
write "Input n please";
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    94
read n;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    95
write "The factors of n are";
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    96
f := 2;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    97
while n != 1 do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    98
    while (n / f) * f == n do {
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    99
        write f;
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
   100
        n := n / f
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
   101
    };
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
   102
    f := f + 1
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
   103
}
b147a10be8dd updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
   104
*/