progs/lexer/token.sc
author Christian Urban <christian.urban@kcl.ac.uk>
Sat, 04 Sep 2021 14:08:00 +0100
changeset 833 7c3b8bb4a174
parent 827 47437f461fff
permissions -rw-r--r--
cwupdates
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
725
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     1
// This produces more meaningful tokens and
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     2
// also filters out comments and whitespaces
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     3
//
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     4
// call with
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     5
//
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     6
//  amm token.sc
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     7
//
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     8
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
     9
// load the lexer
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    10
import $file.lexer
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    11
import lexer._ 
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    12
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    13
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    14
// The tokens for the WHILE language
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    15
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    16
abstract class Token 
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    17
case object T_SEMI extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    18
case object T_LPAREN extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    19
case object T_RPAREN extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    20
case class T_ID(s: String) extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    21
case class T_OP(s: String) extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    22
case class T_NUM(n: Int) extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    23
case class T_KWD(s: String) extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    24
case class T_STR(s: String) extends Token
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    25
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    26
val token : PartialFunction[(String, String), Token] = {
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    27
  case ("s", _) => T_SEMI
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    28
  case ("p", "{") => T_LPAREN
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    29
  case ("p", "}") => T_RPAREN
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    30
  case ("i", s) => T_ID(s)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    31
  case ("o", s) => T_OP(s)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    32
  case ("n", s) => T_NUM(s.toInt)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    33
  case ("k", s) => T_KWD(s)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    34
  case ("str", s) => T_STR(s)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    35
}
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    36
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    37
// by using collect we filter out all unwanted tokens
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    38
def tokenise(s: String) : List[Token] = 
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    39
  lexing_simp(WHILE_REGS, s).collect(token)
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    40
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    41
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    42
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    43
827
47437f461fff updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    44
@arg(doc = "Tokens for fib and loops programs.")
725
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    45
@main
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    46
def main() = {
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    47
  println("Fib program")
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    48
  println(tokenise(prog2))
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    49
  println("Loops program")
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    50
  println(tokenise(prog3))
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    51
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    52
  for (i <- 0 to 20 by 5) {
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    53
    println(f"$i%2.0f: ${time(tokenise(prog3 * i))._2}")
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    54
  }
cd72ba78c287 updated
Christian Urban <christian.urban@kcl.ac.uk>
parents:
diff changeset
    55
729
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    56
}
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    57
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    58
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    59
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    60
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    61
// Primes program
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    62
//================
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    63
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    64
/*
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    65
end := 100;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    66
n := 2;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    67
while (n < end) do {
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    68
  f := 2;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    69
  tmp := 0;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    70
  while ((f < n / 2 + 1) && (tmp == 0)) do {
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    71
    if ((n / f) * f == n) then  { tmp := 1 } else { skip };
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    72
    f := f + 1
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    73
  };
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    74
  if (tmp == 0) then { write(n) } else { skip };
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    75
  n  := n + 1
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    76
}
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    77
*/
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    78
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    79
// Factors program
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    80
//=================
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    81
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    82
/*
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    83
write "Input n please";
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    84
read n;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    85
write "The factors of n are";
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    86
f := 2;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    87
while n != 1 do {
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    88
    while (n / f) * f == n do {
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    89
        write f;
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    90
        n := n / f
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    91
    };
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    92
    f := f + 1
81257df1591b updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 725
diff changeset
    93
}
827
47437f461fff updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    94
*/
47437f461fff updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    95
47437f461fff updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    96
47437f461fff updated
Christian Urban <christian.urban@kcl.ac.uk>
parents: 729
diff changeset
    97
// runs with amm2 and amm3