4 // - Higher-Order functions

     4 // last week:

     5 // - maps (behind for-comprehensions)

     5 // higher-order functions

     6 

     6 // maps

     7 

     8 // - recursion

     9 // - Sudoku

    10 // - string interpolations

     9 def fib(n: Int) : Int = n match {

    13 // A Recursive Web Crawler / Email Harvester

    10   case 0 => 1

    14 //===========================================

    11   case 1 =>  1

    15 //

    12   case n => fib(n - 1) + fib(n - 2)

    16 // the idea is to look for links using the

    13 }

    17 // regular expression "https?://[^"]*" and for

    14 

    18 // email addresses using another regex.

    15 

    19 

    16 abstract class Rexp

    20 import io.Source

    17 case object ZERO extends Rexp                      // matches nothing

    21 import scala.util._

    18 case object ONE extends Rexp                       // matches the empty string

    22 

    19 case class CHAR(c: Char) extends Rexp              // matches a character c

    23 // gets the first 10K of a web-page

    20 case class ALT(r1: Rexp, r2: Rexp) extends Rexp    // alternative

    24 def get_page(url: String) : String = {

    21 case class SEQ(r1: Rexp, r2: Rexp) extends Rexp    // sequence

    25   Try(Source.fromURL(url)("ISO-8859-1").take(10000).mkString).

    22 case class STAR(r: Rexp) extends Rexp              // star

    26     getOrElse { println(s"  Problem with: $url"); ""}

    23 

    27 }

    24 def depth(r: Rexp) : Int = r match {

    28 

    25   case ZERO => 1

    29 // regex for URLs and emails

    26   case ONE => 1

    30 val http_pattern = """"https?://[^"]*"""".r

    27   case CHAR(_) => 1

    31 val email_pattern = """([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})""".r

    28   case ALT(r1, r2) => 1 + List(depth(r1), depth(r2)).max

    32 

    29   case SEQ(r1, r2) => 1 + List(depth(r1), depth(r2)).max

    33 //test case:

    30   case STAR(r1) => 1 + depth(r1)

    34 //email_pattern.findAllIn

    31 }

    35 //  ("foo bla christian@kcl.ac.uk 1234567").toList

    32 

    36 

    33 

    37 

    34 // - String-Interpolations

    38 // drops the first and last character from a string

    39 def unquote(s: String) = s.drop(1).dropRight(1)

    40 

    41 def get_all_URLs(page: String): Set[String] = 

    42   http_pattern.findAllIn(page).map(unquote).toSet

    43 

    44 // naive version of crawl - searches until a given depth,

    45 // visits pages potentially more than once

    46 def crawl(url: String, n: Int) : Unit = {

    47   if (n == 0) ()

    48   else {

    49     println(s"  Visiting: $n $url")

    50     for (u <- get_all_URLs(get_page(url))) crawl(u, n - 1)

    51   }

    52 }

    53 

    54 // some starting URLs for the crawler

    55 val startURL = """https://nms.kcl.ac.uk/christian.urban/"""

    56 

    57 crawl(startURL, 2)

    58 

    59 

    60 // a primitive email harvester

    61 def emails(url: String, n: Int) : Set[String] = {

    62   if (n == 0) Set()

    63   else {

    64     println(s"  Visiting: $n $url")

    65     val page = get_page(url)

    66     val new_emails = email_pattern.findAllIn(page).toSet

    67     new_emails ++ (for (u <- get_all_URLs(page)) yield emails(u, n - 1)).flatten

    68   }

    69 }

    70 

    71 emails(startURL, 2)

    72 

    73 

    74 

    75 // Sudoku 

    76 //========

    77 

    78 // THE POINT OF THIS CODE IS NOT TO BE SUPER

    79 // EFFICIENT AND FAST, just explaining exhaustive

    80 // depth-first search

    81 

    82 

    83 val game0 = """.14.6.3..

    84               |62...4..9

    85               |.8..5.6..

    86               |.6.2....3

    87               |.7..1..5.

    88               |5....9.6.

    89               |..6.2..3.

    90               |1..5...92

    91               |..7.9.41.""".stripMargin.replaceAll("\\n", "")

    92 

    93 type Pos = (Int, Int)

    94 val EmptyValue = '.'

    95 val MaxValue = 9

    96 

    97 def pretty(game: String): String = 

    98   "\n" + (game.grouped(MaxValue).mkString("\n"))

    99 

   100 pretty(game0)

   101 

   102 

   103 val allValues = "123456789".toList

   104 val indexes = (0 to 8).toList

   105 

   106 def empty(game: String) = game.indexOf(EmptyValue)

   107 def isDone(game: String) = empty(game) == -1 

   108 def emptyPosition(game: String) : Pos = {

   109   val e = empty(game)

   110   (e % MaxValue, e / MaxValue)

   111 }

   112 

   113 def get_row(game: String, y: Int) = 

   114   indexes.map(col => game(y * MaxValue + col))

   115 def get_col(game: String, x: Int) = 

   116   indexes.map(row => game(x + row * MaxValue))

   117 

   118 //get_row(game0, 0)

   119 //get_row(game0, 1)

   120 //get_col(game0, 0)

   121 

   122 def get_box(game: String, pos: Pos): List[Char] = {

   123     def base(p: Int): Int = (p / 3) * 3

   124     val x0 = base(pos._1)

   125     val y0 = base(pos._2)

   126     val ys = (y0 until y0 + 3).toList

   127     (x0 until x0 + 3).toList

   128       .flatMap(x => ys.map(y => game(x + y * MaxValue)))

   129 }

   130 

   131 

   132 //get_box(game0, (3, 1))

   133 

   134 

   135 // this is not mutable!!

   136 def update(game: String, pos: Int, value: Char): String = 

   137   game.updated(pos, value)

   138 

   139 def toAvoid(game: String, pos: Pos): List[Char] = 

   140   (get_col(game, pos._1) ++ 

   141    get_row(game, pos._2) ++ 

   142    get_box(game, pos))

   143 

   144 def candidates(game: String, pos: Pos): List[Char] = 

   145   allValues.diff(toAvoid(game, pos))

   146 

   147 //candidates(game0, (0,0))

   148 

   149 

   150 def search(game: String): List[String] = {

   151   if (isDone(game)) List(game)

   152   else {

   153     val cs = candidates(game, emptyPosition(game))

   154     cs.par.map(c => search(update(game, empty(game), c))).flatten.toList

   155   }

   156 }

   157 

   158 pretty(game0)

   159 search(game0).map(pretty)

   160 

   161 val game1 = """23.915...

   162               |...2..54.

   163               |6.7......

   164               |..1.....9

   165               |89.5.3.17

   166               |5.....6..

   167               |......9.5

   168               |.16..7...

   169               |...329..1""".stripMargin.replaceAll("\\n", "")

   170 

   171 search(game1).map(pretty)

   172 

   173 // a game that is in the hard category

   174 val game2 = """8........

   175               |..36.....

   176               |.7..9.2..

   177               |.5...7...

   178               |....457..

   179               |...1...3.

   180               |..1....68

   181               |..85...1.

   182               |.9....4..""".stripMargin.replaceAll("\\n", "")

   183 

   184 search(game2).map(pretty)

   185 

   186 // game with multiple solutions

   187 val game3 = """.8...9743

   188               |.5...8.1.

   189               |.1.......

   190               |8....5...

   191               |...8.4...

   192               |...3....6

   193               |.......7.

   194               |.3.5...8.

   195               |9724...5.""".stripMargin.replaceAll("\\n", "")

   196 

   197 search(game3).map(pretty).foreach(println)

   198 

   199 // for measuring time

   200 def time_needed[T](i: Int, code: => T) = {

   201   val start = System.nanoTime()

   202   for (j <- 1 to i) code

   203   val end = System.nanoTime()

   204   s"${(end - start) / 1.0e9} secs"

   205 }

   206 

   207 time_needed(2, search(game2))

   208 

   209 

   210 // concurrency 

   211 // scala-cli --extra-jars scala-parallel-collections_3-1.0.4.jar 

   212 // import scala.collection.parallel.CollectionConverters._

   213 

   214 

   215 

   104 // User-defined Datatypes

   272 // Pattern Matching

   105 //========================

   273 //==================

   274 

   275 // A powerful tool which has even landed in Java during 

   276 // the last few years (https://inside.java/2021/06/13/podcast-017/).

   277 // ...Scala already has it for many years and the concept is

   278 // older than your friendly lecturer, that is stone old  ;o)

   279 

   280 // The general schema:

   281 //

   282 //    expression match {

   283 //       case pattern1 => expression1

   284 //       case pattern2 => expression2

   285 //       ...

   286 //       case patternN => expressionN

   287 //    }

   288 

   289 

   290 // recall

   291 def len(xs: List[Int]) : Int = {

   292     if (xs == Nil) 0

   293     else 1 + len(xs.tail)

   294 }    

   295 

   296 def len(xs: List[Int]) : Int = xs match {

   297     case Nil => 0

   298     case hd::tail => 1 + len(tail)

   299 }  

   300 

   301 

   302 def my_map_int(lst: List[Int], f: Int => Int) : List[Int] = 

   303   lst match {

   304     case Nil => Nil

   305     case x::xs => f(x)::my_map_int(xs, f)

   306   }

   307 

   308 def my_map_option(opt: Option[Int], f: Int => Int) : Option[Int] = 

   309   opt match {

   310     case None => None

   311     case Some(x) => Some(f(x))

   312   }

   313 

   314 my_map_option(None, x => x * x)

   315 my_map_option(Some(8), x => x * x)

   316 

   317 

   318 // you can also have cases combined

   319 def season(month: String) : String = month match {

   320   case "March" | "April" | "May" => "It's spring"

   321   case "June" | "July" | "August" => "It's summer"

   322   case "September" | "October" | "November" => "It's autumn"

   323   case "December" => "It's winter"

   324   case "January" | "February" => "It's unfortunately winter"

   325   case _ => "Wrong month"

   326 }

   327 

   328 // pattern-match on integers

   329 

   330 def fib(n: Int) : Int = n match { 

   331   case 0 | 1 => 1

   332   case n => fib(n - 1) + fib(n - 2)

   333 }

   334 

   335 fib(10)

   336 

   337 // pattern-match on results

   338 

   339 // Silly: fizz buzz

   340 def fizz_buzz(n: Int) : String = (n % 3, n % 5) match {

   341   case (0, 0) => "fizz buzz"

   342   case (0, _) => "fizz"

   343   case (_, 0) => "buzz"

   344   case _ => n.toString  

   345 }

   346 

   347 for (n <- 1 to 20) 

   348  println(fizz_buzz(n))

   349 

   350 // guards in pattern-matching

   351 

   352 def foo(xs: List[Int]) : String = xs match {

   353   case Nil => s"this list is empty"

   354   case x :: xs if x % 2 == 0 

   355      => s"the first elemnt is even"

   356   case x :: y :: rest if x == y

   357      => s"this has two elemnts that are the same"

   358   case hd :: tl => s"this list is standard $hd::$tl"

   359 }

   360 

   361 foo(Nil)

   362 foo(List(1,2,3))

   363 foo(List(1,2))

   364 foo(List(1,1,2,3))

   365 foo(List(2,2,2,3))

   366 

   367 

   368 // Trees

   257 // guards in pattern-matching

   541 

   258 

   542 

   259 def foo(xs: List[Int]) : String = xs match {

   543 // Map type (upper-case)

   260   case Nil => s"this list is empty"

   544 //=======================

   261   case x :: xs if x % 2 == 0

   545 

   262      => s"the first elemnt is even"

   546 // Note the difference between map and Map

   263   case x :: y :: rest if x == y

   547 

   264      => s"this has two elemnts that are the same"

   548 val m = Map(1 -> "one", 2 -> "two", 10 -> "many")

   265   case hd :: tl => s"this list is standard $hd::$tl"

   549 

   266 }

   550 List((1, "one"), (2, "two"), (10, "many")).toMap

   267 

   551 

   268 foo(Nil)

   552 m.get(1)

   269 foo(List(1,2,3))

   553 m.get(4)

   270 foo(List(1,2))

   554 

   271 foo(List(1,1,2,3))

   555 m.getOrElse(1, "")

   272 foo(List(2,2,2,3))

   556 m.getOrElse(4, "")

   557 

   558 val new_m = m + (10 -> "ten")

   559 

   560 new_m.get(10)

   561 

   562 val m2 = for ((k, v) <- m) yield (k, v.toUpperCase)

   563 

   564 

   565 

   566 // groupBy function on Maps

   567 val lst = List("one", "two", "three", "four", "five")

   568 lst.groupBy(_.head)

   569 

   570 lst.groupBy(_.length)

   571 

   572 lst.groupBy(_.length).get(3)

   573 

   574 val grps = lst.groupBy(_.length)

   575 grps.keySet

   576 

   577 

   578 

   319 // Sudoku

   625 

   320 //========

   626 

   321 

   627 

   322 // uses Strings for games

   628 

   323 

   629 

   324 type Pos = (Int, Int)

   630 // Aside: concurrency 

   325 val emptyValue = '.'

   631 // scala-cli --extra-jars scala-parallel-collections_3-1.0.4.jar 

   326 val maxValue = 9

   632 

   327 

   633 for (n <- (1 to 10)) println(n)

   328 val allValues = "123456789".toList

   634 

   329 val indexes = (0 to 8).toList

   635 import scala.collection.parallel.CollectionConverters._

   330 

   636 

   331 

   637 for (n <- (1 to 10).par) println(n)

   332 def empty(game: String) = game.indexOf(emptyValue)

   638 

   333 def isDone(game: String) = empty(game) == -1 

   639 

   334 def emptyPosition(game: String) : Pos = 

   640 // for measuring time

   335   (empty(game) % maxValue, empty(game) / maxValue)

   641 def time_needed[T](n: Int, code: => T) = {

   336 

   642   val start = System.nanoTime()

   337 

   643   for (i <- (0 to n)) code

   338 def get_row(game: String, y: Int) = indexes.map(col => game(y * maxValue + col))

   644   val end = System.nanoTime()

   339 def get_col(game: String, x: Int) = indexes.map(row => game(x + row * maxValue))

   645   (end - start) / 1.0e9

   340 

   646 }

   341 def get_box(game: String, pos: Pos): List[Char] = {

   647 

   342     def base(p: Int): Int = (p / 3) * 3

   648 val list = (1L to 10_000_000L).toList

   343     val x0 = base(pos._1)

   649 time_needed(10, for (n <- list) yield n + 42)

   344     val y0 = base(pos._2)

   650 time_needed(10, for (n <- list.par) yield n + 42)

   345     for (x <- (x0 until x0 + 3).toList;

   651 

   346          y <- (y0 until y0 + 3).toList) yield game(x + y * maxValue)

   652 // ...but par does not make everything faster

   347 }         

   653 

   348 

   654 list.sum

   349 

   655 list.par.sum

   350 def update(game: String, pos: Int, value: Char): String = 

   656 

   351   game.updated(pos, value)

   657 time_needed(10, list.sum)

   352 

   658 time_needed(10, list.par.sum)

   353 def toAvoid(game: String, pos: Pos): List[Char] = 

   659 

   354   (get_col(game, pos._1) ++ get_row(game, pos._2) ++ get_box(game, pos))

   660 

   355 

   661 // Mutable vs Immutable

   356 def candidates(game: String, pos: Pos): List[Char] = 

   662 //======================

   357   allValues.diff(toAvoid(game, pos))

   663 //

   358 

   664 // Remember:

   359 def search(game: String): List[String] = {

   665 // - no vars, no ++i, no +=

   360   if (isDone(game)) List(game)

   666 // - no mutable data-structures (no Arrays, no ListBuffers)

   361   else 

   667 

   362     candidates(game, emptyPosition(game)).

   668 // But what the heck....lets try to count to 1 Mio in parallel

   363       map(c => search(update(game, empty(game), c))).flatten

   364 }

   365 

   366 

   367 def search1T(games: List[String]): Option[String] = games match {

   368   case Nil => None

   369   case game::rest => {

   370     if (isDone(game)) Some(game)

   371     else {

   372       val cs = candidates(game, emptyPosition(game))

   373       search1T(cs.map(c => update(game, empty(game), c)) ::: rest)

   374     }

   375   }

   376 }

   377 

   378 def pretty(game: String): String = 

   379   "\n" + (game.sliding(maxValue, maxValue).mkString(",\n"))

   380 

   381 

   382 // tail recursive version that searches 

   383 // for all solutions

   384 

   385 def searchT(games: List[String], sols: List[String]): List[String] = games match {

   386   case Nil => sols

   387   case game::rest => {

   388     if (isDone(game)) searchT(rest, game::sols)

   389     else {

   390       val cs = candidates(game, emptyPosition(game))

   391       searchT(cs.map(c => update(game, empty(game), c)) ::: rest, sols)

   392     }

   393   }

   394 }

   395 

   396 searchT(List(game3), List()).map(pretty)

   397 

   398 

   399 // tail recursive version that searches 

   400 // for a single solution

   401 

   402 def search1T(games: List[String]): Option[String] = games match {

   403   case Nil => None

   404   case game::rest => {

   405     if (isDone(game)) Some(game)

   406     else {

   407       val cs = candidates(game, emptyPosition(game))

   408       search1T(cs.map(c => update(game, empty(game), c)) ::: rest)

   409     }

   410   }

   411 }

   412 

   413 search1T(List(game3)).map(pretty)

   414 time_needed(10, search1T(List(game3)))

   415 

   416 

   417 // game with multiple solutions

   418 val game3 = """.8...9743

   419               |.5...8.1.

   420               |.1.......

   421               |8....5...

   422               |...8.4...

   423               |...3....6

   424               |.......7.

   425               |.3.5...8.

   426               |9724...5.""".stripMargin.replaceAll("\\n", "")

   427 

   428 searchT(List(game3), Nil).map(pretty)

   429 search1T(List(game3)).map(pretty)

   430 

   431 // Moral: Whenever a recursive function is resource-critical

   432 // (i.e. works with large recursion depth), then you need to

   433 // write it in tail-recursive fashion.

   435 // Unfortuantely, Scala because of current limitations in 

   670 // requires

   436 // the JVM is not as clever as other functional languages. It can 

   671 // scala-cli --extra-jars scala- parallel-collections_3-1.0.4.jar

   437 // only optimise "self-tail calls". This excludes the cases of 

   672 

   438 // multiple functions making tail calls to each other. Well,

   673 import scala.collection.parallel.CollectionConverters._

   439 // nothing is perfect. 

   674 

   440 

   675 def test() = {

   676   var cnt = 0

   677 

   678   for(i <- (1 to 100_000).par) cnt += 1

   679 

   680   println(s"Should be 100000: $cnt")

   681 }

   682 

   683 test()

   684 

   685 // Or

   686 // Q: Count how many elements are in the intersections of 

   687 //    two sets?

   688 // A; IMPROPER WAY (mutable counter)

   689 

   690 def count_intersection(A: Set[Int], B: Set[Int]) : Int = {

   691   var count = 0

   692   for (x <- A.par; if B contains x) count += 1 

   693   count

   694 }

   695 

   696 val A = (0 to 999).toSet

   697 val B = (0 to 999 by 4).toSet

   698 

   699 count_intersection(A, B)

   700 

   701 // but do not try to add .par to the for-loop above

   702 

   703 

   704 //propper parallel version

   705 def count_intersection2(A: Set[Int], B: Set[Int]) : Int = 

   706   A.par.count(x => B contains x)

   707 

   708 count_intersection2(A, B)

   709 

   710