// A Web Crawler / Email Harvester

//=================================

//

// the idea is to look for links using the

// regular expression "https?://[^"]*" and for

// email addresses using another regex.

import io.Source

import scala.util._

// gets the first 10K of a web-page

def get_page(url: String) : String = {

  Try(Source.fromURL(url)("ISO-8859-1").take(10000).mkString).

    getOrElse { println(s"  Problem with: $url"); ""}

}

// regex for URLs and emails

val http_pattern = """"https?://[^"]*"""".r

val email_pattern = """([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})""".r

//email_pattern.findAllIn

//  ("foo bla christian@kcl.ac.uk 1234567").toList

// drops the first and last character from a string

def unquote(s: String) = s.drop(1).dropRight(1)

def get_all_URLs(page: String): Set[String] = 

  http_pattern.findAllIn(page).map(unquote).toSet

// naive version of crawl - searches until a given depth,

// visits pages potentially more than once

def crawl(url: String, n: Int) : Set[String] = {

  if (n == 0) Set()

  else {

    println(s"  Visiting: $n $url")

    val page = get_page(url)

    val new_emails = email_pattern.findAllIn(page).toSet

    new_emails ++ 

      (for (u <- get_all_URLs(page)) yield crawl(u, n - 1)).flatten

  }

// some starting URLs for the crawler

val startURL = """https://nms.kcl.ac.uk/christian.urban/"""

crawl(startURL, 2)

// User-defined Datatypes and Pattern Matching

//============================================

abstract class Exp

case class N(n: Int) extends Exp

case class Plus(e1: Exp, e2: Exp) extends Exp

case class Times(e1: Exp, e2: Exp) extends Exp

// string of an Exp

// eval of an Exp

// simp an Exp

// Tokens

// Reverse Polish Notation

// compute RP

// transform RP into Exp

// process RP string and generate Exp

def string(e: Exp) : String = e match {

  case N(n) => n.toString

  case Plus(e1, e2) => "(" + string(e1) + " + " + string(e2) + ")"

  case Times(e1, e2) => "(" + string(e1) + " * " + string(e2) + ")"

}

val e = Plus(N(9), Times(N(3), N(4)))

println(string(e))

def eval(e: Exp) : Int = e match {

  case N(n) => n

  case Plus(e1, e2) => eval(e1) + eval(e2)

  case Times(e1, e2) => eval(e1) * eval(e2)

eval(e)

def simp(e: Exp) : Exp = e match {

  case N(n) => N(n)

  case Plus(e1, e2) => (simp(e1), simp(e2)) match {

    case (N(0), e2s) => e2s

    case (e1s, N(0)) => e1s

    case (e1s, e2s) => Plus(e1s, e2s) 

  }

  case Times(e1, e2) => (simp(e1), simp(e2)) match {

    case (N(0), e2s) => N(0)

    case (e1s, N(0)) => N(0)

    case (N(1), e2s) => e2s

    case (e1s, N(1)) => e1s

    case (e1s, e2s) => Times(e1s, e2s) 

  }

}

val e2 = Times(Plus(N(0), N(1)), Plus(N(0), N(9)))

println(string(e2))

println(string(simp(e2)))

// Token and Reverse Polish Notation

abstract class Token

case class T(n: Int) extends Token

case object PL extends Token

case object TI extends Token

def rp(e: Exp) : List[Token] = e match {

  case N(n) => List(T(n))

  case Plus(e1, e2) => rp(e1) ::: rp(e2) ::: List(PL)

  case Times(e1, e2) => rp(e1) ::: rp(e2) ::: List(TI)

def comp(ts: List[Token], stk: List[Int]) : Int = (ts, stk) match {

  case (Nil, st) => st.head

  case (T(n)::rest, st) => comp(rest, n::st)

  case (PL::rest, n1::n2::st) => comp(rest, n1 + n2::st)

  case (TI::rest, n1::n2::st) => comp(rest, n1 * n2::st)

}

def exp(ts: List[Token], st: List[Exp]) : Exp = (ts, st) match {

  case (Nil, st) => st.head

  case (T(n)::rest, st) => exp(rest, N(n)::st)

  case (PL::rest, n1::n2::st) => exp(rest, Plus(n2, n1)::st)

  case (TI::rest, n1::n2::st) => exp(rest, Times(n2, n1)::st)

}

exp(toks(e2), Nil)

def proc(s: String) = s match {

  case "+" => PL

  case "*" => TI

  case n => T(n.toInt)

}

string(exp("1 2 + 4 * 5 + 3 +".split(" ").toList.map(proc), Nil))

// Jumping Towers

//================

// the first n prefixes of xs

// for 1 => include xs

def moves(xs: List[Int], n: Int) : List[List[Int]] = (xs, n) match {

  case (Nil, _) => Nil

  case (xs, 0) => Nil

  case (x::xs, n) => (x::xs) :: moves(xs, n - 1)

}

moves(List(5,1,0), 1)

moves(List(5,1,0), 2)

moves(List(5,1,0), 5)

// checks whether a jump tour exists at all

// in the second case it needs to be < instead of <=

def search(xs: List[Int]) : Boolean = xs match {

  case Nil => true

  case (x::xs) =>

    if (xs.length < x) true else moves(xs, x).exists(search(_))

}

search(List(5,3,2,5,1,1))

search(List(3,5,1,0,0,0,1))

search(List(3,5,1,0,0,0,0,1))

search(List(3,5,1,0,0,0,1,1))

search(List(3,5,1))

search(List(5,1,1))

search(Nil)

search(List(1))

search(List(5,1,1))

search(List(3,5,1,0,0,0,0,0,0,0,0,1))

// generates *all* jump tours

//    if we are only interested in the shortes one, we could

//    shortcircut the calculation and only return List(x) in

//    case where xs.length < x, because no tour can be shorter

//    than 1

// 

def jumps(xs: List[Int]) : List[List[Int]] = xs match {

  case Nil => Nil

  case (x::xs) => {

    val children = moves(xs, x)

    val results = children.flatMap((cs) => jumps(cs).map(x :: _))

    if (xs.length < x) List(x) :: results else results

  }

}

jumps(List(5,3,2,5,1,1))

jumps(List(3,5,1,2,1,2,1))

jumps(List(3,5,1,2,3,4,1))

jumps(List(3,5,1,0,0,0,1))

jumps(List(3,5,1))

jumps(List(5,1,1))

jumps(Nil)

jumps(List(1))

jumps(List(5,1,2))

moves(List(1,2), 5)

jumps(List(1,5,1,2))

jumps(List(3,5,1,0,0,0,0,0,0,0,0,1))

jumps(List(5,3,2,5,1,1)).minBy(_.length)

jumps(List(1,3,5,8,9,2,6,7,6,8,9)).minBy(_.length)

jumps(List(1,3,6,1,0,9)).minBy(_.length)

jumps(List(2,3,1,1,2,4,2,0,1,1)).minBy(_.length)

// Sudoku 

//========

// THE POINT OF THIS CODE IS NOT TO BE SUPER

// EFFICIENT AND FAST, just explaining exhaustive

// depth-first search

//get_row(game0, 0)

//get_row(game0, 1)

//get_box(game0, (3,1))

search(game0).map(pretty)

val game1 = """23.915...

              |...2..54.

              |6.7......

              |..1.....9

              |89.5.3.17

              |5.....6..

              |......9.5

              |.16..7...

              |...329..1""".stripMargin.replaceAll("\\n", "")

// game that is in the hard category

val game2 = """8........

              |..36.....

              |.7..9.2..

              |.5...7...

              |....457..

              |...1...3.

              |..1....68

              |..85...1.

              |.9....4..""".stripMargin.replaceAll("\\n", "")

// game with multiple solutions

val game3 = """.8...9743

              |.5...8.1.

              |.1.......

              |8....5...

              |...8.4...

              |...3....6

              |.......7.

              |.3.5...8.

              |9724...5.""".stripMargin.replaceAll("\\n", "")

search(game1).map(pretty)

search(game3).map(pretty)

search(game2).map(pretty)

// for measuring time

def time_needed[T](i: Int, code: => T) = {

  val start = System.nanoTime()

  for (j <- 1 to i) code

  val end = System.nanoTime()

  ((end - start) / 1.0e9) + " secs"

}

time_needed(1, search(game2))

time_needed(10, search1T(List(game3)))