import io.Source
import scala.util.matching.Regex

// gets the first ~10K of a page
def get_page(url: String) : String = { 
  try {
    Source.fromURL(url).take(10000).mkString  
  }
  catch {
    case e => {
      println("  Problem with: " + url)
      ""
    }
  }
}


// regex for URLs
val http_pattern = """\"https?://[^\"]*\"""".r

def unquote(s: String) = s.drop(1).dropRight(1)

def get_all_URLs(page: String) : Set[String] = {
  (http_pattern.findAllIn(page)).map { unquote(_) }.toSet
}

// naive version - seraches until a given depth
// visits pages potentially more than once
def crawl(url: String, n: Int) : Unit = {
  if (n == 0) ()
  else {
    println("Visiting: " + n + " " + url)
    for (u <- get_all_URLs(get_page(url))) crawl(u, n - 1)
  }
}

// staring URL for the crawler
val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc/"""
//val startURL = """http://www.inf.kcl.ac.uk/staff/mml/"""


// call on the command line 
crawl(startURL, 2)

crawl("""http://www.dcs.kcl.ac.uk/staff/urbanc/msc-projects-12.html""", 2)