diff -r 1f3d89fe9820 -r 91145f6d9b0e progs/crawler3.scala --- a/progs/crawler3.scala Tue Sep 24 23:31:53 2013 +0100 +++ b/progs/crawler3.scala Wed Sep 25 20:35:54 2013 +0100 @@ -3,9 +3,10 @@ import scala.util._ // gets the first ~10K of a page -def get_page(url: String) : String = +def get_page(url: String) : String = { Try(Source.fromURL(url).take(10000).mkString) getOrElse { println(s" Problem with: $url"); ""} +} // staring URL for the crawler val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc/"""