diff -r 1f3d89fe9820 -r 91145f6d9b0e progs/crawler1.scala --- a/progs/crawler1.scala Tue Sep 24 23:31:53 2013 +0100 +++ b/progs/crawler1.scala Wed Sep 25 20:35:54 2013 +0100 @@ -3,10 +3,10 @@ import scala.util._ // gets the first ~10K of a page -def get_page(url: String) : String = +def get_page(url: String) : String = { Try(Source.fromURL(url).take(10000).mkString) getOrElse { println(s" Problem with: $url"); ""} - +} // regex for URLs val http_pattern = """\"https?://[^\"]*\"""".r