changeset 99 | 91145f6d9b0e |
parent 96 | 9fcd3de53c06 |
child 100 | cbc2270c2938 |
--- a/progs/crawler3.scala Tue Sep 24 23:31:53 2013 +0100 +++ b/progs/crawler3.scala Wed Sep 25 20:35:54 2013 +0100 @@ -3,9 +3,10 @@ import scala.util._ // gets the first ~10K of a page -def get_page(url: String) : String = +def get_page(url: String) : String = { Try(Source.fromURL(url).take(10000).mkString) getOrElse { println(s" Problem with: $url"); ""} +} // staring URL for the crawler val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc/"""