progs/crawler3.scala
changeset 101 4758a6155878
parent 100 cbc2270c2938
child 112 95ee5cc5c05d
equal deleted inserted replaced
100:cbc2270c2938 101:4758a6155878
     3 
     3 
     4 import io.Source
     4 import io.Source
     5 import scala.util.matching.Regex
     5 import scala.util.matching.Regex
     6 import scala.util._
     6 import scala.util._
     7 
     7 
     8 // gets the first ~10K of a page
     8 // gets the first ~10K of a web-page
     9 def get_page(url: String) : String = {
     9 def get_page(url: String) : String = {
    10   Try(Source.fromURL(url).take(10000).mkString) getOrElse 
    10   Try(Source.fromURL(url).take(10000).mkString) getOrElse 
    11     { println(s"  Problem with: $url"); ""}
    11     { println(s"  Problem with: $url"); ""}
    12 }
    12 }
    13 
    13