|
1
|
1 |
import io.Source
|
|
|
2 |
import scala.util.matching.Regex
|
|
|
3 |
|
|
|
4 |
// gets the first ~10K of a page
|
|
|
5 |
def get_page(url: String) : String = {
|
|
|
6 |
try {
|
|
|
7 |
Source.fromURL(url).take(10000).mkString
|
|
|
8 |
}
|
|
|
9 |
catch {
|
|
|
10 |
case e => {
|
|
|
11 |
println(" Problem with: " + url)
|
|
|
12 |
""
|
|
|
13 |
}
|
|
|
14 |
}
|
|
|
15 |
}
|
|
|
16 |
|
|
|
17 |
|
|
|
18 |
// regex for URLs
|
|
|
19 |
val http_pattern = """\"https?://[^\"]*\"""".r
|
|
|
20 |
|
|
|
21 |
def unquote(s: String) = s.drop(1).dropRight(1)
|
|
|
22 |
|
|
|
23 |
def get_all_URLs(page: String) : Set[String] = {
|
|
|
24 |
(http_pattern.findAllIn(page)).map { unquote(_) }.toSet
|
|
|
25 |
}
|
|
|
26 |
|
|
|
27 |
// naive version - seraches until a given depth
|
|
|
28 |
// visits pages potentially more than once
|
|
|
29 |
def crawl(url: String, n: Int) : Unit = {
|
|
|
30 |
if (n == 0) ()
|
|
|
31 |
else {
|
|
|
32 |
println("Visiting: " + n + " " + url)
|
|
|
33 |
for (u <- get_all_URLs(get_page(url))) crawl(u, n - 1)
|
|
|
34 |
}
|
|
|
35 |
}
|
|
|
36 |
|
|
3
|
37 |
// staring URL for the crawler
|
|
|
38 |
val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc/"""
|
|
7
|
39 |
//val startURL = """http://www.inf.kcl.ac.uk/staff/mml/"""
|
|
|
40 |
|
|
3
|
41 |
|
|
|
42 |
// call on the command line
|
|
1
|
43 |
crawl(startURL, 2)
|
|
|
44 |
|