100
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
1 |
// This version of the crawler only
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
2 |
// checks links in the "domain" urbanc
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
3 |
|
1
|
4 |
import io.Source
|
|
5 |
import scala.util.matching.Regex
|
96
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
6 |
import scala.util._
|
1
|
7 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
8 |
// gets the first 10K of a web-page
|
100
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
9 |
def get_page(url: String) : String = {
|
428
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
10 |
Try(Source.fromURL(url)("ISO-8859-1").take(10000).mkString).
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
11 |
getOrElse { println(s" Problem with: $url"); ""}
|
100
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
12 |
}
|
1
|
13 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
14 |
// regexes for URLs and "my" domain
|
242
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
15 |
val http_pattern = """"https?://[^"]*"""".r
|
550
|
16 |
val my_urls = """urban""".r /*@\label{myurlline}@*/
|
|
17 |
//val my_urls = """kcl.ac.uk""".r
|
1
|
18 |
|
|
19 |
def unquote(s: String) = s.drop(1).dropRight(1)
|
|
20 |
|
254
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
21 |
def get_all_URLs(page: String) : Set[String] =
|
96
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
22 |
http_pattern.findAllIn(page).map(unquote).toSet
|
1
|
23 |
|
|
24 |
def crawl(url: String, n: Int) : Unit = {
|
427
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
25 |
if (n == 0) () /*@\label{changestartline}@*/
|
116
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
26 |
else if (my_urls.findFirstIn(url) == None) {
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
27 |
println(s"Visiting: $n $url")
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
28 |
get_page(url); ()
|
427
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
29 |
} /*@\label{changeendline}@*/
|
1
|
30 |
else {
|
95
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
31 |
println(s"Visiting: $n $url")
|
550
|
32 |
for (u <- get_all_URLs(get_page(url)).par) crawl(u, n - 1)
|
1
|
33 |
}
|
|
34 |
}
|
|
35 |
|
242
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
36 |
// starting URL for the crawler
|
550
|
37 |
val startURL = """https://nms.kcl.ac.uk/christian.urban/"""
|
|
38 |
//val startURL = """https://nms.kcl.ac.uk/christian.urban/bsc-projects-17.html"""
|
|
39 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
40 |
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
41 |
// can now deal with depth 3 and beyond
|
550
|
42 |
crawl(startURL, 3)
|
1
|
43 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
44 |
|