112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
1 |
// This version of the crawler that also
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
2 |
// "harvests" email addresses from webpages
|
100
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
3 |
|
7
|
4 |
import io.Source
|
|
5 |
import scala.util.matching.Regex
|
96
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
6 |
import scala.util._
|
7
|
7 |
|
99
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
8 |
def get_page(url: String) : String = {
|
428
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
9 |
Try(Source.fromURL(url)("ISO-8859-1").take(10000).mkString).
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
10 |
getOrElse { println(s" Problem with: $url"); ""}
|
99
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
11 |
}
|
7
|
12 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
13 |
// regexes for URLs, for "my" domain and for email addresses
|
242
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
14 |
val http_pattern = """"https?://[^"]*"""".r
|
427
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
15 |
val email_pattern = """([a-z0-9_\.-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})""".r /*@\label{emailline}@*/
|
7
|
16 |
|
|
17 |
def unquote(s: String) = s.drop(1).dropRight(1)
|
|
18 |
|
254
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
19 |
def get_all_URLs(page: String) : Set[String] =
|
96
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
20 |
http_pattern.findAllIn(page).map(unquote).toSet
|
7
|
21 |
|
242
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
22 |
def print_str(s: String) =
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
23 |
if (s == "") () else println(s)
|
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
24 |
|
7
|
25 |
def crawl(url: String, n: Int) : Unit = {
|
|
26 |
if (n == 0) ()
|
|
27 |
else {
|
550
|
28 |
println(s" Visiting: $n $url")
|
7
|
29 |
val page = get_page(url)
|
427
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
30 |
print_str(email_pattern.findAllIn(page).mkString("\n")) /*@\label{mainline}@*/
|
242
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
31 |
for (u <- get_all_URLs(page).par) crawl(u, n - 1)
|
7
|
32 |
}
|
|
33 |
}
|
|
34 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
35 |
// staring URL for the crawler
|
550
|
36 |
val startURL = """https://nms.kcl.ac.uk/christian.urban/"""
|
|
37 |
|
112
Christian Urban <christian dot urban at kcl dot ac dot uk>
diff
changeset
|
38 |
|
7
|
39 |
crawl(startURL, 3)
|