equal
deleted
inserted
replaced
18 def unquote(s: String) = s.drop(1).dropRight(1) |
18 def unquote(s: String) = s.drop(1).dropRight(1) |
19 |
19 |
20 def get_all_URLs(page: String) : Set[String] = |
20 def get_all_URLs(page: String) : Set[String] = |
21 http_pattern.findAllIn(page).map(unquote).toSet /*@\label{findallline}@*/ |
21 http_pattern.findAllIn(page).map(unquote).toSet /*@\label{findallline}@*/ |
22 |
22 |
23 |
|
24 // naive version of crawl - searches until a given depth, |
23 // naive version of crawl - searches until a given depth, |
25 // visits pages potentially more than once |
24 // visits pages potentially more than once |
26 def crawl(url: String, n: Int) : Unit = { |
25 def crawl(url: String, n: Int) : Unit = { |
27 if (n == 0) () |
26 if (n == 0) () |
28 else { |
27 else { |
30 for (u <- get_all_URLs(get_page(url))) crawl(u, n - 1) |
29 for (u <- get_all_URLs(get_page(url))) crawl(u, n - 1) |
31 } |
30 } |
32 } |
31 } |
33 |
32 |
34 // some starting URLs for the crawler |
33 // some starting URLs for the crawler |
35 val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc""" |
34 val startURL = """https://nms.kcl.ac.uk/christian.urban/""" |
36 //val startURL = """http://www.inf.kcl.ac.uk/staff/mcburney""" |
35 //val startURL = """https://nms.kcl.ac.uk/luc.moreau/""" |
37 |
36 |
|
37 crawl(startURL, 3) |
38 |
38 |
39 crawl(startURL, 2) |
|
40 |
|