author | Christian Urban <christian dot urban at kcl dot ac dot uk> |
Mon, 15 Sep 2014 07:25:17 +0100 | |
changeset 250 | b79e704acb72 |
parent 114 | 735f7bbfae9b |
child 257 | 70c307641d05 |
permissions | -rw-r--r-- |
103
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
1 |
// A Web-Scraper that extracts the daily Shanghai polution |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
2 |
// data from the web-page |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
3 |
// |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
4 |
// http://www.envir.gov.cn/eng/airep/index.asp |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
5 |
// |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
6 |
// |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
7 |
// Important! They stopped providing this data in November |
103
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
8 |
// 2012, but kept the historical data since 2001. So dates |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
9 |
// must be in that range. |
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
10 |
|
1 | 11 |
import java.io.OutputStreamWriter |
12 |
import java.net.URL |
|
13 |
import scala.io.Source.fromInputStream |
|
14 |
||
15 |
val url = new URL("http://www.envir.gov.cn/eng/airep/index.asp") |
|
16 |
||
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
17 |
// connecting to url |
1 | 18 |
val conn = url.openConnection |
19 |
conn.setDoOutput(true) |
|
20 |
conn.connect |
|
21 |
||
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
22 |
// sending data |
1 | 23 |
val wr = new OutputStreamWriter(conn.getOutputStream()) |
103
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
24 |
|
bea2dd1c7e73
links
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
99
diff
changeset
|
25 |
//possible date ranges |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
26 |
wr.write("Fdate=2012-8-24&Tdate=2012-09-25") |
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
27 |
//wr.write("Fdate=2001-9-18&Tdate=2012-09-25") |
1 | 28 |
wr.flush |
29 |
wr.close |
|
30 |
||
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
31 |
// receiving data as page made of HTML |
1 | 32 |
val page = fromInputStream(conn.getInputStream).getLines.mkString("\n") |
33 |
||
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
34 |
// received data can be seen with |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
35 |
// println(page) |
1 | 36 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
37 |
// regular expression for extracting rows: |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
38 |
// - the usual . would exclude newlines, |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
39 |
// - therefore we have to use [\S\s], which really |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
40 |
// matches everything |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
41 |
// - *? is the "lazy" version of * |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
42 |
val regex1 = """<tr align="center">[\S\s]*?</tr>""".r |
1 | 43 |
val rows = regex1.findAllIn(page).toList |
44 |
||
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
45 |
// data in rows |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
46 |
// println(rows) |
1 | 47 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
48 |
// extracting row entries (date, PM_10, SO_2, NO_2) |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
49 |
// the use of (..) allows us to extract the matched text |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
50 |
val regex2 = """<td align="center">([\S\s]*?)</td>""".r |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
51 |
|
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
52 |
def extract(s: String) : List[String] = { |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
53 |
for (regex2(value) <- regex2.findAllIn(s).toList) yield value.trim |
1 | 54 |
} |
55 |
||
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
56 |
//data completely extracted |
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
57 |
val data = rows.map(extract) |
1 | 58 |
|
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
59 |
//for comparing elements from an array |
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
60 |
def compare(i: Int)(e: List[String], f: List[String]) = e(i).toInt > f(i).toInt |
1 | 61 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
62 |
println("The day with the highest particle pollution (PM_10)") |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
63 |
println(data.sortWith(compare(1)).head.mkString(",")) |
1 | 64 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
65 |
println("The day with the highest sulfur dioxide (SO_2)") |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
66 |
println(data.sortWith(compare(2)).head.mkString(",")) |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
67 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
68 |
println("The day with the highest nitro dioxide (NO_2)") |
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
69 |
println(data.sortWith(compare(3)).head.mkString(",")) |
1 | 70 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
71 |
println("The day(s) with the highest PM_10") |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
72 |
val groups1 = data.groupBy(_(1).toInt) |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
73 |
val max_key1 = groups1.keySet.max |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
74 |
println(groups1(max_key1).map(_.mkString(",")).mkString("\n")) |
1 | 75 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
76 |
println("The day(s) with the highest SO_2") |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
77 |
val groups2 = data.groupBy(_(2).toInt) |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
78 |
val max_key2 = groups2.keySet.max |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
79 |
println(groups2(max_key2).map(_.mkString(",")).mkString("\n")) |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
80 |
|
250
b79e704acb72
updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
114
diff
changeset
|
81 |
println("The day(s) with the highest NO_2") |
114
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
82 |
val groups3 = data.groupBy(_(3).toInt) |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
83 |
val max_key3 = groups3.keySet.max |
735f7bbfae9b
added
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents:
103
diff
changeset
|
84 |
println(groups3(max_key3).map(_.mkString(",")).mkString("\n")) |