progs/scraper.scala
author Christian Urban <christian dot urban at kcl dot ac dot uk>
Mon, 15 Sep 2014 07:25:17 +0100
changeset 250 b79e704acb72
parent 114 735f7bbfae9b
child 257 70c307641d05
permissions -rw-r--r--
updated
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
103
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     1
// A Web-Scraper that extracts the daily Shanghai polution
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     2
// data from the web-page
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     3
//
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     4
//   http://www.envir.gov.cn/eng/airep/index.asp
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     5
//
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     6
//
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
     7
// Important! They stopped providing this data in November
103
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     8
// 2012, but kept the historical data since 2001. So dates
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
     9
// must be in that range.
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
    10
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    11
import java.io.OutputStreamWriter
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    12
import java.net.URL
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    13
import scala.io.Source.fromInputStream
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    14
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    15
val url = new URL("http://www.envir.gov.cn/eng/airep/index.asp")
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    16
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    17
// connecting to url
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    18
val conn = url.openConnection
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    19
conn.setDoOutput(true)
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    20
conn.connect
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    21
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    22
// sending data
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    23
val wr = new OutputStreamWriter(conn.getOutputStream())
103
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
    24
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 99
diff changeset
    25
//possible date ranges
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    26
wr.write("Fdate=2012-8-24&Tdate=2012-09-25")
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    27
//wr.write("Fdate=2001-9-18&Tdate=2012-09-25")
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    28
wr.flush
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    29
wr.close
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    30
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    31
// receiving data as page made of HTML
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    32
val page = fromInputStream(conn.getInputStream).getLines.mkString("\n")
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    33
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    34
// received data can be seen with
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    35
// println(page)
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    36
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    37
// regular expression for extracting rows: 
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    38
// - the usual . would exclude newlines, 
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    39
// - therefore we have to use [\S\s], which really 
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    40
//   matches everything
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    41
// - *? is the "lazy" version of *
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    42
val regex1 = """<tr align="center">[\S\s]*?</tr>""".r
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    43
val rows = regex1.findAllIn(page).toList
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    44
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    45
// data in rows
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    46
// println(rows)
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    47
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    48
// extracting row entries (date, PM_10, SO_2, NO_2)
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    49
// the use of (..) allows us to extract the matched text
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    50
val regex2 = """<td align="center">([\S\s]*?)</td>""".r
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    51
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    52
def extract(s: String) : List[String] = {
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    53
  for (regex2(value) <- regex2.findAllIn(s).toList) yield value.trim
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    54
}
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    55
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    56
//data completely extracted
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    57
val data = rows.map(extract)
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    58
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    59
//for comparing elements from an array
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    60
def compare(i: Int)(e: List[String], f: List[String]) = e(i).toInt > f(i).toInt
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    61
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    62
println("The day with the highest particle pollution (PM_10)")
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    63
println(data.sortWith(compare(1)).head.mkString(","))
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    64
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    65
println("The day with the highest sulfur dioxide (SO_2)")
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    66
println(data.sortWith(compare(2)).head.mkString(","))
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    67
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    68
println("The day with the highest nitro dioxide (NO_2)")
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    69
println(data.sortWith(compare(3)).head.mkString(","))
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    70
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    71
println("The day(s) with the highest PM_10")
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    72
val groups1 = data.groupBy(_(1).toInt)
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    73
val max_key1 = groups1.keySet.max
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    74
println(groups1(max_key1).map(_.mkString(",")).mkString("\n"))
1
b606c9439fa6 new version
Christian Urban <urbanc@in.tum.de>
parents:
diff changeset
    75
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    76
println("The day(s) with the highest SO_2")
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    77
val groups2 = data.groupBy(_(2).toInt)
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    78
val max_key2 = groups2.keySet.max
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    79
println(groups2(max_key2).map(_.mkString(",")).mkString("\n"))
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    80
250
b79e704acb72 updated
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 114
diff changeset
    81
println("The day(s) with the highest NO_2")
114
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    82
val groups3 = data.groupBy(_(3).toInt)
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    83
val max_key3 = groups3.keySet.max
Christian Urban <christian dot urban at kcl dot ac dot uk>
parents: 103
diff changeset
    84
println(groups3(max_key3).map(_.mkString(",")).mkString("\n"))