afl-material: progs/scraper.scala@735f7bbfae9b (annotated)

103 bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	1	// A Web-Scraper that extracts the daily Shanghai polution
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	2	// data from the web-page
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	3	//
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	4	// http://www.envir.gov.cn/eng/airep/index.asp
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	5	//
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	6	//
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	7	// Important! They stopped providing this data in November
103 bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	8	// 2012, but kept the historical data since 2001. So dates
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	9	// must be in that range.
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	10
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	11	import java.io.OutputStreamWriter
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	12	import java.net.URL
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	13	import scala.io.Source.fromInputStream
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	14
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	15	val url = new URL("http://www.envir.gov.cn/eng/airep/index.asp")
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	16
103 bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	17	//connecting to url
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	18	val conn = url.openConnection
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	19	conn.setRequestProperty("User-Agent", "")
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	20	conn.setDoOutput(true)
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	21	conn.connect
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	22
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	23	//sending data
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	24	val wr = new OutputStreamWriter(conn.getOutputStream())
103 bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	25
bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	26	//possible date ranges
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	27	wr.write("Fdate=2012-8-24&Tdate=2012-09-25")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	28	//wr.write("Fdate=2001-9-18&Tdate=2012-09-24")
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	29	wr.flush
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	30	wr.close
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	31
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	32	//receiving data
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	33	val page = fromInputStream(conn.getInputStream).getLines.mkString("\n")
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	34
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	35	//data encoded as an HTML-string, which you can see with
103 bea2dd1c7e73 links Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 99 diff changeset	36	//println(page)
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	37
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	38	// regular expression: excludes newlines,
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	39	// therefore we have to use [\S\s]
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	40	val regex1 = """<tr align=\"center\">[\S\s]*?</tr>""".r
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	41	val rows = regex1.findAllIn(page).toList
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	42
7 73cf4406b773 updated Christian Urban <urbanc@in.tum.de> parents: 1 diff changeset	43	//print(rows)
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	44
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	45	val regex2 = """<td align=\"center\">([\S\s]*?)</td>""".r
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	46
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	47	def aux(s: String) : Array[String] = {
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	48	for (m <- regex2.findAllIn(s).toArray) yield m match {
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	49	case regex2(value) => value.trim
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	50	}
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	51	}
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	52
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	53	//data completely extracted
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	54	val data = rows.map { aux }
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	55
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	56	//for comparing elements from an array
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	57	def compare(i: Int)(e: Array[String], f: Array[String]) = e(i).toInt < f(i).toInt
b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	58
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	59	println("The day with highest particle pollution (PM_10)")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	60	println(data.sortWith(compare(1)).last.mkString(","))
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	61
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	62	println("The day with highest sulfur dioxide (SO_2)")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	63	println(data.sortWith(compare(2)).last.mkString(","))
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	64
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	65	println("The day with highest nitro dioxide (NO_2)")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	66	println(data.sortWith(compare(3)).last.mkString(","))
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	67
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	68	println("The day(s) with highest PM_10")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	69	val groups1 = data.groupBy(_(1).toInt)
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	70	val max_key1 = groups1.keySet.max
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	71	println(groups1(max_key1).map(_.mkString(",")).mkString("\n"))
1 b606c9439fa6 new version Christian Urban <urbanc@in.tum.de> parents: diff changeset	72
114 735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	73	println("The day(s) with highest SO_2")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	74	val groups2 = data.groupBy(_(2).toInt)
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	75	val max_key2 = groups2.keySet.max
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	76	println(groups2(max_key2).map(_.mkString(",")).mkString("\n"))
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	77
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	78	println("The day(s) with highest NO_2")
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	79	val groups3 = data.groupBy(_(3).toInt)
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	80	val max_key3 = groups3.keySet.max
735f7bbfae9b added Christian Urban <christian dot urban at kcl dot ac dot uk> parents: 103 diff changeset	81	println(groups3(max_key3).map(_.mkString(",")).mkString("\n"))

author	Christian Urban <christian dot urban at kcl dot ac dot uk>
	Fri, 27 Sep 2013 12:22:43 +0100
changeset 114	735f7bbfae9b
parent 103	bea2dd1c7e73
child 250	b79e704acb72
permissions	-rw-r--r--