// A Web-Scraper that extracts the daily Shanghai polution// data from the web-page//// http://www.envir.gov.cn/eng/airep/index.asp////// Important! They stopped providing this data in November// 2012, but kept the historical data since 2001. So dates// must be in that range.import java.io.OutputStreamWriterimport java.net.URLimport scala.io.Source.fromInputStreamval url = new URL("http://www.envir.gov.cn/eng/airep/index.asp")//connecting to urlval conn = url.openConnectionconn.setRequestProperty("User-Agent", "")conn.setDoOutput(true)conn.connect//sending dataval wr = new OutputStreamWriter(conn.getOutputStream())//possible date rangeswr.write("Fdate=2012-8-24&Tdate=2012-09-25")//wr.write("Fdate=2001-9-18&Tdate=2012-09-24")wr.flushwr.close//receiving dataval page = fromInputStream(conn.getInputStream).getLines.mkString("\n")//data encoded as an HTML-string, which you can see with//println(page)// regular expression: excludes newlines, // therefore we have to use [\S\s]val regex1 = """<tr align=\"center\">[\S\s]*?</tr>""".rval rows = regex1.findAllIn(page).toList//print(rows)val regex2 = """<td align=\"center\">([\S\s]*?)</td>""".rdef aux(s: String) : Array[String] = { for (m <- regex2.findAllIn(s).toArray) yield m match { case regex2(value) => value.trim }}//data completely extractedval data = rows.map { aux }//for comparing elements from an arraydef compare(i: Int)(e: Array[String], f: Array[String]) = e(i).toInt < f(i).toIntprintln("The day with highest particle pollution (PM_10)")println(data.sortWith(compare(1)).last.mkString(","))println("The day with highest sulfur dioxide (SO_2)")println(data.sortWith(compare(2)).last.mkString(","))println("The day with highest nitro dioxide (NO_2)")println(data.sortWith(compare(3)).last.mkString(","))println("The day(s) with highest PM_10")val groups1 = data.groupBy(_(1).toInt)val max_key1 = groups1.keySet.maxprintln(groups1(max_key1).map(_.mkString(",")).mkString("\n"))println("The day(s) with highest SO_2")val groups2 = data.groupBy(_(2).toInt)val max_key2 = groups2.keySet.maxprintln(groups2(max_key2).map(_.mkString(",")).mkString("\n"))println("The day(s) with highest NO_2")val groups3 = data.groupBy(_(3).toInt)val max_key3 = groups3.keySet.maxprintln(groups3(max_key3).map(_.mkString(",")).mkString("\n"))