// A Web-Scraper that extracts the daily Shanghai polution

// data from the web-page

//

//   http://www.envir.gov.cn/eng/airep/index.asp

//

//

// Important! They stoped providing this data in November

// 2012, but kept the historical data since 2001. So dates

// must be in that range.

import java.io.OutputStreamWriter

import java.net.URL

import scala.io.Source.fromInputStream

val url = new URL("http://www.envir.gov.cn/eng/airep/index.asp")

//connecting to url

val conn = url.openConnection

conn.setRequestProperty("User-Agent", "")

conn.setDoOutput(true)

conn.connect

//sending data

val wr = new OutputStreamWriter(conn.getOutputStream())

//possible date ranges

wr.write("Fdate=2012-9-24&Tdate=2012-08-25")

//wr.write("Fdate=2012-9-18&Tdate=2012-09-24")

//wr.write("Fdate=2001-5-18&Tdate=2012-09-25")

wr.flush

wr.close

//receiving data

val page = fromInputStream(conn.getInputStream).getLines.mkString("\n")

//data encoded as a string, which you can see with

//println(page)

// regular expression . excludes newlines, 

// therefore we have to use [\S\s]

val regex1 = """<tr align="center">[\S\s]*?</tr>""".r

val rows = regex1.findAllIn(page).toList

//print(rows)

val regex2 = """<td align="center">([\S\s]*?)</td>""".r

def aux(s: String) : Array[String] = {

  for (m <- regex2.findAllIn(s).toArray) yield m match {

    case regex2(value) => value.trim

  }

}

val data = rows.map { aux }

def compare(i: Int)(e: Array[String], f: Array[String]) = e(i).toInt < f(i).toInt

//day with highest particle pollution (PM_10)

println(data.sortWith(compare(1)).last)

//day with highest sulfur dioxide (SO_2)

println(data.sortWith(compare(2)).last)

//day with highest nitro dioxide (NO_2)

println(data.sortWith(compare(3)).last)

//days with highest PM_10

val groups = data.groupBy(_(1).toInt)

val max_key = groups.keySet.max

println(groups(max_key))