# HG changeset patch
# User Christian Urban <christian dot urban at kcl dot ac dot uk>
# Date 1411389734 -3600
# Node ID 70c307641d05b7ff39f31263e94ac951c7db5ebb
# Parent  bc72478edca114b394578b14004533f643244b0d
updated

diff -r bc72478edca1 -r 70c307641d05 progs/crawler1.scala
--- a/progs/crawler1.scala	Mon Sep 22 02:01:41 2014 +0100
+++ b/progs/crawler1.scala	Mon Sep 22 13:42:14 2014 +0100
@@ -32,8 +32,8 @@
 }
 
 // some starting URLs for the crawler
-val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc"""
-//val startURL = """http://www.inf.kcl.ac.uk/staff/mcburney"""
+//val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc"""
+val startURL = """http://www.inf.kcl.ac.uk/staff/mcburney"""
 
 crawl(startURL, 2)
 
diff -r bc72478edca1 -r 70c307641d05 progs/scraper.scala
--- a/progs/scraper.scala	Mon Sep 22 02:01:41 2014 +0100
+++ b/progs/scraper.scala	Mon Sep 22 13:42:14 2014 +0100
@@ -16,6 +16,7 @@
 
 // connecting to url
 val conn = url.openConnection
+conn.setRequestProperty("User-Agent", "")
 conn.setDoOutput(true)
 conn.connect
 
@@ -24,6 +25,7 @@
 
 //possible date ranges
 wr.write("Fdate=2012-8-24&Tdate=2012-09-25")
+//wr.write("Fdate=2011-8-24&Tdate=2011-09-25")
 //wr.write("Fdate=2001-9-18&Tdate=2012-09-25")
 wr.flush
 wr.close