Mercurial Mercurial > hg > afl-material / changeset
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | raw | gz | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
updated
authorChristian Urban <christian dot urban at kcl dot ac dot uk>
Mon, 22 Sep 2014 13:42:14 +0100
changeset 257 70c307641d05
parent 256 bc72478edca1
child 258 1e4da6d2490c
updated
progs/crawler1.scala file | annotate | diff | comparison | revisions
progs/scraper.scala file | annotate | diff | comparison | revisions 
--- a/progs/crawler1.scala	Mon Sep 22 02:01:41 2014 +0100
+++ b/progs/crawler1.scala	Mon Sep 22 13:42:14 2014 +0100
@@ -32,8 +32,8 @@
 }
 
 // some starting URLs for the crawler
-val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc"""
-//val startURL = """http://www.inf.kcl.ac.uk/staff/mcburney"""
+//val startURL = """http://www.inf.kcl.ac.uk/staff/urbanc"""
+val startURL = """http://www.inf.kcl.ac.uk/staff/mcburney"""
 
 crawl(startURL, 2)
 
--- a/progs/scraper.scala	Mon Sep 22 02:01:41 2014 +0100
+++ b/progs/scraper.scala	Mon Sep 22 13:42:14 2014 +0100
@@ -16,6 +16,7 @@
 
 // connecting to url
 val conn = url.openConnection
+conn.setRequestProperty("User-Agent", "")
 conn.setDoOutput(true)
 conn.connect
 
@@ -24,6 +25,7 @@
 
 //possible date ranges
 wr.write("Fdate=2012-8-24&Tdate=2012-09-25")
+//wr.write("Fdate=2011-8-24&Tdate=2011-09-25")
 //wr.write("Fdate=2001-9-18&Tdate=2012-09-25")
 wr.flush
 wr.close
afl-material