--- a/testing2/docdiff.scala	Tue Nov 12 10:47:27 2019 +0000
+++ b/testing2/docdiff.scala	Tue Nov 19 00:40:27 2019 +0000
@@ -2,7 +2,8 @@
 //========================================
 
 
-object CW7a { // for purposes of generating a jar
+object CW7a { 
+
 
 //(1) Complete the clean function below. It should find
 //    all words in a string using the regular expression
@@ -12,16 +13,39 @@
 //
 //    The words should be Returned as a list of strings.
 
-def clean(s: String) : List[String] = 
-  ("""\w+""".r).findAllIn(s).toList
+
+def clean(s: String) : List[String] = {
+    val regex = """\w+""".r;
+    val list_of_words = s.split(" ").toList
+    for(word <- list_of_words;
+        actual_word <- divide_string_where_different(word, regex.findAllIn(word).mkString, 0)) yield actual_word
+}
 
+/*
+    A secondary function that takes as parameters @param original which is the original word, @param returned which is thea word after the process of removing 
+    some characters not allowed by a regular expression, and @param i which is the index where to start compare the characters of the two words.
+    It @return a List of strings which represents all the substrings of returned which were previously divided by characters not allowed by the regular expression applied on it.
+*/
+def divide_string_where_different(original: String, returned: String, i : Int): List[String] ={
+    val max_i = original.length -1
+    if(original(i) != returned(i)) returned.substring(0, i)::divide_string_where_different(original.substring(i+1), returned.substring(i), 0).filter(_.nonEmpty)
+    else if (i == max_i) List(returned)
+    else divide_string_where_different(original,returned, i +1)
+    
+}
 
 //(2) The function occurrences calculates the number of times  
 //    strings occur in a list of strings. These occurrences should 
 //    be calculated as a Map from strings to integers.
 
-def occurrences(xs: List[String]): Map[String, Int] =
-  (for (x <- xs.distinct) yield (x, xs.count(_ == x))).toMap
+
+def occurrences(xs: List[String]): Map[String, Int] = {
+    val lst = xs.distinct
+    val word_pairs = (for (word <- lst) yield (word, xs.count(_==word))).toList
+    word_pairs.toMap
+}
+
+
 
 //(3) This functions calculates the dot-product of two documents
 //    (list of strings). For this it calculates the occurrence
@@ -29,29 +53,33 @@
 //    If a string does not occur in a document, the product is zero.
 //    The function finally sums up all products. 
 
+
 def prod(lst1: List[String], lst2: List[String]) : Int = {
-    val words = (lst1 ::: lst2).distinct
-    val occs1 = occurrences(lst1)
-    val occs2 = occurrences(lst2)
-    words.map{ w => occs1.getOrElse(w, 0) * occs2.getOrElse(w, 0) }.sum
+    val map1 = occurrences(lst1)
+    val map2 = occurrences(lst2)
+    print(s"map1 is $map1 \n and map2 is $map2")
+    val pairs = (for(pair1 <- map1 if(map2.get(pair1._1) != None)) yield (pair1._2, map2.get(pair1._1).get)).toList
+    print(s"\n pairs are $pairs")
+    val products = (for(pair <- pairs) yield pair._1 * pair._2).toList
+    products.sum
+
 }
 
+
 //(4) Complete the functions overlap and similarity. The overlap of
 //    two documents is calculated by the formula given in the assignment
 //    description. The similarity of two strings is given by the overlap
-//    of the cleaned (see (1)) strings.  
-
-def overlap(lst1: List[String], lst2: List[String]) : Double = {
-    val m1 = prod(lst1, lst1)
-    val m2 = prod(lst2, lst2) 
-    prod(lst1, lst2).toDouble / (List(m1, m2).max)
-}
-
-def similarity(s1: String, s2: String) : Double =
-  overlap(clean(s1), clean(s2))
+//    of the cleaned strings (see (1)).  
 
 
-/*
+//def overlap(lst1: List[String], lst2: List[String]) : Double = ...
+
+//def similarity(s1: String, s2: String) : Double = ...
+
+
+
+
+/* Test cases
 
 
 val list1 = List("a", "b", "b", "c", "d") 
@@ -61,6 +89,8 @@
 occurrences(List("d", "b", "d", "b", "d"))   // Map(d -> 3, b -> 2)
 
 prod(list1,list2) // 7 
+prod(list1,list1)
+prod(list2,list2)
 
 overlap(list1, list2)   // 0.5384615384615384
 overlap(list2, list1)   // 0.5384615384615384
@@ -81,7 +111,7 @@
 heritage which ensures Australia's capacity to attract international
 ecotourists."""
 
-similarity(orig1, plag1)
+similarity(orig1, plag1) // 0.8679245283018868
 
 
 // Plagiarism examples from 
@@ -105,13 +135,15 @@
 recovery: a controversial tactic that is often implemented immediately
 following an oil spill."""
 
-overlap(clean(orig2), clean(plag2))
-similarity(orig2, plag2)
+overlap(clean(orig2), clean(plag2))  // 0.728
+similarity(orig2, plag2)             // 0.728
+
 
+ 
 // The punchline: everything above 0.6 looks suspicious and 
-// should be looked at by staff.
+// should be investigated by staff.
 
 */
 
+}
 
-}