testing2/danube.scala
changeset 326 e5453add7df6
parent 284 9a04eb6a2291
child 329 8a34b2ebc8cc
equal deleted inserted replaced
325:ca9c1cf929fa 326:e5453add7df6
     1 // Core Part about Movie Recommendations 
     1 // Core Part about Movie Recommendations 
     2 // at Danube.co.uk
     2 // at Danube.co.uk
     3 //===========================================
     3 //===========================================
     4 
     4 
       
     5 object CW7b {
       
     6 
     5 import io.Source
     7 import io.Source
     6 import scala.util._
     8 import scala.util._
     7 
       
     8 object CW7b { // for purposes of generating a jar
       
     9 
     9 
    10 // (1) Implement the function get_csv_url which takes an url-string
    10 // (1) Implement the function get_csv_url which takes an url-string
    11 //     as argument and requests the corresponding file. The two urls
    11 //     as argument and requests the corresponding file. The two urls
    12 //     of interest are ratings_url and movies_url, which correspond 
    12 //     of interest are ratings_url and movies_url, which correspond 
    13 //     to CSV-files.
    13 //     to CSV-files.
    14 //     The function should ReTurn the CSV file appropriately broken
    14 //
       
    15 //     The function should ReTurn the CSV-file appropriately broken
    15 //     up into lines, and the first line should be dropped (that is without
    16 //     up into lines, and the first line should be dropped (that is without
    16 //     the header of the CSV file). The result is a list of strings (lines
    17 //     the header of the CSV-file). The result is a list of strings (lines
    17 //     in the file).
    18 //     in the file).
    18 
    19 
    19 def get_csv_url(url: String) : List[String] = {
    20 def get_csv_url(url: String) : List[String] = {
    20   val csv = Source.fromURL(url)("ISO-8859-1")
    21     Try(Source.fromURL(url)("UTF-8").mkString.split("\n").toList.tail).getOrElse(List())
    21   csv.mkString.split("\n").toList.drop(1)
    22 }
    22 }
    23 
    23 
    24 
    24 val ratings_url = """https://nms.kcl.ac.uk/christian.urban/ratings.csv"""
    25 val ratings_url = """https://nms.kcl.ac.uk/christian.urban/ratings.csv"""
    25 val movies_url = """https://nms.kcl.ac.uk/christian.urban/movies.csv"""
    26 val movies_url = """https://nms.kcl.ac.uk/christian.urban/movies.csv"""
    26 
    27 
    27 // test cases
    28 // testcases
    28 
    29 //-----------
    29 //val ratings = get_csv_url(ratings_url)
    30 // val ratings = get_csv_url(ratings_url)
    30 //val movies = get_csv_url(movies_url)
    31 // val movies = get_csv_url(movies_url)
    31 
    32 
    32 //ratings.length  // 87313
    33 //ratings.length  // 87313
    33 //movies.length   // 9742
    34 //movies.length   // 9742
    34 
    35 
    35 // (2) Implement two functions that process the CSV files. The ratings
    36 
       
    37 
       
    38 // (2) Implement two functions that process the CSV-files from (1). The ratings
    36 //     function filters out all ratings below 4 and ReTurns a list of 
    39 //     function filters out all ratings below 4 and ReTurns a list of 
    37 //     (userID, movieID) pairs. The movies function just ReTurns a list 
    40 //     (userID, movieID) pairs. The movies function just ReTurns a list 
    38 //     of (movieId, title) pairs.
    41 //     of (movieID, title) pairs.
    39 
    42 
    40 
    43 
    41 def process_ratings(lines: List[String]) : List[(String, String)] = {
    44 def process_ratings(lines: List[String]) : List[(String, String)] = {
    42   for (cols <- lines.map(_.split(",").toList); 
    45     val filteredLines = lines.filter(line => line.split(",")(2).toInt >= 4)
    43        if (cols(2).toFloat >= 4)) yield (cols(0), cols(1))  
    46     filteredLines.map(line => (line.split(",")(0), line.split(",")(1)))
    44 }
    47 }
    45 
    48 
    46 def process_movies(lines: List[String]) : List[(String, String)] = {
    49 def process_movies(lines: List[String]) : List[(String, String)] = {
    47   for (cols <- lines.map(_.split(",").toList)) yield (cols(0), cols(1))  
    50     lines.map(line => (line.split(",")(0), line.split(",")(1)))
    48 }
    51 }
    49 
    52 
    50 // test cases
    53 
    51 
    54 // testcases
    52 //val good_ratings = process_ratings(ratings)
    55 //-----------
    53 //val movie_names = process_movies(movies)
    56 // val good_ratings = process_ratings(ratings)
       
    57 // val movie_names = process_movies(movies)
    54 
    58 
    55 //good_ratings.length   //48580
    59 //good_ratings.length   //48580
    56 //movie_names.length    // 9742
    60 //movie_names.length    // 9742
    57 
    61 
    58 //==============================================
    62 
    59 // Do not change anything below, unless you want 
    63 
    60 // to submit the file for the advanced part 3!
    64 
    61 //==============================================
    65 // (3) Implement a grouping function that calculates a Map
    62 
    66 //     containing the userIDs and all the corresponding recommendations 
    63 
    67 //     (list of movieIDs). This  should be implemented in a tail
    64 // (3) Implement a grouping function that calulates a map
    68 //     recursive fashion, using a Map m as accumulator. This Map m
    65 //     containing the userIds and all the corresponding recommendations 
    69 //     is set to Map() at the beginning of the calculation.
    66 //     (list of movieIds). This  should be implemented in a tail
       
    67 //     recursive fashion, using a map m as accumulator. This map
       
    68 //     is set to Map() at the beginning of the claculation.
       
    69 
    70 
    70 def groupById(ratings: List[(String, String)], 
    71 def groupById(ratings: List[(String, String)], 
    71               m: Map[String, List[String]]) : Map[String, List[String]] = ratings match {
    72               m: Map[String, List[String]]) : Map[String, List[String]] = {
    72   case Nil => m
    73     if (ratings.length == 0) m
    73   case (id, mov) :: rest => {
    74     else {
    74     val old_ratings = m.getOrElse (id, Nil)
    75         val firstUser = ratings(0)._1
    75     val new_ratings = m + (id -> (mov :: old_ratings))
    76         val userRatings = ratings.filter(r => r._1 == firstUser)
    76     groupById(rest, new_ratings)
    77         val movieIds = userRatings.map(r => r._2)
    77   }
    78         val newMap = m + (firstUser -> movieIds)
    78 }
    79         groupById(ratings.filter(r => r._1 != firstUser), newMap)
    79 
    80     }
    80 //
    81 }
    81 //val ls = List(("1", "a"), ("2", "a"), ("1", "c"), ("2", "a"), ("1", "c"))
    82 
    82 //
    83 
    83 //val m = groupById(ls, Map())
    84 // testcases
    84 //
    85 //-----------
    85 //m.getOrElse("1", Nil).count(_ == "c") // => 2
       
    86 //m.getOrElse("1", Nil).count(_ == "a") // => 1
       
    87 
       
    88 // test cases
       
    89 //val ratings_map = groupById(good_ratings, Map())
    86 //val ratings_map = groupById(good_ratings, Map())
    90 //groupById(good_ratings, Map()).get("214")
       
    91 //groupById(good_ratings, Map()).toList.minBy(_._2.length)
       
    92 //val movies_map = movie_names.toMap
    87 //val movies_map = movie_names.toMap
    93 
    88 
    94 //ratings_map.get("414").get.map(movies_map.get(_)) // most prolific recommender with 1227 positive ratings
    89 //ratings_map.get("414").get.map(movies_map.get(_)) 
    95 //ratings_map.get("474").get.map(movies_map.get(_)) // second-most prolific recommender with 787 positive ratings
    90 //    => most prolific recommender with 1227 positive ratings
    96 //ratings_map.get("214").get.map(movies_map.get(_)) // least prolific recommender with only 1 positive rating
    91 
    97 
    92 //ratings_map.get("474").get.map(movies_map.get(_)) 
    98 
    93 //    => second-most prolific recommender with 787 positive ratings
    99 
    94 
   100 //(4) Implement a function that takes a ratings map and a movie_name as argument.
    95 //ratings_map.get("214").get.map(movies_map.get(_)) 
   101 // The function calculates all suggestions containing
    96 //    => least prolific recommender with only 1 positive rating
   102 // the movie mov in its recommendations. It ReTurns a list of all these
    97 
   103 // recommendations (each of them is a list and needs to have mov deleted, 
    98 
   104 // otherwise it might happen we recommend the same movie).
    99 
   105 
   100 // (4) Implement a function that takes a ratings map and a movie_name as argument.
   106 def favourites(m: Map[String, List[String]], mov: String) : List[List[String]] = 
   101 //     The function calculates all suggestions containing
   107   (for (id <- m.keys.toList;
   102 //     the movie in its recommendations. It ReTurns a list of all these
   108         if m(id).contains(mov)) yield m(id).filter(_ != mov))
   103 //     recommendations (each of them is a list and needs to have the movie deleted, 
   109 
   104 //     otherwise it might happen we recommend the same movie).
   110 
   105 
   111 
   106 
   112 // test cases
   107 def favourites(m: Map[String, List[String]], mov: String) : List[List[String]] = {
       
   108     val movieLists = m.map(r => r._2).toList.filter(_.contains(mov))
       
   109     for (movieList <- movieLists) yield {
       
   110         movieList.filter(_!=mov)
       
   111     }
       
   112 }
       
   113 
       
   114 
       
   115 // testcases
       
   116 //-----------
   113 // movie ID "912" -> Casablanca (1942)
   117 // movie ID "912" -> Casablanca (1942)
   114 //          "858" -> Godfather
   118 //          "858" -> Godfather
   115 //          "260" -> Star Wars: Episode IV - A New Hope (1977)
   119 //          "260" -> Star Wars: Episode IV - A New Hope (1977)
   116 
   120 
   117 //favourites(ratings_map, "912").length  // => 80
   121 //favourites(ratings_map, "912").length  // => 80
   119 // That means there are 80 users that recommend the movie with ID 912.
   123 // That means there are 80 users that recommend the movie with ID 912.
   120 // Of these 80  users, 55 gave a good rating to movie 858 and
   124 // Of these 80  users, 55 gave a good rating to movie 858 and
   121 // 52 a good rating to movies 260, 318, 593.
   125 // 52 a good rating to movies 260, 318, 593.
   122 
   126 
   123 
   127 
       
   128 
   124 // (5) Implement a suggestions function which takes a rating
   129 // (5) Implement a suggestions function which takes a rating
   125 // map and a movie_name as arguments. It calculates all the recommended
   130 //     map and a movie_name as arguments. It calculates all the recommended
   126 // movies sorted according to the most frequently suggested movie(s) first.
   131 //     movies sorted according to the most frequently suggested movie(s) first.
   127 
   132 
   128 // needed in Scala 2.13.
       
   129  
       
   130 def mapValues[S, T, R](m: Map[S, T], f: T => R) =
       
   131   m.map { case (x, y) => (x, f(y)) }
       
   132 
   133 
   133 def suggestions(recs: Map[String, List[String]], 
   134 def suggestions(recs: Map[String, List[String]], 
   134                     mov_name: String) : List[String] = {
   135                 mov_name: String) : List[String] = {
   135   val favs = favourites(recs, mov_name).flatten
   136     val favs = favourites(recs, mov_name).flatten
   136   val favs_counted = mapValues(favs.groupBy(identity), (v:List[String]) => v.size).toList
   137     favs.map(x => (x, favs.count(_==x)))
   137   val favs_sorted = favs_counted.sortBy(_._2).reverse
   138         .sortBy(_._1)
   138   favs_sorted.map(_._1)
   139         .reverse
   139 }
   140         .sortBy(_._2)
   140 
   141         .reverse
   141 // check
   142         .distinct
   142 // groupMap is equivalent to groupBy(key).mapValues(_.map(f))
   143         .map(_._1)
   143 
   144 }
   144 // test cases
   145 
       
   146 
       
   147 // testcases
       
   148 //-----------
   145 
   149 
   146 //suggestions(ratings_map, "912")
   150 //suggestions(ratings_map, "912")
   147 //suggestions(ratings_map, "912").length  
   151 //suggestions(ratings_map, "912").length  
   148 // => 4110 suggestions with List(858, 260, 318, 593, ...)
   152 // => 4110 suggestions with List(858, 260, 318, 593, ...)
   149 //    being the most frequently suggested movies
   153 //    being the most frequently suggested movies
   150 
   154 
   151 // (6) Implement recommendations functions which generates at most
   155 
   152 // *two* of the most frequently suggested movies. It Returns the 
   156 
   153 // actual movie names, not the movieIDs.
   157 // (6) Implement a recommendations function which generates at most
       
   158 //     *two* of the most frequently suggested movies. It ReTurns the 
       
   159 //     actual movie names, not the movieIDs.
       
   160 
   154 
   161 
   155 def recommendations(recs: Map[String, List[String]],
   162 def recommendations(recs: Map[String, List[String]],
   156                    movs: Map[String, String],
   163                     movs: Map[String, String],
   157                    mov_name: String) : List[String] =
   164                     mov_name: String) : List[String] = {
   158   suggestions(recs, mov_name).take(2).map(movs.get(_).get)                 
   165     val sug = suggestions(recs, mov_name)
   159 
   166     val toptwo = sug.take(2)
   160 
   167     if (toptwo.length == 0) Nil
   161 // testcases
   168     else toptwo.map(movs(_))
   162 
   169 }
       
   170 
       
   171 
       
   172 
       
   173 // testcases
       
   174 //-----------
   163 // recommendations(ratings_map, movies_map, "912")
   175 // recommendations(ratings_map, movies_map, "912")
   164 //   => List(Godfather, Star Wars: Episode IV - A NewHope (1977))
   176 //   => List(Godfather, Star Wars: Episode IV - A NewHope (1977))
   165 
   177 
   166 //recommendations(ratings_map, movies_map, "260")
   178 //recommendations(ratings_map, movies_map, "260")
   167 //   => List(Star Wars: Episode V - The Empire Strikes Back (1980), 
   179 //   => List(Star Wars: Episode V - The Empire Strikes Back (1980), 
   175 
   187 
   176 // recommendations(ratings_map, movies_map, "1")
   188 // recommendations(ratings_map, movies_map, "1")
   177 //   => List(Shawshank Redemption, Forrest Gump (1994))
   189 //   => List(Shawshank Redemption, Forrest Gump (1994))
   178 
   190 
   179 // recommendations(ratings_map, movies_map, "4")
   191 // recommendations(ratings_map, movies_map, "4")
   180 //   => Nil  (there are three ratings fro this movie in ratings.csv but they are not positive)     
   192 //   => Nil  (there are three ratings for this movie in ratings.csv but they are not positive)     
   181 
   193 
   182 
   194 
   183 // If you want to calculate the recomendations for all movies.
   195 // If you want to calculate the recommendations for all movies,
   184 // Will take a few seconds calculation time.
   196 // then use this code (it will take a few seconds calculation time).
   185 
   197 
   186 //val all = for (name <- movie_names.map(_._1)) yield {
   198 //val all = for (name <- movie_names.map(_._1)) yield {
   187 //  recommendations(ratings_map, movies_map, name)
   199 //  recommendations(ratings_map, movies_map, name)
   188 //}
   200 //}
   189 
   201 
   191 //List().take(2)
   203 //List().take(2)
   192 //List(1).take(2)
   204 //List(1).take(2)
   193 //List(1,2).take(2)
   205 //List(1,2).take(2)
   194 //List(1,2,3).take(2)
   206 //List(1,2,3).take(2)
   195 
   207 
   196 }
   208 
       
   209 
       
   210 }
       
   211 
       
   212 // val ratings_url = """https://nms.kcl.ac.uk/christian.urban/ratings.csv"""
       
   213 // val movies_url = """https://nms.kcl.ac.uk/christian.urban/movies.csv"""
       
   214 
       
   215 // val ratings = CW7b.get_csv_url(ratings_url)
       
   216 // val movies = CW7b.get_csv_url(movies_url)
       
   217 
       
   218 // println(movies.length)
       
   219 // val good_ratings = CW7b.process_ratings(ratings)
       
   220 // val movie_names = CW7b.process_movies(movies)
       
   221 
       
   222 // val ratings_map = CW7b.groupById(good_ratings, Map())
       
   223 // val movies_map = movie_names.toMap
       
   224 
       
   225 
       
   226 
       
   227 //println(CW7b.recommendations(ratings_map, movies_map, "912"))
       
   228 /*
       
   229 val ratings_url = """https://nms.kcl.ac.uk/christian.urban/ratings.csv"""
       
   230 
       
   231 val ratings = CW7b.get_csv_url(ratings_url)
       
   232 
       
   233 val good_ratings = CW7b.process_ratings(ratings)
       
   234 val ratings_map = CW7b.groupById(good_ratings, Map())
       
   235 
       
   236 println(CW7b.suggestions(ratings_map, "912").length)*/