# -*- coding: utf-8 -*-
"""Matrix Completion.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wvA7Zu8pgABvBCoOuu498QpRkKbSvFC_

# Matrix Completion using SVD Matrix Factorization

Based on https://beckernick.github.io/matrix-factorization-recommender/

Data: 1 million movie ratings available from the MovieLens project. 
The MovieLens datasets were collected by GroupLens Research at the University of Minnesota.

# Preprocessing
"""

import numpy as np
from scipy.sparse.linalg import norm
import scipy.sparse as ss
import random
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import mean_squared_error

ratings_list = [i.strip().split("::") for i in open('/content/ratings.dat', 'r').readlines()]
ratings_df = pd.DataFrame(ratings_list, columns = ['UserID', 'MovieID', 'Rating', 'Timestamp'], dtype = int)
ratings_df['Rating']=ratings_df['Rating'].apply(pd.to_numeric)

random.seed(123)

ratings_df.head()

print("#users: "+ str(ratings_df.UserID.nunique()))
print("#movies: "+ str(ratings_df.MovieID.nunique()))
print("#ratings %: "+ str(ratings_df.shape[0]/(ratings_df.UserID.nunique()*ratings_df.MovieID.nunique())*100))

ratings_df.Rating.describe()

"""pivot ratings_df to get user-rating matrix format"""

R_df = ratings_df.pivot(index = 'UserID', columns ='MovieID', values = 'Rating').fillna(0)
R_df.head()

"""de-mean the data (normalize by each users mean) and convert it from a dataframe to a numpy array."""

#R = R_df.as_matrix()
R = R_df.values
user_ratings_mean = np.mean(R, axis = 1)
R_demeaned = R - user_ratings_mean.reshape(-1, 1)

"""# Sample test data

create test data by sampling (and removing) 500 ratings
"""

rated_indices = np.argwhere(R_demeaned>0)

test_set = random.sample(list(rated_indices),2500)

print("1st index is "+str(test_set[0])+" and its rating is "+str(R_demeaned[test_set[0][0],test_set[0][1]]))

test_ratings = []
for i in range(len(test_set)):
    test_ratings.append(R_demeaned[test_set[i][0],test_set[i][1]])
    R_demeaned[test_set[i][0],test_set[i][1]]=0

print("1st index is "+str(test_set[0])+" and its rating is "+str(R_demeaned[test_set[0][0],test_set[0][1]]))

"""# Singular Value Decomposition

Scipy function svds allow to choose the number of latent factors

preprocessing our Matrix:
"""

m1,m2 = R_demeaned.shape

rated_indices = np.argwhere(R_demeaned>0)
n1, n2 = rated_indices.shape

users = []
movies = []
ratings = []

for i in range(n1):
  users.append(rated_indices[i][0])
  movies.append(rated_indices[i][1])
  ratings.append(R_demeaned[rated_indices[i][0],rated_indices[i][1]])

Matrix = ss.csr_matrix((ratings, (users, movies)), shape=(m1, m2))

def SVT(Matrix, iter_num, t, delta, epsilon, lamda):
    n1, n2 = Matrix.shape #rows and columes of the random Matrix
    total_num = len(Matrix.nonzero()[0])
    test_set = random.sample(range(total_num), total_num)
    omega = (Matrix.nonzero()[0][test_set], Matrix.nonzero()[1][test_set]) # sample set

# recover a low-rank Matrix from a subset of sampled entries 
    r_0 = 0
    a = Matrix[omega]
    P_omega_M = ss.csr_matrix((np.ravel(a), omega), shape=(n1, n2)) # sample entries, the sparse matrix we need to decompose with
    normProjM = norm(P_omega_M)
    k_0 = np.ceil(t / (delta * normProjM))
    Y_0 = k_0 * delta * P_omega_M
    i = 0
    rmse = []

    for k in range(iter_num):
        s_k = r_0 + 1
        while True:
            u0, s0, v0 = sparsesvd(ss.csc_matrix(Y_0), s_k)
            if s0[s_k - 1] <= t: 
              break
            s_k = min(s_k + lamda, n1, n2)
            if s_k == min(n1, n2): 
              break
        
        r_0 = np.sum(s0 > t)
        U = u0.T[:, :r_0]
        V = v0[:r_0, :]
        S = s0[:r_0] - t
        x = (U * S).dot(V)
        P_omega_x_k = ss.csr_matrix((x[omega], omega), shape=(n1, n2))
        if norm(P_omega_x_k - P_omega_M) / norm(P_omega_M) < epsilon:
            break
        delt = P_omega_M - P_omega_x_k
        Y_0 += delta * delt
        rmse_t = float(np.linalg.norm(Matrix[Matrix.nonzero()] - x[Matrix.nonzero()]) / np.sqrt(len(x[Matrix.nonzero()])))
        #print('Iter %d , RMSE %.3f' % (i, rmse_t))
        rmse.append(rmse_t)
        i += 1

    #return rmse, x
    return x

total_num = len(Matrix.nonzero()[0])
iter_num =[50,25,75]
t = [total_num / 100,total_num / 50,total_num / 200]
delta = [2, 1.5, 4]
epsilon = [0.001, 0.005, 0.05]
lamda = [5, 10, 50]
predicted_ratings = []

for a in iter_num:
  for b in t:
    for c in delta:
      for d in epsilon:
        for e in lamda:
          print(a, b, c, d, e)
          predicted_ratings.append(SVT(Matrix, a, b, c, d, e))

s = []
for i in range(len(predicted_ratings)):
  predicted_ratings[i] = predicted_ratings[i] + user_ratings_mean.reshape(-1, 1)
  preds_df = pd.DataFrame(predicted_ratings[i], columns = R_df.columns)
  pred_set = [max(predicted_ratings[i][x[0],x[1]],0) for x in test_set]
  s.append(np.sqrt(mean_squared_error(test_ratings,pred_set)))
print(s)

# -*- coding: utf-8 -*-
"""Matrix Completion.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wvA7Zu8pgABvBCoOuu498QpRkKbSvFC_

# Matrix Completion using SVD Matrix Factorization

Based on https://beckernick.github.io/matrix-factorization-recommender/

Data: 1 million movie ratings available from the MovieLens project. 
The MovieLens datasets were collected by GroupLens Research at the University of Minnesota.

# Preprocessing
"""

import numpy as np
from scipy.sparse.linalg import norm
import scipy.sparse as ss
import random
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import mean_squared_error

ratings_list = [i.strip().split("::") for i in open('/content/ratings.dat', 'r').readlines()]
ratings_df = pd.DataFrame(ratings_list, columns = ['UserID', 'MovieID', 'Rating', 'Timestamp'], dtype = int)
ratings_df['Rating']=ratings_df['Rating'].apply(pd.to_numeric)

random.seed(123)

ratings_df.head()

print("#users: "+ str(ratings_df.UserID.nunique()))
print("#movies: "+ str(ratings_df.MovieID.nunique()))
print("#ratings %: "+ str(ratings_df.shape[0]/(ratings_df.UserID.nunique()*ratings_df.MovieID.nunique())*100))

ratings_df.Rating.describe()

"""pivot ratings_df to get user-rating matrix format"""

R_df = ratings_df.pivot(index = 'UserID', columns ='MovieID', values = 'Rating').fillna(0)
R_df.head()

"""de-mean the data (normalize by each users mean) and convert it from a dataframe to a numpy array."""

#R = R_df.as_matrix()
R = R_df.values
user_ratings_mean = np.mean(R, axis = 1)
R_demeaned = R - user_ratings_mean.reshape(-1, 1)

"""# Sample test data

create test data by sampling (and removing) 500 ratings
"""

rated_indices = np.argwhere(R_demeaned>0)

test_set = random.sample(list(rated_indices),2500)

print("1st index is "+str(test_set[0])+" and its rating is "+str(R_demeaned[test_set[0][0],test_set[0][1]]))

test_ratings = []
for i in range(len(test_set)):
    test_ratings.append(R_demeaned[test_set[i][0],test_set[i][1]])
    R_demeaned[test_set[i][0],test_set[i][1]]=0

print("1st index is "+str(test_set[0])+" and its rating is "+str(R_demeaned[test_set[0][0],test_set[0][1]]))

"""# Singular Value Decomposition

Scipy function svds allow to choose the number of latent factors

preprocessing our Matrix:
"""

m1,m2 = R_demeaned.shape

rated_indices = np.argwhere(R_demeaned>0)
n1, n2 = rated_indices.shape

users = []
movies = []
ratings = []

for i in range(n1):
  users.append(rated_indices[i][0])
  movies.append(rated_indices[i][1])
  ratings.append(R_demeaned[rated_indices[i][0],rated_indices[i][1]])

Matrix = ss.csr_matrix((ratings, (users, movies)), shape=(m1, m2))

def SVT(Matrix, iter_num, t, delta, epsilon, lamda):
    n1, n2 = Matrix.shape #rows and columes of the random Matrix
    total_num = len(Matrix.nonzero()[0])
    test_set = random.sample(range(total_num), total_num)
    omega = (Matrix.nonzero()[0][test_set], Matrix.nonzero()[1][test_set]) # sample set

# recover a low-rank Matrix from a subset of sampled entries 
    r_0 = 0
    a = Matrix[omega]
    P_omega_M = ss.csr_matrix((np.ravel(a), omega), shape=(n1, n2)) # sample entries, the sparse matrix we need to decompose with
    normProjM = norm(P_omega_M)
    k_0 = np.ceil(t / (delta * normProjM))
    Y_0 = k_0 * delta * P_omega_M
    i = 0
    rmse = []

    for k in range(iter_num):
        s_k = r_0 + 1
        while True:
            u0, s0, v0 = sparsesvd(ss.csc_matrix(Y_0), s_k)
            if s0[s_k - 1] <= t: 
              break
            s_k = min(s_k + lamda, n1, n2)
            if s_k == min(n1, n2): 
              break
        
        r_0 = np.sum(s0 > t)
        U = u0.T[:, :r_0]
        V = v0[:r_0, :]
        S = s0[:r_0] - t
        x = (U * S).dot(V)
        P_omega_x_k = ss.csr_matrix((x[omega], omega), shape=(n1, n2))
        if norm(P_omega_x_k - P_omega_M) / norm(P_omega_M) < epsilon:
            break
        delt = P_omega_M - P_omega_x_k
        Y_0 += delta * delt
        rmse_t = float(np.linalg.norm(Matrix[Matrix.nonzero()] - x[Matrix.nonzero()]) / np.sqrt(len(x[Matrix.nonzero()])))
        #print('Iter %d , RMSE %.3f' % (i, rmse_t))
        rmse.append(rmse_t)
        i += 1

    #return rmse, x
    return x

total_num = len(Matrix.nonzero()[0])
iter_num =[50,25,75]
t = [total_num / 100,total_num / 50,total_num / 200]
delta = [2, 1.5, 4]
epsilon = [0.001, 0.005, 0.05]
lamda = [5, 10, 50]
predicted_ratings = []

for a in iter_num:
  for b in t:
    for c in delta:
      for d in epsilon:
        for e in lamda:
          print(a, b, c, d, e)
          predicted_ratings.append(SVT(Matrix, a, b, c, d, e))

s = []
for i in range(len(predicted_ratings)):
  predicted_ratings[i] = predicted_ratings[i] + user_ratings_mean.reshape(-1, 1)
  preds_df = pd.DataFrame(predicted_ratings[i], columns = R_df.columns)
  pred_set = [max(predicted_ratings[i][x[0],x[1]],0) for x in test_set]
  s.append(np.sqrt(mean_squared_error(test_ratings,pred_set)))
print(s)

# -*- coding: utf-8 -*- Matrix Completion.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1wvA7Zu8pgABvBCoOuu498QpRkKbSvFC_  # Matrix Completion using SVD Matrix Factorization  Based on https://beckernick.github.io/matrix-factorization-recommender/  Data: 1 million movie ratings available from the MovieLens project.  The MovieLens datasets were collected by GroupLens Research at the University of Minnesota.  # Preprocessing pivot ratings_df to get user-rating matrix format de-mean the data (normalize by each users mean) and convert it from a dataframe to a numpy array. #R = R_df.as_matrix() # Sample test data  create test data by sampling (and removing) 500 ratings # Singular Value Decomposition  Scipy function svds allow to choose the number of latent factors  preprocessing our Matrix: #rows and columes of the random Matrix # sample set # recover a low-rank Matrix from a subset of sampled entries # sample entries, the sparse matrix we need to decompose with #print('Iter %d , RMSE %.3f' % (i, rmse_t)) #return rmse, x