<reponame>Arko98/Spam_SMS_Prediction

def load(file_name):
    import pandas as pd
    import os
    #Preparing dataset as type and text sms
    name_list = ['Type','Text']
    path = os.getcwd()+'/'+file_name
    data_frame = pd.read_csv(path, sep = ',', header = None, names = name_list)
    data_frame = data_frame.drop([0], axis = 0)
    return data_frame

def create_labels(data_frame): 
    '''Enamurating the gold labels and storing in a seperated dataframe'''
    import numpy as np
    data_frame["Type"] = data_frame["Type"].astype('category')
    data_frame["Type"] = data_frame["Type"].cat.codes
    data_frame["Type"] = data_frame["Type"].astype('int64')
    labels = np.reshape(np.array(data_frame["Type"]),(len(data_frame["Type"]),1))
    return labels

def make_word_container(data_frame):
    '''function arguments are data_frame and a string which is column of data_frame fow which word_container
       is to be constructed'''
    import re
    max_len_sentence = 0
    word_container = []
    sentence_list = list(data_frame['Text'])
    for i in range(len(sentence_list)):
        w = sentence_list[i]
        word_container.append(re.split('\W',str(w)))
    for sentence in range(len(word_container)):
        for word in word_container[sentence]:
            if word == '':
                word_container[sentence].remove('')
    for i in range(len(word_container)):
        length = len(word_container[i])
        if length >= max_len_sentence:
            max_len_sentence = length
    return word_container,max_len_sentence

def No_Word_padding(word_container, max_len_sentence):
    '''function for padding no_word so that shape remains equal for all word_vec 
       that will be obtained after Word2Vec embedding. The argument max_len_sentence is 
       global max length obtained from 'max_dataset_sentence_length' function. Use after concatenate'''
    for i in range(len(word_container)):
        while len(word_container[i]) < max_len_sentence:
            word_container[i].append('nan')
    return word_container

def Word2Vec(word_container, word_vector_size, min_count):
    '''this function takes arguments word_container, min count it returns the word2vec model, 
       vocabulary.'''
    import numpy as np
    import gensim
    from gensim.models import Word2Vec
    #model training
    model = Word2Vec(word_container, size = word_vector_size, min_count = min_count)
    print('Word2Vec Model details = {}'.format(model))
    vocabs = list(model.wv.vocab)
    return model, vocabs

def Google_Word2Vec():
    '''this function  returns the Google Word2Vec model'''
    import os
    import gensim
    from gensim.models import Word2Vec
    path = os.getcwd()
    google_model = gensim.models.KeyedVectors.load_word2vec_format(path + '/GoogleNews-vectors-negative300.bin', binary=True)  
    google_vocabs = list(google_model.wv.vocab)
    return google_model, google_vocabs

def Word_Embedding(model, word_vector_size, word_container):
    '''function to embedd words to vectors according to the input word2vec model (model can be local or Google Word2Vec)
       model. It returns word_vector_matrix (zero padded) that can be directly fed to LSTM, GRU etc.
       use after concatenate, No_word padding and then after Word2Vec
       Note ::  While using Google_Word2Vec, make vector size = 300'''
    import numpy as np
    sentence_number = len(word_container)
    vector_size = word_vector_size
    max_len_sentence = len(word_container[1])
    for i in range(len(word_container)):
        for j in range(len(word_container[i])):
            try:
                a = model[word_container[i][j]]
            except KeyError:
                word_container[i][j] = 'nan'
    word_vector_matrix = np.zeros(shape = (sentence_number,max_len_sentence,vector_size), dtype = np.float32)
    for i in range(sentence_number):
        for j in range(max_len_sentence):
            word_vector_matrix[i][j] = model[word_container[i][j]]
    return word_vector_matrix


def load(file_name):
    import pandas as pd
    import os
    #Preparing dataset as type and text sms
    name_list = ['Type','Text']
    path = os.getcwd()+'/'+file_name
    data_frame = pd.read_csv(path, sep = ',', header = None, names = name_list)
    data_frame = data_frame.drop([0], axis = 0)
    return data_frame

def create_labels(data_frame): 
    '''Enamurating the gold labels and storing in a seperated dataframe'''
    import numpy as np
    data_frame["Type"] = data_frame["Type"].astype('category')
    data_frame["Type"] = data_frame["Type"].cat.codes
    data_frame["Type"] = data_frame["Type"].astype('int64')
    labels = np.reshape(np.array(data_frame["Type"]),(len(data_frame["Type"]),1))
    return labels

def make_word_container(data_frame):
    '''function arguments are data_frame and a string which is column of data_frame fow which word_container
       is to be constructed'''
    import re
    max_len_sentence = 0
    word_container = []
    sentence_list = list(data_frame['Text'])
    for i in range(len(sentence_list)):
        w = sentence_list[i]
        word_container.append(re.split('\W',str(w)))
    for sentence in range(len(word_container)):
        for word in word_container[sentence]:
            if word == '':
                word_container[sentence].remove('')
    for i in range(len(word_container)):
        length = len(word_container[i])
        if length >= max_len_sentence:
            max_len_sentence = length
    return word_container,max_len_sentence

def No_Word_padding(word_container, max_len_sentence):
    '''function for padding no_word so that shape remains equal for all word_vec 
       that will be obtained after Word2Vec embedding. The argument max_len_sentence is 
       global max length obtained from 'max_dataset_sentence_length' function. Use after concatenate'''
    for i in range(len(word_container)):
        while len(word_container[i]) < max_len_sentence:
            word_container[i].append('nan')
    return word_container

def Word2Vec(word_container, word_vector_size, min_count):
    '''this function takes arguments word_container, min count it returns the word2vec model, 
       vocabulary.'''
    import numpy as np
    import gensim
    from gensim.models import Word2Vec
    #model training
    model = Word2Vec(word_container, size = word_vector_size, min_count = min_count)
    print('Word2Vec Model details = {}'.format(model))
    vocabs = list(model.wv.vocab)
    return model, vocabs

def Google_Word2Vec():
    '''this function  returns the Google Word2Vec model'''
    import os
    import gensim
    from gensim.models import Word2Vec
    path = os.getcwd()
    google_model = gensim.models.KeyedVectors.load_word2vec_format(path + '/GoogleNews-vectors-negative300.bin', binary=True)  
    google_vocabs = list(google_model.wv.vocab)
    return google_model, google_vocabs

def Word_Embedding(model, word_vector_size, word_container):
    '''function to embedd words to vectors according to the input word2vec model (model can be local or Google Word2Vec)
       model. It returns word_vector_matrix (zero padded) that can be directly fed to LSTM, GRU etc.
       use after concatenate, No_word padding and then after Word2Vec
       Note ::  While using Google_Word2Vec, make vector size = 300'''
    import numpy as np
    sentence_number = len(word_container)
    vector_size = word_vector_size
    max_len_sentence = len(word_container[1])
    for i in range(len(word_container)):
        for j in range(len(word_container[i])):
            try:
                a = model[word_container[i][j]]
            except KeyError:
                word_container[i][j] = 'nan'
    word_vector_matrix = np.zeros(shape = (sentence_number,max_len_sentence,vector_size), dtype = np.float32)
    for i in range(sentence_number):
        for j in range(max_len_sentence):
            word_vector_matrix[i][j] = model[word_container[i][j]]
    return word_vector_matrix
#Preparing dataset as type and text sms Enamurating the gold labels and storing in a seperated dataframe function arguments are data_frame and a string which is column of data_frame fow which word_container        is to be constructed function for padding no_word so that shape remains equal for all word_vec         that will be obtained after Word2Vec embedding. The argument max_len_sentence is         global max length obtained from 'max_dataset_sentence_length' function. Use after concatenate this function takes arguments word_container, min count it returns the word2vec model,         vocabulary. #model training this function  returns the Google Word2Vec model function to embedd words to vectors according to the input word2vec model (model can be local or Google Word2Vec)        model. It returns word_vector_matrix (zero padded) that can be directly fed to LSTM, GRU etc.        use after concatenate, No_word padding and then after Word2Vec        Note ::  While using Google_Word2Vec, make vector size = 300