import datetime
from django.http import HttpResponse
import logging
from natsort import natsorted
import pandas as pd
from PyPDF2 import PdfFileReader, PdfFileMerger
from pyper import *
import json

from database.models import ko_entry

import functions


LOG_FILENAME = 'error_log.txt'
pd.set_option('display.max_colwidth', -1)


def getGAGE(request, stops, RID, PID):
    try:
        if request.is_ajax():
            # Get variables from web page
            allJson = request.body.split('&')[0]
            all = json.loads(allJson)
            functions.setBase(RID, 'Step 1 of 6: Reading normalized data file...')

            # Select samples and meta-variables from savedDF
            functions.setBase(RID, 'Step 2 of 6: Selecting your chosen meta-variables...')
            metaValsCat = all['metaValsCat']
            metaIDsCat = all['metaIDsCat']
            metaValsQuant = []
            metaIDsQuant = []

            DepVar = int(all["DepVar"])

            # Create meta-variable DataFrame, final sample list, final category and quantitative field lists based on tree selections
            savedDF, metaDF, finalSampleIDs, catFields, remCatFields, quantFields, catValues, quantValues = functions.getMetaDF(request.user, metaValsCat, metaIDsCat, metaValsQuant, metaIDsQuant, DepVar, levelDep=True)

            if not catFields:
                error = "Selected categorical variable(s) contain only one level.\nPlease select different variable(s)."
                myDict = {'error': error}
                res = json.dumps(myDict)
                return HttpResponse(res, content_type='application/json')

            if not finalSampleIDs:
                error = "No valid samples were contained in your final dataset.\nPlease select different variable(s)."
                myDict = {'error': error}
                res = json.dumps(myDict)
                return HttpResponse(res, content_type='application/json')

            result = ''
            result += 'Categorical variables selected by user: ' + ", ".join(catFields + remCatFields) + '\n'
            result += 'Categorical variables not included in the statistical analysis (contains only 1 level): ' + ", ".join(remCatFields) + '\n'
            result += 'Quantitative variables selected by user: ' + ", ".join(quantFields) + '\n'
            result += '===============================================\n\n'

            functions.setBase(RID, 'Step 2 of 6: Selecting your chosen meta-variables...done')

            # /\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\//\ #
            if stops[PID] == RID:
                res = ''
                return HttpResponse(res, content_type='application/json')
            # /\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\//\ #

            functions.setBase(RID, 'Step 3 of 6: Mapping phylotypes to KEGG pathways...')

            if os.name == 'nt':
                r = R(RCMD="R/R-Portable/App/R-Portable/bin/R.exe", use_pandas=True)
            else:
                r = R(RCMD="R/R-Linux/bin/R", use_pandas=True)

            functions.setBase(RID, 'Verifying R packages...missing packages are being installed')

            # R packages from biocLite
            r("list.of.packages <- c('gage', 'edgeR', 'pathview')")
            r("new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,'Package'])]")
            r("if (length(new.packages)) source('http://bioconductor.org/biocLite.R')")
            r("if (length(new.packages)) biocLite(new.packages, type='source', suppressUpdate=T, dependencies=T)")

            # R packages from cran
            r("list.of.packages <- c('png', 'grid', 'plyr')")
            r("new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,'Package'])]")
            r("if (length(new.packages)) install.packages(new.packages, repos='http://cran.us.r-project.org', dependencies=T)")

            functions.setBase(RID, 'Step 3 of 6: Mapping phylotypes to KEGG pathways...')

            r("library(gage)")
            r("library(edgeR)")
            r("library(pathview)")
            r("library(png)")
            r("library(grid)")
            r("library(plyr)")

            keggString = all["kegg"]
            keggDict = json.JSONDecoder(object_pairs_hook=functions.multidict).decode(keggString)
            nameList = []
            for value in keggDict.itervalues():
                if isinstance(value, list):
                    nameList.extend(value)
                else:
                    nameList.append(value)

            # Enable this only if you want to update gage data and pathways
            '''
            r("list.of.packages <- c('gageData')")
            r("new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,'Package'])]")
            r("if (length(new.packages)) source('http://bioconductor.org/biocLite.R')")
            print r("if (length(new.packages)) biocLite(new.packages)")

            r('library(gageData)')
            r("data(kegg.sets.ko)")
            r("save(kegg.sets.ko, file='myPhyloDB/media/kegg/kegg.sets.ko.RData')")
            r("for (name in names(kegg.sets.ko)) { \
                id = substr(name, 3, 7); \
                download.kegg(pathway
.id=id, species='ko', kegg.dir='myPhyloDB/media/kegg/pathways', file.type=c('xml', 'png')) \
                } \
            ")
       
     '''

            r("load('myPhyloDB/media/kegg/kegg.sets.ko.RData')")

            keggDict = {}
            r("selPaths <- vector()")
            for i in nameList:
                pathStr = i.split('[PATH:')[1].split(']')[0]
                r.assign("pathStr", pathStr)
                r("selPath <- kegg.sets.ko[grepl(paste(pathStr), names(kegg.sets.ko))]")
                key = r.get("names(selPath)")
                value = r.get("selPath$ko")
                keggDict[key] = value.tolist()
                r("selPaths <- append(selPaths, names(selPath))")

                # /\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\//\ #
                if stops[PID] == RID:
                    res = ''
                    return HttpResponse(res, content_type='application/json')
                # /\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\//\ #

            keggAll = 4
            mapTaxa = 'no'
            finalDF, junk = functions.getKeggDF(keggAll, '', savedDF, metaDF, DepVar, mapTaxa, RID, stops, PID)

            # make sure column types are correct
            finalDF[catFields] = finalDF[catFields].astype(str)

            functions.setBase(RID, 'Step 4 of 6: Performing GAGE analysis...')

            # save location info to session
            myDir = 'myPhyloDB/media/temp/gage/'
            if not os.path.exists(myDir):
                os.makedirs(myDir)

            path = str(myDir) + str(RID) + '.biom'
            functions.imploding_panda(path, 2, DepVar, finalSampleIDs, metaDF, finalDF)

            count_rDF = pd.DataFrame()
            if DepVar == 0:
                count_rDF = finalDF.pivot(index='rank_id', columns='sampleid', values='abund')
            elif DepVar == 4:
                count_rDF = finalDF.pivot(index='rank_id', columns='sampleid', values='abund_16S')

            # need to change rank_id to kegg orthologies for gage analysis
            count_rDF.reset_index(drop=False, inplace=True)
            count_rDF.rename(columns={'index': 'rank_id'}, inplace=True)
            idList = count_rDF.rank_id.tolist()
            idDict = {}
            for id in idList:
                entry = ko_entry.objects.using('picrust').get(ko_lvl4_id=id).ko_orthology
                idDict[id] = entry
            count_rDF['ko'] = count_rDF['rank_id'].map(idDict)
            count_rDF.drop('rank_id', axis=1, inplace=True)
            count_rDF.drop_duplicates(keep='last', inplace=True)  # remove dups - KOs mapped to multiple pathways
            count_rDF.set_index('ko', drop=True, inplace=True)

            # make metaDF R compatible, remove offending characters in categorical variables
            for cat in catFields:
                metaDF[cat] = metaDF[ca