#!/usr/bin/python
# -*- coding: utf8 -*-

# National Library of Norway, 2014-2015

# load the packages
from pysqlite2 import dbapi2 as sqlite3
from collections import Counter
from operator import itemgetter
from itertools import chain
from flask import Flask, Response, request, session, g, redirect, url_for, \
     abort, render_template, flash, jsonify
from contextlib import closing
import re
import json
import sys
import operator
import itertools

## CONFIGURATION
# path to databases
try:
    path = str(sys.argv[1])
except:
    path = ''

# specify port (default: 5000)
try:
    port = int(sys.argv[2])
except:
    port = 5000

# specify host (default: 127.0.0.1)
try:
    host = str(sys.argv[3])
except:
    host = '127.0.0.1'

# paths for the databases
UNIGRAM = path + 'unigram-one-row.db'
BIGRAM = path + 'bigram-one-row.db'
TRIGRAM = path + 'trigram-one-row.db'
AVIS_UNIGRAM = path + 'avis-unigram-one-row.db'
AVIS_BIGRAM = path + 'avis-bigram-one-row.db'
AVIS_TRIGRAM = path + 'avis-trigram-one-row.db'

# database structure
db_names = {'bok': 'bok_', 'avis': 'avis_'}
table_names = ['unigram', 'bigram', 'trigram']
index_names = {'bok': ['_lff_','_lfsf_','_lfstf_'], 'avis': ['_ff_','_fsf_','_fstf_']}
field_names = ['first', 'second', 'third']

# Allowed paramaters
languages = 'all|nob|nno'
corpora = 'bok|avis'

# Default paramaters
default_params = {'terms': '', 'lang': 'all', 'case_sens': '0', 'freq': 'rel', 'corpus': 'bok'};

# Maximum values
maxTerms = 10
maxNgram = 3 # currently, only unigram, bigram, trigram is supported
maxChar = 200 # cut-off-point at 200 characters for query string
maxWildcards = 5
maxAdd = 10
maxTrunct = 5

# loads a JSON object holding the max. frequencies per year for calculation of relative frequency in python (optional: you might want to store these in the database itself)
with open ('totals.json', 'r') as f:
    freqs_per_year = json.load(f)

# initiating Flask (with settings from environment variable - for use in development and production environments)
app = Flask(__name__, static_url_path='/ngram/static')
app.config.from_object(__name__)
app.config.from_envvar('FLASK_NGRAM_SETTINGS')

# connection to DB
def connect_db(self):
    rv = sqlite3.connect(self)
    #rv.row_factory = sqlite3.Row
    return rv


@app.before_request
def before_request():
    """ establish connection upon request """
    g.db = connect_db(UNIGRAM)
    
    # Attach databases
    g.db.execute("ATTACH DATABASE '" + UNIGRAM + "' as bok_unigram;")
    g.db.execute("ATTACH DATABASE '" + BIGRAM + "' as bok_bigram;")
    g.db.execute("ATTACH DAT
ABASE '" + TRIGRAM + "' as bok_trigram;")
    g.db.execute("ATTACH DATABASE '" + AVIS_UNIGRAM + "' as avis_unigram;")
    g.db.execute("ATTACH DATABASE '" + AVIS_BIGRAM + "' as avis_bigram")
    g.db.execute("ATTACH DATABASE '" + AVIS_TRIGRAM + "' as avis_trigram")

@app.after_request
def after_request(response):
    """ Close connection after request """
    
    g.db.close()
    return response

def query_db_dict(query, args=(), one=False):
    """ Return results as dictionary """
    
    cur = g.db.execute(query, args)
    rv = [dict((cur.description[idx][0], value)
               for idx, value in enumerate(row)) for row in cur.fetchall()]
    return (rv[0] if rv else None) if one else rv

def query_db_row(query, args=(), one=False):
    """ Return results as rows """
    
    cur = g.db.execute(query, args)
    rv = [list((value)
                for idx, value in enumerate(row)) for row in cur.fetchall()]
    return (rv[0] if rv else None) if one else rv

def return_terms(terms):
    """Gets a string of terms and returns them as a list, with some clean-up"""
    
    # index for wildcards (not allowed to exceed maxWildcards, these ones are powerful)
    wildcardIdx = 0
    # we only allow a certain amount of characters in the terms string
    terms = terms[:maxChar]
    # removes unnecessary whitespace or empty query terms
    terms = re.sub(r',\s{0,},',',', terms)
    # splits on comma (with following whitespace): commas may be masked by quoatation marks
    terms = re.findall('[^\,\"]+|\"[^"]*\"', terms)
    # gets number of terms
    nTerms = len(terms)
    # checks if number exceeds maxTerms, remaining ones are removed
    if nTerms >= maxTerms:
        terms = terms[:maxTerms]
        nTerms = maxTerms
    # loops through each term
    for i in range(nTerms):
        # substitutes '*' with '%' for SQL queries, removes illegal wildcards (according to maxWildcards)
        if "*" in terms[i] and wildcardIdx < maxWildcards:
            wildcardIdx += 1
            terms[i] = terms[i].replace("*", "%")
        else:
            terms[i] = terms[i].replace("*", "")
        # removes whitespace at the beginning or the end of the string
        terms[i] = re.sub(r'^\s+', '', terms[i])
        terms[i] = re.sub(r'\s+$', '', terms[i])
        # removes mask for comma
        if terms[i] == '","':
            terms[i] = re.sub(r'","',',', terms[i])
        # removes whitespace between '+' and terms
        if "+" in terms[i]:
            terms[i] = re.sub(r'\s+\+', '+', terms[i])
            terms[i] = re.sub(r'\+\s+', '+', terms[i])
    
    return terms

def query_factory(ngrams, lang, case_sens, corpus):
    """ Creates a sql query for each item in the object """
    
    sql_array = []
    args_array = []
    label_array = []
    lang_array = []
    corpus_array = []

    for ngram in ngrams:
        sql, args, query_lang, query_corpus = build_query_single(ngram, lang, case_sens, corpus)
        sql_array.append(sql)
        args_array.append(args)
        label_array.append(' '.join(ngram))
        lang_array.append(query_lang)
        corpus_array.append(query_corpus)

    return sql_array, args_array, label_array, lang_array, corpus_array

def extract_info(term):
    """ Extracts information after colon, returns only ngram and dictionary of arguments"""
    
    ngram = []
    argumentDict = {}
    lastElement = len(term) - 1
    splitted = term[lastElement].split(':')

    if len(splitted) >= 2:
        ngram.extend(term[:lastElement])
        ngram.extend([splitted[0]])

        extension = splitted[1:]

        for element in extension:
            if re.match(r'nob|nno|all', element):
                argumentDict['lang'] = element
            if re.match(r'bok|avis', element):
                argumentDict["db"]  = element
                if re.match (r'avis', element):
                    argumentDict["lang"]  = 'all'
                if re.match (r'bok', element) and re.match(r'nob|nno|all', element) != -1:
                    argumentDict["lang"]  = 'all'
            if re.match(r'[0-9]{4}', element):
                argumentDict["anno"]  = element

        return ngram, argumentDict

def wildcard_search(ngrams, lang, case_sens, corpus):
    """ Returns the ten most common ngrams matching query """
    
    whereClause = []
    whereParams = []
    args = []
    ngramSize = len(ngrams)
    argumentDict = {"ngram": [], "lang": lang, "db": corpus}

    if extract_info(ngrams) != None:
        ngrams, argumentsExtracted = extract_info(ngrams)
        argumentDict = dict_merge(argumentDict, argumentsExtracted)

    # values based on input
    params = 'in (?,?)' if case_sens == '0' else 'in (?)'
    langClause = 'and lang = ?' if argumentDict["lang"] != "all" else ''
    getFieldNames = ', '.join(field_names[:ngramSize])
    getTableNames = db_names[argumentDict["db"]] + table_names[ngramSize-1] + "." + table_names[ngramSize-1]

    for ngram in ngrams:
        if "%" in ngram:
            argumentDict["ngram"].append(ngram)
            whereParams.append("LIKE ?")
        else:
            whereParams.append(params)
            if case_sens == '0':
                argumentDict["ngram"].extend(swapcase([ngram]))
            else:
                argumentDict["ngram"].append(ngram)

    idxName = query_planner(whereParams,argumentDict["ngram"])

    whereClause = " and ".join( list(('(%s %s)' % (field_names[idx],whereParams[idx]))
                    for idx, val in enumerate(ngrams)) ) + (langClause if argumentDict["db"] == 'bok' else '') 

    sql = "SELECT DISTINCT %s FROM (SEL