"""
preprocessing twitter dataset
"""
import time
import os
from os import path
from os.path import join, expanduser as expand
import json
import re
import csv
from collections import defaultdict
import argparse

import nltk

from preprocessing import twitter_airlines


all_companies = 'AppleSupport,Amazon,Uber_Support,SpotifyCares,Delta,Tesco,AmericanAir,TMobileHelp,comcastcares,British_Airways,SouthwestAir,VirginTrains,Ask_Spectrum,XboxSupport,sprintcare,hulu_support,AskPlayStation,ChipotleTweets,UPSHelp,AskTarget'.split(',')

"""
example rows:

tweet_id,author_id,inbound,created_at,text,response_tweet_id,in_response_to_tweet_id
1,sprintcare,False,Tue Oct 31 22:10:47 +0000 2017,@115712 I understand. I would like to assist you. We would need to get you into a private secured link to further assist.,2,3
2,115712,True,Tue Oct 31 22:11:45 +0000 2017,@sprintcare and how do you propose we do that,,1
"""

class NullPreprocessor(object):
    def __call__(self, text, info_dict=None):
        is_valid = True
        if 'DM' in text:
            is_valid = False
        if 'https://t.co' in text:
            is_valid = False

        text = text.replace('\n', ' ').replace('\r', ' ')

        target_company = info_dict['target_company']
        cust_twitter_id = info_dict['cust_twitter_id']
        text = text.replace('@' + target_company, ' __company__ ').replace('@' + cust_twitter_id, ' __cust__ ')

        return is_valid, text

def run_for_company(in_csv_file, in_max_rows, examples_writer, target_company, no_preprocess):
    f_in = open(expand(in_csv_file), 'r')
    node_by_id = {}
    start_node_ids = []
    dict_reader = csv.DictReader(f_in)
    next_by_prev = {}
    for row in dict_reader:
        id = row['tweet_id']
        prev = row['in_response_to_tweet_id']
        next_by_prev[prev] = id
        if prev == '' and ('@' + target_company) in row['text']:
            start_node_ids.append(id)
        node_by_id[id] = row
        if in_max_rows is not None and len(node_by_id) >= in_max_rows:
            print('reached max rows', in_max_rows, '=> breaking')
            break
    print('len(node_by_id)', len(node_by_id))
    print('len(start_node_ids)', len(start_node_ids))
    count_by_status = defaultdict(int)
    count_by_count = defaultdict(int)

    preprocessor = twitter_airlines.Preprocessor() if not no_preprocess else NullPreprocessor()

    for i, start_node_id in enumerate(start_node_ids):
        conversation_texts = []
        first_utterance = None
        is_valid = True
        node = node_by_id[start_node_id]
        cust_twitter_id = node['author_id']
        while True:
            text = node['text'].replace('\n', ' ')
            if node['inbound'] == 'True':
                start_tok = '<cust__'
                end_tok = '__cust>'
            else:
                start_tok = '<rep__'
                end_tok = '__rep>'

            _valid, text = preprocessor(text, info_dict={
                'target_company': target_company,
                'cust_twitter_id': cust_twitter_id
            })
            if not _valid:
                is_valid = False
            text = start_tok + ' ' + text + ' ' + end_tok
            if first_utterance is None:
                first_utterance = text
            else:
                conversation_texts.append(text)
            response_id = next_by_prev.get(node['tweet_id'], None)
            if response_id is None:
                count_by_count[len(conversation_texts) + 1] += 1
                if is_valid:
                    examples_writer.writerow({
                        'first_tweet_id': start_node_id,
                        'first_utterance': first_utterance,
                        'context': ' '.join(conversation_texts)
                    })
                    count_by_status['accept'] += 1
                else:
                    count_by_status['not_valid'] += 1
                break
            if response_id not in node_by_id:
                count_by_status['response id not found'] += 1
                print('warning: response_id', response_id, 'not found => skipping conversation')
                break
            node = node_by_id[response_id]
    print(count_by_status)

def run_aggregated(in_csv_file, in_max_rows, out_examples, target_companies, no_preprocess):
    with open(expand(out_examples), 'w') as f_examples:
        examples_writer = csv.DictWriter(f_examples, fieldnames=['first_tweet_id', 'first_utterance', 'context'])
        examples_writer.writeheader()
        for company in target_companies:
            print(company)
            run_for_company(in_csv_file, in_max_rows, examples_writer, company, no_preprocess)

def run_by_company(in_csv_file, in_max_rows, out_examples_templ, target_companies, no_preprocess):
    for company in target_companies:
        print(company)
        out_examples = out_examples_templ.format(company=company)
        with open(expand(out_examples), 'w') as f_examples:
            examples_writer = csv.DictWriter(f_examples, fieldnames=['first_tweet_id', 'first_utterance', 'context'])
            examples_writer.writeheader()
            run_for_company(in_csv_file, in_max_rows, examples_writer, company, no_preprocess)

if __name__ == '__main__':
    root_parser = argparse.ArgumentParser()
    parsers = root_parser.add_subparsers()

    parser = parsers.add_parser('run-aggregated')
    parser.add_argument('--in-csv-file', type=str, default='~/data/twittersupport/twcs.csv')
    parser.add_argument('--in-max-rows', type=int, help='for dev/debugging mostly')
    parser.add_argument('--out-examples', type=str, required=True)
    parser.add_argument('--target-companies', type=str, default='Delta,AmericanAir,British_Airways,SouthwestAir')
    parser.add_argument('--no-preprocess', action='store_true')
    parser.set_defaults(func=run_aggregated)

    parser = parsers.add_parser('run-by-company')
    parser.add_argument('--in-csv-file', type=str, default='~/data/twittersupport/twcs.csv')
    parser.add_argument('--in-max-rows', type=int, help='for dev/debugging mostly')
    parser.add_argument('--out-examples-templ', type=str, default='~/data/twittersupport/{company}.csv')
    parser.add_argument('--target-companies', type=str, default=','.join(all_companies))
    parser.add_argument('--no-preprocess', action='store_true')
    parser.set_defaults(func=run_by_company)

    args = root_parser.parse_args()

    if args.func == run_aggregated:
        args.out_examples = args.out_examples.format(**args.__dict__)
        args.target_companies = args.target_companies.split(',')
    elif args.func == run_by_company:
        args.target_companies = args.target_companies.split(',')

    func = args.func
    del args.__dict__['func']
    func(**args.__dict__)

"""
preprocessing twitter dataset
"""
import time
import os
from os import path
from os.path import join, expanduser as expand
import json
import re
import csv
from collections import defaultdict
import argparse

import nltk

from preprocessing import twitter_airlines


all_companies = 'AppleSupport,Amazon,Uber_Support,SpotifyCares,Delta,Tesco,AmericanAir,TMobileHelp,comcastcares,British_Airways,SouthwestAir,VirginTrains,Ask_Spectrum,XboxSupport,sprintcare,hulu_support,AskPlayStation,ChipotleTweets,UPSHelp,AskTarget'.split(',')

"""
example rows:

tweet_id,author_id,inbound,created_at,text,response_tweet_id,in_response_to_tweet_id
1,sprintcare,False,Tue Oct 31 22:10:47 +0000 2017,@115712 I understand. I would like to assist you. We would need to get you into a private secured link to further assist.,2,3
2,115712,True,Tue Oct 31 22:11:45 +0000 2017,@sprintcare and how do you propose we do that,,1
"""

class NullPreprocessor(object):
    def __call__(self, text, info_dict=None):
        is_valid = True
        if 'DM' in text:
            is_valid = False
        if 'https://t.co' in text:
            is_valid = False

        text = text.replace('\n', ' ').replace('\r', ' ')

        target_company = info_dict['target_company']
        cust_twitter_id = info_dict['cust_twitter_id']
        text = text.replace('@' + target_company, ' __company__ ').replace('@' + cust_twitter_id, ' __cust__ ')

        return is_valid, text

def run_for_company(in_csv_file, in_max_rows, examples_writer, target_company, no_preprocess):
    f_in = open(expand(in_csv_file), 'r')
    node_by_id = {}
    start_node_ids = []
    dict_reader = csv.DictReader(f_in)
    next_by_prev = {}
    for row in dict_reader:
        id = row['tweet_id']
        prev = row['in_response_to_tweet_id']
        next_by_prev[prev] = id
        if prev == '' and ('@' + target_company) in row['text']:
            start_node_ids.append(id)
        node_by_id[id] = row
        if in_max_rows is not None and len(node_by_id) >= in_max_rows:
            print('reached max rows', in_max_rows, '=> breaking')
            break
    print('len(node_by_id)', len(node_by_id))
    print('len(start_node_ids)', len(start_node_ids))
    count_by_status = defaultdict(int)
    count_by_count = defaultdict(int)

    preprocessor = twitter_airlines.Preprocessor() if not no_preprocess else NullPreprocessor()

    for i, start_node_id in enumerate(start_node_ids):
        conversation_texts = []
        first_utterance = None
        is_valid = True
        node = node_by_id[start_node_id]
        cust_twitter_id = node['author_id']
        while True:
            text = node['text'].replace('\n', ' ')
            if node['inbound'] == 'True':
                start_tok = '<cust__'
                end_tok = '__cust>'
            else:
                start_tok = '<rep__'
                end_tok = '__rep>'

            _valid, text = preprocessor(text, info_dict={
                'target_company': target_company,
                'cust_twitter_id': cust_twitter_id
            })
            if not _valid:
                is_valid = False
            text = start_tok + ' ' + text + ' ' + end_tok
            if first_utterance is None:
                first_utterance = text
            else:
                conversation_texts.append(text)
            response_id = next_by_prev.get(node['tweet_id'], None)
            if response_id is None:
                count_by_count[len(conversation_texts) + 1] += 1
                if is_valid:
                    examples_writer.writerow({
                        'first_tweet_id': start_node_id,
                        'first_utterance': first_utterance,
                        'context': ' '.join(conversation_texts)
                    })
                    count_by_status['accept'] += 1
                else:
                    count_by_status['not_valid'] += 1
                break
            if response_id not in node_by_id:
                count_by_status['response id not found'] += 1
                print('warning: response_id', response_id, 'not found => skipping conversation')
                break
            node = node_by_id[response_id]
    print(count_by_status)

def run_aggregated(in_csv_file, in_max_rows, out_examples, target_companies, no_preprocess):
    with open(expand(out_examples), 'w') as f_examples:
        examples_writer = csv.DictWriter(f_examples, fieldnames=['first_tweet_id', 'first_utterance', 'context'])
        examples_writer.writeheader()
        for company in target_companies:
            print(company)
            run_for_company(in_csv_file, in_max_rows, examples_writer, company, no_preprocess)

def run_by_company(in_csv_file, in_max_rows, out_examples_templ, target_companies, no_preprocess):
    for company in target_companies:
        print(company)
        out_examples = out_examples_templ.format(company=company)
        with open(expand(out_examples), 'w') as f_examples:
            examples_writer = csv.DictWriter(f_examples, fieldnames=['first_tweet_id', 'first_utterance', 'context'])
            examples_writer.writeheader()
            run_for_company(in_csv_file, in_max_rows, examples_writer, company, no_preprocess)

if __name__ == '__main__':
    root_parser = argparse.ArgumentParser()
    parsers = root_parser.add_subparsers()

    parser = parsers.add_parser('run-aggregated')
    parser.add_argument('--in-csv-file', type=str, default='~/data/twittersupport/twcs.csv')
    parser.add_argument('--in-max-rows', type=int, help='for dev/debugging mostly')
    parser.add_argument('--out-examples', type=str, required=True)
    parser.add_argument('--target-companies', type=str, default='Delta,AmericanAir,British_Airways,SouthwestAir')
    parser.add_argument('--no-preprocess', action='store_true')
    parser.set_defaults(func=run_aggregated)

    parser = parsers.add_parser('run-by-company')
    parser.add_argument('--in-csv-file', type=str, default='~/data/twittersupport/twcs.csv')
    parser.add_argument('--in-max-rows', type=int, help='for dev/debugging mostly')
    parser.add_argument('--out-examples-templ', type=str, default='~/data/twittersupport/{company}.csv')
    parser.add_argument('--target-companies', type=str, default=','.join(all_companies))
    parser.add_argument('--no-preprocess', action='store_true')
    parser.set_defaults(func=run_by_company)

    args = root_parser.parse_args()

    if args.func == run_aggregated:
        args.out_examples = args.out_examples.format(**args.__dict__)
        args.target_companies = args.target_companies.split(',')
    elif args.func == run_by_company:
        args.target_companies = args.target_companies.split(',')

    func = args.func
    del args.__dict__['func']
    func(**args.__dict__)

preprocessing twitter dataset example rows:  tweet_id,author_id,inbound,created_at,text,response_tweet_id,in_response_to_tweet_id 1,sprintcare,False,Tue Oct 31 22:10:47 +0000 2017,@115712 I understand. I would like to assist you. We would need to get you into a private secured link to further assist.,2,3 2,115712,True,Tue Oct 31 22:11:45 +0000 2017,@sprintcare and how do you propose we do that,,1