#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright © 2010 University of Zürich
# Author: Rico Sennrich <sennrich@cl.uzh.ch>
# For licensing information, see LICENSE

from __future__ import division,print_function,unicode_literals
import sys
import time
import math
from operator import itemgetter
from bleualign.gale_church import align_texts
import bleualign.score as bleu
from bleualign.utils import evaluate, finalevaluation
import io
import platform


if sys.version_info >= (2,6) and platform.system() != "Windows":
  import multiprocessing
  multiprocessing_enabled = 1
else:
  multiprocessing_enabled = 0


def collect_article(src,srctotarget,target,targettosrc,options):

    EOF = False
    while not EOF:

        all_texts = []
        all_translations = []

        for text,translations in [(src,srctotarget),(target,targettosrc)]:
            textlist = []
  
          translist = [[] for i in translations]

            for line in text:

                if line.rstrip() == options['end_of_article_marker']:
                    for f in translations:
                        f.readline()
 
                   break

                for i,f in enumerate(translations):
                    translist[i].append(f.readline().rstrip())

                if options['factored']:
                    rawline = ' '.join(word.split('|')[0] for word in line.split())
                    textlist.append((rawline,line.rstrip()))
                else:
                    textlist.append(line.rstrip())
            else:
                EOF = True

            all_texts.append(textlist)
            all_translations.append(translist)

        sourcelist, targetlist = all_texts
        translist1, translist2 = all_translations
        yield sourcelist,targetlist,translist1,translist2


#takes a queue as argument and puts all articles to be aligned in it.
#best call this in a separate process because we limit the queue size for memory reasons
def tasks_producer(tasks,num_tasks,data,num_processes):
    for i,task in enumerate(collect_article(*data)):
        num_tasks.value += 1
        tasks.put((i,task),True)
        
    #poison pills
    for i in range(num_processes):
        tasks.put((None,None))
    num_tasks.value -= 1 # only if this point is reached, process finishes when all tasks are done.

class Aligner:
    default_options = {
        #source and target files needed by Aligner
        #they can be filenames, arrays of strings or io objects.
        'srcfile':None, 'targetfile': None,

        #the format of srcfile and targetfile
        #False for normal text, True for 'text | other information', seprating by '|'
        'factored': False,

        #translations of srcfile and targetfile, not influenced by 'factored'
        #they can be filenames, arrays of strings or io objects, too.
        'srctotarget': [], 'targettosrc': [],
        #run aligner without srctotarget and targettosrc
        'no_translation_override':False,
        
        #only consider target sentences for bleu-based alignment that are among top N alternatives for a given source sentence
        'maxalternatives':3,
        
        #bleu scoring algorithm works with 4-grams by default. We got better results when using 2-grams (since there are less 0 scores then)
        'bleu_ngrams' : 2,

        #BLEU is word-based by default, but character-level BLEU is more suitable for some languages, e.g. continuous script languages without space.
        #it is a good idea to also increase bleu_ngrams when switching to character-level BLEU
        'bleu_charlevel' : False,

        #consider N to 1 (and 1 to N) alignment in gapfilling (complexity is size_of_gap*value^2, so don't turn this unnecessarily high)
        #also, there are potential precision issues.
        #set to 1 to disable bleu-based 1 to N alignments and let gale & church fill the gaps
        'Nto1' : 2,
        
        #do only gale-church, no bleualign
        'galechurch': None,

        #gapfillheuristics: what to do with sentences that aren't aligned one-to-one by the first BLEU pass, nor have a 1 to N alignment validated by BLEU?
        #possible members are: bleu1to1, galechurch
        #what they do is commented in the source code
        'gapfillheuristics' : ["bleu1to1","galechurch"],

        #defines string that identifies hard boundaries (articles, chapters etc.)
        #string needs to be on a line of its own (see examples in eval directory)
        #must be reliable (article i in the source text needs to correspond to article i in the target text)
        'end_of_article_marker' : ".EOA",

        #filtering out bad alignments by bleuscore
        #filter has sentences or articles type
        #filterthreshold means choosing the best X% of alignments (according to BLEU)
        #bleuthreshold requires a sentence pair to achieve a certain BLEU score to be included in the output
        #set filterlang True, whose when you want to filter alignemts which src is similar to target than translation
        'filter': None, 'filterthreshold': 90, 'bleuthreshold': 0, 'filterlang': None,

        #it will print unalignemt pair(zero to one or one to zero pair)
        'printempty': False,
        
        #setting output for four output filenames, it will add suffixes automatically
        #or passing filenames or io object for them in respectly.
        #if not passing anything or assigning None, they will use StringIO to save results.
        'output': None,
        'output-src': None, 'output-target': None,
        'output-src-bad': None, 'output-target-bad': None,
        #the best alignment of corpus for evaluation
        'eval': None,
        #defines amount of debugging output.
        'verbosity': 1, 'log_to':sys.stdout,
        #number of parallel processes
        'num_processes': 1
        }
    def __init__(self,options):
      self.src, self.target = None,None
      self.srctotarget, self.targettosrc= [],[]
      self.out1, self.out2, self.out_bad1, self.out_bad2 = None,None,None,None
      self.sources_out,self.targets_out = [],[]
      self.finalbleu = []
      self.bleualign = []
      self.close_src, self.close_target = False, False
      self.close_srctotarget, self.close_targettosrc = [], []
      self.close_out1, self.close_out2 = False, False
      self.close_out_bad1, self.close_out_bad2 = False, False 
      self.options = self.default_options.copy()
      self.options.update(options)
      
      if not self.options['srcfile']:
        raise ValueError('Source file not specified.')
      if not self.options['targetfile']:
        raise ValueError('Target file not specified.')
      if not self.options['srctotarget'] and not self.options['targettosrc']\
            and not self.options['no_translation_override']:
        raise ValueError("ERROR: no translation available: BLEU scores can be computed between the source and target text, but this is not the intended usage of Bleualign and may result in poor performance! If you're *really* sure that this is what you want, set 'galechurch' for the options.")

      self.src, self.close_src = \
            self._inputObjectFromParameter(self.options['srcfile'])
      self.target, self.close_target = \
            self._inputObjectFromParameter(self.options['targetfile'])

      for f in self.options['srctotarget']:
            obj, close_obj = \
                self._inputObjectFromParameter(f)
            self.srctotarget.append(obj)
            self.close_srctotarget.append(close_obj)
      for f in self.options['targettosrc']:
            obj, close_obj = \
                self._inputObjectFromParameter(f)
            self.targettosrc.append(obj)
            self.close_targettosrc.append(close_obj)

      self.out1,self.close_out1=self._outputObjectFromParameter(
            self.options['output-src'], self.options['output'], '-s')
      self.out2,self.close_out2=self._outputObjectFromParameter(
            self.options['output-target'], self.options['output'], '-t')

      if self.options['filter']:
        self.out_bad1,self.close_out_bad1=self._outputObjectFromParameter(
            self.options['output-src-bad'], self.optio