# some string contain hard-to-find diacritical marks as
# independent characters.
# It is difficult to clean strings that contain such marks.
# In those cases, we just remove them from the training set.
def is_bad_word_alpha( word ):
    for c in word:
        # U+0300 - U+036F combining diacritical marks.
        if 768 <= ord(c) and ord(c) <= 879:
            return True
        elif c in bad_chars_alpha:
            return True
    return False

# Generates a set of conversion rules for latin-based characters.
# It lowers the case and convert the accented characters to [a-zç].
# e.g.
# 'Ä' => 'ae'
# We retain 'Ç' as is, as conerting to some character sequence like 'c'
# will deteriorate the performance.
def create_char_maps_alpha():
    # Maps chars down to a-z, ç (27 chars)

    char_map = {}

    char_map['A'] = 'a'
    char_map['B'] = 'b'
    char_map['C'] = 'c'
    char_map['D'] = 'd'
    char_map['E'] = 'e'
    char_map['F'] = 'f'
    char_map['G'] = 'g'
    char_map['H'] = 'h'
    char_map['I'] = 'i'
    char_map['J'] = 'j'
    char_map['K'] = 'k'
    char_map['L'] = 'l'
    char_map['M'] = 'm'
    char_map['N'] = 'n'
    char_map['O'] = 'o'
    char_map['P'] = 'p'
    char_map['Q'] = 'q'
    char_map['R'] = 'r'
    char_map['S'] = 's'
    char_map['T'] = 't'
    char_map['U'] = 'u'
    char_map['V'] = 'v'
    char_map['W'] = 'w'
    char_map['X'] = 'x'
    char_map['Y'] = 'y'
    char_map['Z'] = 'z'
    char_map['a'] = 'a'
    char_map['b'] = 'b'
    char_map['c'] = 'c'
    char_map['d'] = 'd'
    char_map['e'] = 'e'
    char_map['f'] = 'f'
    char_map['g'] = 'g'
    char_map['h'] = 'h'
    char_map['i'] = 'i'
    char_map['j'] = 'j'
    char_map['k'] = 'k'
    char_map['l'] = 'l'
    char_map['m'] = 'm'
    char_map['n'] = 'n'
    char_map['o'] = 'o'
    char_map['p'] = 'p'
    char_map['q'] = 'q'
    char_map['r'] = 'r'
    char_map['s'] = 's'
    char_map['t'] = 't'
    char_map['u'] = 'u'
    char_map['v'] = 'v'
    char_map['w'] = 'w'
    char_map['x'] = 'x'
    char_map['y'] = 'y'
    char_map['z'] = 'z'
    char_map['À'] = 'a'
    char_map['Á'] = 'a'
    char_map['Â'] = 'a'
    char_map['Ä'] = 'ae'
    char_map['Å'] = 'aa'
    char_map['Æ'] = 'ae'
    char_map['Ç'] = 'ç'
    char_map['È'] = 'e'
    char_map['É'] = 'e'
    char_map['Í'] = 'i'
    char_map['Ï'] = 'i'
    char_map['Ò'] = 'o'
    char_map['Ó'] = 'o'
    char_map['Ö'] = 'oe'
    char_map['Ø'] = 'oe'
    char_map['Û'] = 'u'
    char_map['Ü'] = 'ue'
    char_map['Ý'] = 'y'
    char_map['ß'] = 'ss'
    char_map['à'] = 'a'
    char_map['á'] = 'a'
    char_map['â'] = 'a'
    char_map['ã'] = 'a'
    char_map['ä'] = 'ae'
    char_map['å'] = 'aa'
    char_map['æ'] = 'ae'
    char_map['ç'] = 'ç'
    char_map['è'] = 'e'
    char_map['é'] = 'e'
    char_map['ê'] = 'e'
    char_map['ë'] = 'e'
    char_map['ì'] = 'i'
    char_map['í'] = 'i'
    char_map['î'] = 'i'
    char_map['ï'] = 'i'
    char_map['ñ'] = 'nh'
    char_map['ò'] = 'o'
    char_map['ó'] = 'o'
    char_map['ô'] = 'o'
    char_map['õ'] = 'on'
    char_map['ö'] = 'oe'
    char_map['ø'] = 'oe'
    char_map['ù'] = 'u'
    char_map['ú'] = 'u'
    char_map['û'] = 'u'
    char_map['ü'] = 'ue'
    char_map['ý'] = 'y'
    char_map['ÿ'] = 'y'
    char_map['Ā'] = 'a'
    char_map['ā'] = 'a'
    char_map['Ć'] = 'ch'
    char_map['ć'] = 'ch'
    char_map['ĉ'] = 'c'
    char_map['Č'] = 'ch'
    char_map['č'] = 'ch'
    char_map['Ď'] = 'd'
    char_map['Ē'] = 'e'
    char_map['ē'] = 'e'
    char_map['ě'] = 'e'
    char_map['ĩ'] = 'i'
    char_map['Ī'] = 'i'
    char_map['ī'] = 'i'
    char_map['Ĭ'] = 'i'
    char_map['İ'] = 'i'
    char_map['ı'] = 'e'
    char_map['ķ'] = 'k'
    char_map['Ĺ'] = 'l'
    char_map['ĺ'] = 'l'
    char_map['Ł'] = 'l'
    char_map['ł'] = 'w'
    char_map['Ń'] = 'ny'
    char_map['ń'] = 'ny'
    char_map['ņ'] = 'n'
    char_map['ň'] = 'nh'
    char_map['Ō'] = 'o'
    char_map['ō'] = 'o'
    char_map['Ŏ'] = 'o'
    char_map['ő'] = 'e'
    char_map['œ'] = 'oe'
    char_map['ŕ'] = 'r'
    char_map['Ř'] = 'r'
    char_map['ř'] = 'zh'
    char_map['Ś'] = 'sh'
    char_map['ś'] = 'sh'
    char_map['Ş'] = 'sh'
    char_map['ş'] = 'sh'
    char_map['Š'] = 'sh'
    char_map['š'] = 'sh'
    char_map['ũ'] = 'u'
    char_map['ū'] = 'u'
    char_map['ŭ'] = 'u'
    char_map['ů'] = 'u'
    char_map['ű'] = 'u'
    char_map['ŵ'] = 'w'
    char_map['Ź'] = 'dj'
    char_map['ź'] = 'dj'
    char_map['Ż'] = 'j'
    char_map['ż'] = 'j'
    char_map['Ž'] = 'zh'
    char_map['ž'] = 'zh'
    char_map['ǎ'] = 'a'
    char_map['ǐ'] = 'i'
    char_map['ǧ'] = 'gh'
    char_map['ǰ'] = 'j'
    char_map['ǵ'] = 'gh'
    char_map['ǹ'] = 'nh'
    char_map['ș'] = 'sh'
    char_map['ț'] = 'ts'
    char_map['ȩ'] = 'e'
    char_map['ȳ'] = 'y'
    char_map['Ḅ'] = 'b'
    char_map['ḋ'] = 'd'
    char_map['Ḍ'] = 'd'
    char_map['ḍ'] = 'd'
    char_map['Ḏ'] = 'd'
    char_map['ḏ'] = 'd'
    char_map['Ḡ'] = 'g'
    char_map['Ḥ'] = 'h'
    char_map['ḥ'] = 'h'
    char_map['ḱ'] = 'ch'
    char_map['Ḳ'] = 'k'
    char_map['ḳ'] = 'k'
    char_map['Ḵ'] = 'k'
    char_map['ḵ'] = 'k'
    char_map['ḻ'] = 'l'
    char_map['Ḿ'] = 'm'
    char_map['ḿ'] = 'm'
    char_map['ṁ'] = 'm'
    char_map['ṃ'] = 'm'
    char_map['Ṅ'] = 'n'
    char_map['ṅ'] = 'n'
    char_map['ṇ'] = 'n'
    char_map['ṉ'] = 'n'
    char_map['ṕ'] = 'p'
    char_map['ṛ'] = 'r'
    char_map['ṟ'] = 'r'
    char_map['ṡ'] = 's'
    char_map['Ṣ'] = 's'
    char_map['ṣ'] = 's'
    char_map['ṫ'] = 't'
    char_map['Ṭ'] = 't'
    char_map['ṭ'] = 't'
    char_map['ṯ'] = 't'
    char_map['Ẓ'] = 'z'
    char_map['ẓ'] = 'z'
    char_map['ẖ'] = 'h'
    char_map['ạ'] = 'a'
    char_map['Ị'] = 'i'
    char_map['-'] = ''
    return char_map

# Generates a set of conversion rules for Japanese Kanas.
# It does two things:
#  'ヮ' => 'ワ'
#  '・' => ''
def create_char_maps_kana():
    char_map = {}
    char_map['ァ'] = 'ァ'
    char_map['ア'] = 'ア'
    char_map['ィ'] = 'ィ'
    char_map['イ'] = 'イ'
    char_map['ゥ'] = 'ゥ'
    char_map['ウ'] = 'ウ'
    char_map['ェ'] = 'ェ'
    char_map['エ'] = 'エ'
    char_map['ォ'] = 'ォ'
    char_map['オ'] = 'オ'
    char_map['カ'] = 'カ'
    char_map['ガ'] = 'ガ'
    char_map['キ'] = 'キ'
    char_map['ギ'] = 'ギ'
    char_map['ク'] = 'ク'
    char_map['グ'] = 'グ'
    char_map['ケ'] = 'ケ'
    char_map['ゲ'] = 'ゲ'
    char_map['コ'] = 'コ'
    char_map['ゴ'] = 'ゴ'
    char_map['サ'] = 'サ'
    char_map['ザ'] = 'ザ'
    char_map['シ'] = 'シ'
    char_map['ジ'] = 'ジ'
    char_map['ス'] = 'ス'
    char_map['ズ'] = 'ズ'
    char_map['セ'] = 'セ'
    char_map['ゼ'] = 'ゼ'
    char_map['ソ'] = 'ソ'
    char_map['ゾ'] = 'ゾ'
    char_map['タ'] = 'タ'
    char_map['ダ'] = 'ダ'
    char_map['チ'] = 'チ'
    char_map['ッ'] = 'ッ'
    char_map['ツ'] = 'ツ'
    char_map['テ'] = 'テ'
    char_map['デ'] = 'デ'
    char_map['ト'] = 'ト'
    char_map['ド'] = 'ド'
    char_map['ナ'] = 'ナ'
    char_map['ニ'] = 'ニ'
    char_map['ヌ'] = 'ヌ'
    char_map['ネ'] = 'ネ'
    char_map['ノ'] = 'ノ'
    char_map['ハ'] = 'ハ'
    char_map['バ'] = 'バ'
    char_map['パ'] = 'パ'
    char_map['ヒ'] = 'ヒ'
    char_map['ビ'] = 'ビ'
    char_map['ピ'] = 'ピ'
    char_map['フ'] = 'フ'
    char_map['ブ'] = 'ブ'
    char_map['プ'] = 'プ'
    char_map['ヘ'] = 'ヘ'
    char_map['ベ'] = 'ベ'
    char_map['ペ'] = 'ペ'
    char_map['ホ'] = 'ホ'
    char_map['ボ'] = 'ボ'
    char_map['ポ'] = 'ポ'
    char_map['マ'] = 'マ'
    char_map['ミ'] = 'ミ'
    char_map['ム'] = 'ム'
    char_map['メ'] = 'メ'
    char_map['モ'] = 'モ'
    char_map['ャ'] = 'ャ'
    char_map['ヤ'] = 'ヤ'
    char_map['ュ'] = 'ュ'
    char_map['ユ'] = 'ユ'
    char_map['ョ'] = 'ョ'
    char_map['ヨ'] = 'ヨ'
    char_map['ラ'] = 'ラ'
    char_map['リ'] = 'リ'
    char_map['ル'] = 'ル'
    char_map['レ'] = 'レ'
    char_map['ロ'] = 'ロ'
    char_map['ヮ'] = 'ワ'
    char_map['ワ'] = 'ワ'
    char_map['ン'] = 'ン'
    char_map['ヴ'] = 'ヴ'
    char_map['・'] = ''
    char_map['ー'] = 'ー'
    return char_map

# some string contain hard-to-find diacritical marks as
# independent characters.
# It is difficult to clean strings that contain such marks.
# In those cases, we just remove them from the training set.
def is_bad_word_alpha( word ):
    for c in word:
        # U+0300 - U+036F combining diacritical marks.
        if 768 <= ord(c) and ord(c) <= 879:
            return True
        elif c in bad_chars_alpha:
            return True
    return False

# Generates a set of conversion rules for latin-based characters.
# It lowers the case and convert the accented characters to [a-zç].
# e.g.
# 'Ä' => 'ae'
# We retain 'Ç' as is, as conerting to some character sequence like 'c'
# will deteriorate the performance.
def create_char_maps_alpha():
    # Maps chars down to a-z, ç (27 chars)

    char_map = {}

    char_map['A'] = 'a'
    char_map['B'] = 'b'
    char_map['C'] = 'c'
    char_map['D'] = 'd'
    char_map['E'] = 'e'
    char_map['F'] = 'f'
    char_map['G'] = 'g'
    char_map['H'] = 'h'
    char_map['I'] = 'i'
    char_map['J'] = 'j'
    char_map['K'] = 'k'
    char_map['L'] = 'l'
    char_map['M'] = 'm'
    char_map['N'] = 'n'
    char_map['O'] = 'o'
    char_map['P'] = 'p'
    char_map['Q'] = 'q'
    char_map['R'] = 'r'
    char_map['S'] = 's'
    char_map['T'] = 't'
    char_map['U'] = 'u'
    char_map['V'] = 'v'
    char_map['W'] = 'w'
    char_map['X'] = 'x'
    char_map['Y'] = 'y'
    char_map['Z'] = 'z'
    char_map['a'] = 'a'
    char_map['b'] = 'b'
    char_map['c'] = 'c'
    char_map['d'] = 'd'
    char_map['e'] = 'e'
    char_map['f'] = 'f'
    char_map['g'] = 'g'
    char_map['h'] = 'h'
    char_map['i'] = 'i'
    char_map['j'] = 'j'
    char_map['k'] = 'k'
    char_map['l'] = 'l'
    char_map['m'] = 'm'
    char_map['n'] = 'n'
    char_map['o'] = 'o'
    char_map['p'] = 'p'
    char_map['q'] = 'q'
    char_map['r'] = 'r'
    char_map['s'] = 's'
    char_map['t'] = 't'
    char_map['u'] = 'u'
    char_map['v'] = 'v'
    char_map['w'] = 'w'
    char_map['x'] = 'x'
    char_map['y'] = 'y'
    char_map['z'] = 'z'
    char_map['À'] = 'a'
    char_map['Á'] = 'a'
    char_map['Â'] = 'a'
    char_map['Ä'] = 'ae'
    char_map['Å'] = 'aa'
    char_map['Æ'] = 'ae'
    char_map['Ç'] = 'ç'
    char_map['È'] = 'e'
    char_map['É'] = 'e'
    char_map['Í'] = 'i'
    char_map['Ï'] = 'i'
    char_map['Ò'] = 'o'
    char_map['Ó'] = 'o'
    char_map['Ö'] = 'oe'
    char_map['Ø'] = 'oe'
    char_map['Û'] = 'u'
    char_map['Ü'] = 'ue'
    char_map['Ý'] = 'y'
    char_map['ß'] = 'ss'
    char_map['à'] = 'a'
    char_map['á'] = 'a'
    char_map['â'] = 'a'
    char_map['ã'] = 'a'
    char_map['ä'] = 'ae'
    char_map['å'] = 'aa'
    char_map['æ'] = 'ae'
    char_map['ç'] = 'ç'
    char_map['è'] = 'e'
    char_map['é'] = 'e'
    char_map['ê'] = 'e'
    char_map['ë'] = 'e'
    char_map['ì'] = 'i'
    char_map['í'] = 'i'
    char_map['î'] = 'i'
    char_map['ï'] = 'i'
    char_map['ñ'] = 'nh'
    char_map['ò'] = 'o'
    char_map['ó'] = 'o'
    char_map['ô'] = 'o'
    char_map['õ'] = 'on'
    char_map['ö'] = 'oe'
    char_map['ø'] = 'oe'
    char_map['ù'] = 'u'
    char_map['ú'] = 'u'
    char_map['û'] = 'u'
    char_map['ü'] = 'ue'
    char_map['ý'] = 'y'
    char_map['ÿ'] = 'y'
    char_map['Ā'] = 'a'
    char_map['ā'] = 'a'
    char_map['Ć'] = 'ch'
    char_map['ć'] = 'ch'
    char_map['ĉ'] = 'c'
    char_map['Č'] = 'ch'
    char_map['č'] = 'ch'
    char_map['Ď'] = 'd'
    char_map['Ē'] = 'e'
    char_map['ē'] = 'e'
    char_map['ě'] = 'e'
    char_map['ĩ'] = 'i'
    char_map['Ī'] = 'i'
    char_map['ī'] = 'i'
    char_map['Ĭ'] = 'i'
    char_map['İ'] = 'i'
    char_map['ı'] = 'e'
    char_map['ķ'] = 'k'
    char_map['Ĺ'] = 'l'
    char_map['ĺ'] = 'l'
    char_map['Ł'] = 'l'
    char_map['ł'] = 'w'
    char_map['Ń'] = 'ny'
    char_map['ń'] = 'ny'
    char_map['ņ'] = 'n'
    char_map['ň'] = 'nh'
    char_map['Ō'] = 'o'
    char_map['ō'] = 'o'
    char_map['Ŏ'] = 'o'
    char_map['ő'] = 'e'
    char_map['œ'] = 'oe'
    char_map['ŕ'] = 'r'
    char_map['Ř'] = 'r'
    char_map['ř'] = 'zh'
    char_map['Ś'] = 'sh'
    char_map['ś'] = 'sh'
    char_map['Ş'] = 'sh'
    char_map['ş'] = 'sh'
    char_map['Š'] = 'sh'
    char_map['š'] = 'sh'
    char_map['ũ'] = 'u'
    char_map['ū'] = 'u'
    char_map['ŭ'] = 'u'
    char_map['ů'] = 'u'
    char_map['ű'] = 'u'
    char_map['ŵ'] = 'w'
    char_map['Ź'] = 'dj'
    char_map['ź'] = 'dj'
    char_map['Ż'] = 'j'
    char_map['ż'] = 'j'
    char_map['Ž'] = 'zh'
    char_map['ž'] = 'zh'
    char_map['ǎ'] = 'a'
    char_map['ǐ'] = 'i'
    char_map['ǧ'] = 'gh'
    char_map['ǰ'] = 'j'
    char_map['ǵ'] = 'gh'
    char_map['ǹ'] = 'nh'
    char_map['ș'] = 'sh'
    char_map['ț'] = 'ts'
    char_map['ȩ'] = 'e'
    char_map['ȳ'] = 'y'
    char_map['Ḅ'] = 'b'
    char_map['ḋ'] = 'd'
    char_map['Ḍ'] = 'd'
    char_map['ḍ'] = 'd'
    char_map['Ḏ'] = 'd'
    char_map['ḏ'] = 'd'
    char_map['Ḡ'] = 'g'
    char_map['Ḥ'] = 'h'
    char_map['ḥ'] = 'h'
    char_map['ḱ'] = 'ch'
    char_map['Ḳ'] = 'k'
    char_map['ḳ'] = 'k'
    char_map['Ḵ'] = 'k'
    char_map['ḵ'] = 'k'
    char_map['ḻ'] = 'l'
    char_map['Ḿ'] = 'm'
    char_map['ḿ'] = 'm'
    char_map['ṁ'] = 'm'
    char_map['ṃ'] = 'm'
    char_map['Ṅ'] = 'n'
    char_map['ṅ'] = 'n'
    char_map['ṇ'] = 'n'
    char_map['ṉ'] = 'n'
    char_map['ṕ'] = 'p'
    char_map['ṛ'] = 'r'
    char_map['ṟ'] = 'r'
    char_map['ṡ'] = 's'
    char_map['Ṣ'] = 's'
    char_map['ṣ'] = 's'
    char_map['ṫ'] = 't'
    char_map['Ṭ'] = 't'
    char_map['ṭ'] = 't'
    char_map['ṯ'] = 't'
    char_map['Ẓ'] = 'z'
    char_map['ẓ'] = 'z'
    char_map['ẖ'] = 'h'
    char_map['ạ'] = 'a'
    char_map['Ị'] = 'i'
    char_map['-'] = ''
    return char_map

# Generates a set of conversion rules for Japanese Kanas.
# It does two things:
#  'ヮ' => 'ワ'
#  '・' => ''
def create_char_maps_kana():
    char_map = {}
    char_map['ァ'] = 'ァ'
    char_map['ア'] = 'ア'
    char_map['ィ'] = 'ィ'
    char_map['イ'] = 'イ'
    char_map['ゥ'] = 'ゥ'
    char_map['ウ'] = 'ウ'
    char_map['ェ'] = 'ェ'
    char_map['エ'] = 'エ'
    char_map['ォ'] = 'ォ'
    char_map['オ'] = 'オ'
    char_map['カ'] = 'カ'
    char_map['ガ'] = 'ガ'
    char_map['キ'] = 'キ'
    char_map['ギ'] = 'ギ'
    char_map['ク'] = 'ク'
    char_map['グ'] = 'グ'
    char_map['ケ'] = 'ケ'
    char_map['ゲ'] = 'ゲ'
    char_map['コ'] = 'コ'
    char_map['ゴ'] = 'ゴ'
    char_map['サ'] = 'サ'
    char_map['ザ'] = 'ザ'
    char_map['シ'] = 'シ'
    char_map['ジ'] = 'ジ'
    char_map['ス'] = 'ス'
    char_map['ズ'] = 'ズ'
    char_map['セ'] = 'セ'
    char_map['ゼ'] = 'ゼ'
    char_map['ソ'] = 'ソ'
    char_map['ゾ'] = 'ゾ'
    char_map['タ'] = 'タ'
    char_map['ダ'] = 'ダ'
    char_map['チ'] = 'チ'
    char_map['ッ'] = 'ッ'
    char_map['ツ'] = 'ツ'
    char_map['テ'] = 'テ'
    char_map['デ'] = 'デ'
    char_map['ト'] = 'ト'
    char_map['ド'] = 'ド'
    char_map['ナ'] = 'ナ'
    char_map['ニ'] = 'ニ'
    char_map['ヌ'] = 'ヌ'
    char_map['ネ'] = 'ネ'
    char_map['ノ'] = 'ノ'
    char_map['ハ'] = 'ハ'
    char_map['バ'] = 'バ'
    char_map['パ'] = 'パ'
    char_map['ヒ'] = 'ヒ'
    char_map['ビ'] = 'ビ'
    char_map['ピ'] = 'ピ'
    char_map['フ'] = 'フ'
    char_map['ブ'] = 'ブ'
    char_map['プ'] = 'プ'
    char_map['ヘ'] = 'ヘ'
    char_map['ベ'] = 'ベ'
    char_map['ペ'] = 'ペ'
    char_map['ホ'] = 'ホ'
    char_map['ボ'] = 'ボ'
    char_map['ポ'] = 'ポ'
    char_map['マ'] = 'マ'
    char_map['ミ'] = 'ミ'
    char_map['ム'] = 'ム'
    char_map['メ'] = 'メ'
    char_map['モ'] = 'モ'
    char_map['ャ'] = 'ャ'
    char_map['ヤ'] = 'ヤ'
    char_map['ュ'] = 'ュ'
    char_map['ユ'] = 'ユ'
    char_map['ョ'] = 'ョ'
    char_map['ヨ'] = 'ヨ'
    char_map['ラ'] = 'ラ'
    char_map['リ'] = 'リ'
    char_map['ル'] = 'ル'
    char_map['レ'] = 'レ'
    char_map['ロ'] = 'ロ'
    char_map['ヮ'] = 'ワ'
    char_map['ワ'] = 'ワ'
    char_map['ン'] = 'ン'
    char_map['ヴ'] = 'ヴ'
    char_map['・'] = ''
    char_map['ー'] = 'ー'
    return char_map

# some string contain hard-to-find diacritical marks as # independent characters. # It is difficult to clean strings that contain such marks. # In those cases, we just remove them from the training set. # U+0300 - U+036F combining diacritical marks. # Generates a set of conversion rules for latin-based characters. # It lowers the case and convert the accented characters to [a-zç]. # e.g. # 'Ä' => 'ae' # We retain 'Ç' as is, as conerting to some character sequence like 'c' # will deteriorate the performance. # Maps chars down to a-z, ç (27 chars) # Generates a set of conversion rules for Japanese Kanas. # It does two things: #  'ヮ' => 'ワ' #  '・' => ''