| import os |
| import json |
| import re |
|
|
|
|
| |
| |
|
|
| def GenSentVocab(): |
| folders = os.listdir('text') |
| vocab = {} |
| out_file = open('sentences.txt','a') |
| for fldr in folders: |
| fldr_pth = os.path.join('text',fldr) |
| files = os.listdir(fldr_pth) |
| for fl in files: |
| fl_pth = os.path.join(fldr_pth,fl) |
| with open(fl_pth,'r') as f: |
| lines = f.readlines() |
| for line in lines: |
| article = json.loads(line) |
| text = article['text'].replace('॥',' ॥\n') |
| text = article['text'].replace('।',' ।\n') |
| text = article['text'].replace('.',' .\n') |
| for s in ['\"',"\'",'?','.',',','(',')','[',']','{','}','-','|',';',':','/','\\','=','’','—','‘','`','!','@','$','~','&','^','%','“','”','+','*','।']: |
| text = text.replace(s,' '+s+' ') |
| ref_tag = re.compile(r'<ref( name= \" .*? \" )*>.*</ref>') |
| text = re.sub(ref_tag,'',text) |
| ref_tag = re.compile(r'<.*?>') |
| text = re.sub(ref_tag,'',text) |
| text = re.sub(' +',' ',text) |
| text = re.sub(r"\n+",'\n',text) |
| text = re.sub(r"[A-Z|a-z]",'',text) |
| text = re.sub(r"[0-9]+",' \1 ',text) |
| sentences = re.split(r"\n",text) |
| for sent in sentences: |
| sentence = sent.strip() |
| if sentence == '': |
| continue |
| out_file.write(sentence+'\n') |
| tokens = sentence.split() |
| for tkn in tokens: |
| if tkn not in vocab: |
| vocab[tkn] = 1 |
| else: |
| vocab[tkn] += 1 |
| out_file.close() |
| with open('vocabulary.txt','a') as voc: |
| for w in vocab: |
| voc.write(w+'\n') |
| voc.write('<UNK>'+'\n') |
| |
|
|
|
|
|
|
| if __name__=='__main__': |
| GenSentVocab() |
