#!/usr/bin/python

import re
from datetime import datetime
import ply.lex as lex

# ------------------------------------------------------------
# query_lexer.py
#
# tokenizer for log query expression 
# ------------------------------------------------------------

class QueryLexer:
    reserved = {
    'in':'IN',
#    'between':'BETWEEN',
    'like':'LIKE',
    'and':'AND'
            }

    tokens = [
        'IP',
        'TIME',
        'DAY',
        'MONTH',
        'NUMBER',
        'STRING',
        'ID',
        'COMMA',
        'LPAREN',
        'RPAREN',
        'GT',
        'LT',
        'GE',
        'LE',
        'EQ',
        'NE'
    ] + list(reserved.values())

    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_COMMA = r','
#    t_BETWEEN = r'between'
    t_IN = r'in'
    t_LIKE = r'like'
    t_AND = r'and'
    t_IP = r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
#    t_TIME = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[01]?[0-9]):([0-5]?[0-9]):([0-5]?[0-9])Z'    #2012-11-25T21:49:00Z
#    t_DAY = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])'     #2012-11-25 
    t_MONTH = r'\b([0-9]{4})-(1[0-2]|0[1-9])\b'   #2012-08
#    t_STRING = r'\'[^\']+\''
    t_NE = r'!='
    t_LE = r'<='
    t_GE = r'>='
    t_LT = r'<'
    t_GT = r'>'
    t_EQ = r'='
#    t_NUMBER = r'\b\d+$'
    # A string containing ignored characters (spaces and tabs)
    t_ignore  = ' \t'
    
    def __init__(self):
        self.lexer = lex.lex(module=self,debug=0)
        
    def t_TIME(self,t):
        r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[01]?[0-9]):([0-5]?[0-9]):([0-5]?[0-9])Z'
        t.value = datetime.strptime(t.value, "%Y-%m-%dT%H:%M:%SZ") 
        return t
        
    def t_DAY(self,t):
        r'\b([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])\b'
        t.value = datetime.strptime(t.value, "%Y-%m-%d") 
        return t
        
    def t_NUMBER(self,t):
        r'\b\d+\s+\b|\b\d+$'
        t.value = int(t.value)
        return t

    def t_STRING(self,t):
        r'\'[^\']+\''
        raw_ip = t.value.strip()
        t.value = re.sub("'",'', raw_ip).strip()
        return t
        
    # Check for reserved words
    def t_ID(self,t):
        r'[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = self.reserved.get(t.value,'ID')
        return t
    
    # Define a rule so we can track line numbers
    def t_newline(self,t):
        r'\n+'
        t.lexer.lineno += len(t.value)
    
    # Error handling rule
    def t_error(self,t):
        print "Illegal character '%s'" % t.value[0]
        t.lexer.skip(1)
    
def test():
#    data = 'ip in (10.30.2.1, 172.16.58.3)'
#    data = 'domain in ( \'aa.com\',\'bb.ca\' ) and size >=3678'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and size >= 346 and ip=172.16.58.3'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346 and create_time > 2010-12-19'
    data = 'create_time >=2012-11-25T21:49:00Z and create_time <= 2012-11-25T21:49:13Z and domain like cnncom'
    ql = QueryLexer()
    lexer = ql.lexer
    lexer.input(data)
    while True:
        token = lexer.token()
        if not token: break
        print token
         
if __name__ == "__main__":test()
#!/usr/bin/python

import re
from datetime import datetime
import ply.lex as lex

# ------------------------------------------------------------
# query_lexer.py
#
# tokenizer for log query expression 
# ------------------------------------------------------------

class QueryLexer:
    reserved = {
    'in':'IN',
#    'between':'BETWEEN',
    'like':'LIKE',
    'and':'AND'
            }

    tokens = [
        'IP',
        'TIME',
        'DAY',
        'MONTH',
        'NUMBER',
        'STRING',
        'ID',
        'COMMA',
        'LPAREN',
        'RPAREN',
        'GT',
        'LT',
        'GE',
        'LE',
        'EQ',
        'NE'
    ] + list(reserved.values())

    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_COMMA = r','
#    t_BETWEEN = r'between'
    t_IN = r'in'
    t_LIKE = r'like'
    t_AND = r'and'
    t_IP = r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
#    t_TIME = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[01]?[0-9]):([0-5]?[0-9]):([0-5]?[0-9])Z'    #2012-11-25T21:49:00Z
#    t_DAY = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])'     #2012-11-25 
    t_MONTH = r'\b([0-9]{4})-(1[0-2]|0[1-9])\b'   #2012-08
#    t_STRING = r'\'[^\']+\''
    t_NE = r'!='
    t_LE = r'<='
    t_GE = r'>='
    t_LT = r'<'
    t_GT = r'>'
    t_EQ = r'='
#    t_NUMBER = r'\b\d+$'
    # A string containing ignored characters (spaces and tabs)
    t_ignore  = ' \t'
    
    def __init__(self):
        self.lexer = lex.lex(module=self,debug=0)
        
    def t_TIME(self,t):
        r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[01]?[0-9]):([0-5]?[0-9]):([0-5]?[0-9])Z'
        t.value = datetime.strptime(t.value, "%Y-%m-%dT%H:%M:%SZ") 
        return t
        
    def t_DAY(self,t):
        r'\b([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])\b'
        t.value = datetime.strptime(t.value, "%Y-%m-%d") 
        return t
        
    def t_NUMBER(self,t):
        r'\b\d+\s+\b|\b\d+$'
        t.value = int(t.value)
        return t

    def t_STRING(self,t):
        r'\'[^\']+\''
        raw_ip = t.value.strip()
        t.value = re.sub("'",'', raw_ip).strip()
        return t
        
    # Check for reserved words
    def t_ID(self,t):
        r'[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = self.reserved.get(t.value,'ID')
        return t
    
    # Define a rule so we can track line numbers
    def t_newline(self,t):
        r'\n+'
        t.lexer.lineno += len(t.value)
    
    # Error handling rule
    def t_error(self,t):
        print "Illegal character '%s'" % t.value[0]
        t.lexer.skip(1)
    
def test():
#    data = 'ip in (10.30.2.1, 172.16.58.3)'
#    data = 'domain in ( \'aa.com\',\'bb.ca\' ) and size >=3678'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and size >= 346 and ip=172.16.58.3'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346'
#    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346 and create_time > 2010-12-19'
    data = 'create_time >=2012-11-25T21:49:00Z and create_time <= 2012-11-25T21:49:13Z and domain like cnncom'
    ql = QueryLexer()
    lexer = ql.lexer
    lexer.input(data)
    while True:
        token = lexer.token()
        if not token: break
        print token
         
if __name__ == "__main__":test()
#!/usr/bin/python # ------------------------------------------------------------ # query_lexer.py # # tokenizer for log query expression # ------------------------------------------------------------ #    'between':'BETWEEN', #    t_BETWEEN = r'between' #    t_TIME = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])T(2[0-3]|[01]?[0-9]):([0-5]?[0-9]):([0-5]?[0-9])Z'    #2012-11-25T21:49:00Z #    t_DAY = r'([0-9]{4})-(1[0-2]|0[1-9])-(3[0-1]|0[1-9]|[1-2][0-9])'     #2012-11-25 #2012-08 #    t_STRING = r'\'[^\']+\'' #    t_NUMBER = r'\b\d+$' # A string containing ignored characters (spaces and tabs) # Check for reserved words # Define a rule so we can track line numbers # Error handling rule #    data = 'ip in (10.30.2.1, 172.16.58.3)' #    data = 'domain in ( \'aa.com\',\'bb.ca\' ) and size >=3678' #    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and size >= 346 and ip=172.16.58.3' #    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346' #    data = 'domain in ( \'aa.com\',\'bb.ca\',\'cc.ca\' ) and ip=172.16.58.3 and size >= 346 and create_time > 2010-12-19'