<gh_stars>0
"""
DO NOT EDIT THIS FILE
This is a Python script for analysing logs after a simulation run.
Type:
    python3 analyselogs.py
for usage.
"""


import sys
import os

cons_t = 'cons'
prod_t = 'prod'
proc_names = { cons_t: 'consumer', prod_t: 'producer' }

"""
The return value is a dictionary/map of dictionaries of the following form:

{ proctype: 
    { procid: [loglines] },
}

"""
def read_logs(lognames):
    logs = {}
    for logname in lognames:
        with open(logname) as l:
            data = l.read().splitlines()

        procname = os.path.basename(logname).split('_')
    
        proctype = procname[1][0:4]
        procid = procname[1][4:6]

        if proctype in logs:
            logs[proctype][procid] = data
        else:
            logs[proctype] = {procid: data}
            
    return logs
            
"""
The return value is a tuple that is a boolean indicator for whether 
violations are detected (there is at least one non-empty set of pairwise 
intersections) plus a list of tuples of the pairwise intersections for each combination of keys, e.g. for keys pid1, pid2, pid3:

(violation_detected, 
    [
        (pid1, pid2, set_of_pid1pid2_intersections), 
        (pid1, pid3, set_of_pid1pid3_intersections), 
        (pid2, pid3, set_of_pid2pid3_intersections),
    ]
)

If set of all members of intersections is needed do:
set([e for list_of_inter in [it[2] for it in inter] for e in list_of_inter])
"""
def get_intersections(logs, type, prod_intersect_byid = False):
    violation = False
    inter = []
    val_len = 9
    
    if type not in logs:
        return (violation, inter)
    
    if type == prod_t:
        if prod_intersect_byid:
            val_len = 5
        else:
            return (violation, inter)
    
    keys = list(logs[type].keys())

    for i in range(len(keys)):
        j = i + 1
        while j <= len(keys) - 1:
            s1 = set([val[0:val_len] for val in logs[type][keys[i]]])
            s2 = set([val[0:val_len] for val in logs[type][keys[j]]])
            inter_set = s1.intersection(s2)
            inter.append((keys[i], keys[j], inter_set))
            violation = bool(inter_set)
            j = j + 1
            
    return (violation, inter)

"""
The return value is the set of all log entries for the given proctype.
"""
def get_union(logs, type):
    all_entries = []
    
    if type not in logs:
        return set()

    for l in logs[type].values():
        all_entries.extend(l)
    
    return set(all_entries)
    
"""
The return value is a tuple that is a boolean indicator for whether violations
where detected (there are duplicate entries any of the logs for the given 
proc type) and dictionary of procid key to set of duplicates. E.g. for 
keys pid1, pid2, pid3:

(violation_detected, 
    {   pid1: set_of_pid1_duplicates,
        pid2: set_of_pid2_duplicates
        pid3: set_of_pid3_duplicates
    }
)
"""
def get_duplicates(logs, type):
    violation = False
    all_dups = {}
    
    if type not in logs:
        return (violation, all_dups)
    
    for k in logs[type].keys():
        seen = set()
        dups = set()

        for l in logs[type][k]:
            if l in seen:
                dups.add(l)
            else:
                seen.add(l)
                
        violation = bool(dups)
        
        all_dups[k] = dups
    
    return (violation, all_dups)

def usage_exit(cmd):
    print('usage: python3 {} [-pi] [-v] logname1 logname2, ...\n'
            '       -pi = do producer id intersections\n'
            '       -v = verbose output\n'
            '       can be invoked with * for all lognames in a directory, '
                    'e.g.\n'
            '       python3 analyselogs.py out/*'
            .format(cmd)
         )
    
    exit()
    
def print_separator(verbose):
    if verbose:
        print('-------------------------------------------------------')
        
    
def print_preamble(subject, violation_type, violation_detected, verbose):
    print_separator(verbose)
    if verbose:
        print('Analysing {} {}s'.format(subject, violation_type))
        
    if violation_detected:
        prem = '*****'
        postm = ' *****'
    else:
        prem = 'OK no'
        postm = ''
    

    print('{} {} violation(s) detected for {}(s){}'
                .format(prem, violation_type, subject, postm))

def print_violations(msg, violations, verbose):
    if verbose and violations:
        print(msg)
        print(' ', '\n  '.join(violations))

def process_args(args): 
    cmd = args.pop(0)
    pi = False
    v = False

    if '-pi' in args:
        args.remove('-pi')
        pi = True
        
    if '-v' in args:
        args.remove('-v')
        v = True
        
    if not args:
        usage_exit(cmd)
        
    return (pi, v, args)

def analyse_intersections(type, intersections, verbose):
    if not intersections[1]:
        return  # no intersections in list
        
    print_preamble(proc_names[type], 'intersection', intersections[0],
        verbose)
        
    for inter in intersections[1]:
        print_violations('jobs processed by both {}{} and {}{}:'
                            .format(type, inter[0], type, inter[1]),
                            inter[2], verbose)
    
def analyse_duplicates(type, duplicates, verbose):
    print_preamble(proc_names[type], 'duplicate', duplicates[0], verbose)

    for k in duplicates[1]:
        print_violations('duplicate jobs for {}{}:'.format(type, k), 
            duplicates[1][k], verbose)
                
def analyse_differences(prod_union, cons_union, verbose):
    if prod_union and cons_union:
        inprod_notcons = prod_union.difference(cons_union)
        incons_notprod = cons_union.difference(prod_union)
    
        print_preamble('producer/consumer', 'difference', 
            inprod_notcons or incons_notprod, verbose)

        print_violations('producer jobs not consumed:', inprod_notcons, verbose)
        print_violations('consumer jobs not produced:', incons_notprod, verbose)


""" MAIN PROGRAM """
(prod_intersect_byid, verbose, args) = process_args(sys.argv)

logs = read_logs(args)

for type in logs:
    intersections = get_intersections(logs, type, prod_intersect_byid)
    analyse_intersections(type, intersections, verbose)
    duplicates = get_duplicates(logs, type)
    analyse_duplicates(type, duplicates, verbose)
    
analyse_differences(get_union(logs, prod_t), get_union(logs, cons_t), verbose)

print_separator(verbose)
"""
DO NOT EDIT THIS FILE
This is a Python script for analysing logs after a simulation run.
Type:
    python3 analyselogs.py
for usage.
"""


import sys
import os

cons_t = 'cons'
prod_t = 'prod'
proc_names = { cons_t: 'consumer', prod_t: 'producer' }

"""
The return value is a dictionary/map of dictionaries of the following form:

{ proctype: 
    { procid: [loglines] },
}

"""
def read_logs(lognames):
    logs = {}
    for logname in lognames:
        with open(logname) as l:
            data = l.read().splitlines()

        procname = os.path.basename(logname).split('_')
    
        proctype = procname[1][0:4]
        procid = procname[1][4:6]

        if proctype in logs:
            logs[proctype][procid] = data
        else:
            logs[proctype] = {procid: data}
            
    return logs
            
"""
The return value is a tuple that is a boolean indicator for whether 
violations are detected (there is at least one non-empty set of pairwise 
intersections) plus a list of tuples of the pairwise intersections for each combination of keys, e.g. for keys pid1, pid2, pid3:

(violation_detected, 
    [
        (pid1, pid2, set_of_pid1pid2_intersections), 
        (pid1, pid3, set_of_pid1pid3_intersections), 
        (pid2, pid3, set_of_pid2pid3_intersections),
    ]
)

If set of all members of intersections is needed do:
set([e for list_of_inter in [it[2] for it in inter] for e in list_of_inter])
"""
def get_intersections(logs, type, prod_intersect_byid = False):
    violation = False
    inter = []
    val_len = 9
    
    if type not in logs:
        return (violation, inter)
    
    if type == prod_t:
        if prod_intersect_byid:
            val_len = 5
        else:
            return (violation, inter)
    
    keys = list(logs[type].keys())

    for i in range(len(keys)):
        j = i + 1
        while j <= len(keys) - 1:
            s1 = set([val[0:val_len] for val in logs[type][keys[i]]])
            s2 = set([val[0:val_len] for val in logs[type][keys[j]]])
            inter_set = s1.intersection(s2)
            inter.append((keys[i], keys[j], inter_set))
            violation = bool(inter_set)
            j = j + 1
            
    return (violation, inter)

"""
The return value is the set of all log entries for the given proctype.
"""
def get_union(logs, type):
    all_entries = []
    
    if type not in logs:
        return set()

    for l in logs[type].values():
        all_entries.extend(l)
    
    return set(all_entries)
    
"""
The return value is a tuple that is a boolean indicator for whether violations
where detected (there are duplicate entries any of the logs for the given 
proc type) and dictionary of procid key to set of duplicates. E.g. for 
keys pid1, pid2, pid3:

(violation_detected, 
    {   pid1: set_of_pid1_duplicates,
        pid2: set_of_pid2_duplicates
        pid3: set_of_pid3_duplicates
    }
)
"""
def get_duplicates(logs, type):
    violation = False
    all_dups = {}
    
    if type not in logs:
        return (violation, all_dups)
    
    for k in logs[type].keys():
        seen = set()
        dups = set()

        for l in logs[type][k]:
            if l in seen:
                dups.add(l)
            else:
                seen.add(l)
                
        violation = bool(dups)
        
        all_dups[k] = dups
    
    return (violation, all_dups)

def usage_exit(cmd):
    print('usage: python3 {} [-pi] [-v] logname1 logname2, ...\n'
            '       -pi = do producer id intersections\n'
            '       -v = verbose output\n'
            '       can be invoked with * for all lognames in a directory, '
                    'e.g.\n'
            '       python3 analyselogs.py out/*'
            .format(cmd)
         )
    
    exit()
    
def print_separator(verbose):
    if verbose:
        print('-------------------------------------------------------')
        
    
def print_preamble(subject, violation_type, violation_detected, verbose):
    print_separator(verbose)
    if verbose:
        print('Analysing {} {}s'.format(subject, violation_type))
        
    if violation_detected:
        prem = '*****'
        postm = ' *****'
    else:
        prem = 'OK no'
        postm = ''
    

    print('{} {} violation(s) detected for {}(s){}'
                .format(prem, violation_type, subject, postm))

def print_violations(msg, violations, verbose):
    if verbose and violations:
        print(msg)
        print(' ', '\n  '.join(violations))

def process_args(args): 
    cmd = args.pop(0)
    pi = False
    v = False

    if '-pi' in args:
        args.remove('-pi')
        pi = True
        
    if '-v' in args:
        args.remove('-v')
        v = True
        
    if not args:
        usage_exit(cmd)
        
    return (pi, v, args)

def analyse_intersections(type, intersections, verbose):
    if not intersections[1]:
        return  # no intersections in list
        
    print_preamble(proc_names[type], 'intersection', intersections[0],
        verbose)
        
    for inter in intersections[1]:
        print_violations('jobs processed by both {}{} and {}{}:'
                            .format(type, inter[0], type, inter[1]),
                            inter[2], verbose)
    
def analyse_duplicates(type, duplicates, verbose):
    print_preamble(proc_names[type], 'duplicate', duplicates[0], verbose)

    for k in duplicates[1]:
        print_violations('duplicate jobs for {}{}:'.format(type, k), 
            duplicates[1][k], verbose)
                
def analyse_differences(prod_union, cons_union, verbose):
    if prod_union and cons_union:
        inprod_notcons = prod_union.difference(cons_union)
        incons_notprod = cons_union.difference(prod_union)
    
        print_preamble('producer/consumer', 'difference', 
            inprod_notcons or incons_notprod, verbose)

        print_violations('producer jobs not consumed:', inprod_notcons, verbose)
        print_violations('consumer jobs not produced:', incons_notprod, verbose)


""" MAIN PROGRAM """
(prod_intersect_byid, verbose, args) = process_args(sys.argv)

logs = read_logs(args)

for type in logs:
    intersections = get_intersections(logs, type, prod_intersect_byid)
    analyse_intersections(type, intersections, verbose)
    duplicates = get_duplicates(logs, type)
    analyse_duplicates(type, duplicates, verbose)
    
analyse_differences(get_union(logs, prod_t), get_union(logs, cons_t), verbose)

print_separator(verbose)
DO NOT EDIT THIS FILE This is a Python script for analysing logs after a simulation run. Type:     python3 analyselogs.py for usage. The return value is a dictionary/map of dictionaries of the following form:  { proctype:      { procid: [loglines] }, } The return value is a tuple that is a boolean indicator for whether  violations are detected (there is at least one non-empty set of pairwise  intersections) plus a list of tuples of the pairwise intersections for each combination of keys, e.g. for keys pid1, pid2, pid3:  (violation_detected,      [         (pid1, pid2, set_of_pid1pid2_intersections),          (pid1, pid3, set_of_pid1pid3_intersections),          (pid2, pid3, set_of_pid2pid3_intersections),     ] )  If set of all members of intersections is needed do: set([e for list_of_inter in [it[2] for it in inter] for e in list_of_inter]) The return value is the set of all log entries for the given proctype. The return value is a tuple that is a boolean indicator for whether violations where detected (there are duplicate entries any of the logs for the given  proc type) and dictionary of procid key to set of duplicates. E.g. for  keys pid1, pid2, pid3:  (violation_detected,      {   pid1: set_of_pid1_duplicates,         pid2: set_of_pid2_duplicates         pid3: set_of_pid3_duplicates     } ) # no intersections in list MAIN PROGRAM