#
# lobbyists.py - Parse and import U.S. Senate LD-1/LD-2 XML documents.
# Copyright (C) 2008 Drew Hess <dhess@bothan.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Parse and import U.S. Senate LD-1/LD-2 XML documents."""

import xml.dom.pulldom


VERSION = '0.12'


# Attribute parsers.

def _identity(x):
    return x


def _optional(x):
    if x is None:
        return 'unspecified'
    else:
        return x


def _amount(x):
    if x is None or x == ' ':
        return None
    else:
        return int(x)


def _period(x):
    if x is None or x == ' ':
        return None
    periods = {'1st Quarter (Jan 1 - Mar 31)': 'Q1',
               '2nd Quarter (Apr 1 - June 30)': 'Q2',
               '3rd Quarter (July 1 - Sep 30)': 'Q3',
               '4th Quarter (Oct 1 - Dec 31)': 'Q4',
               'Mid-Year (Jan 1 - Jun 30)': 'H1',
               'Year-End (July 1 - Dec 31)': 'H2',
               'UNDETERMINED': 'undetermined'}
    return periods[x]


def _is_gov(x):
    values = {None: 'unspecified', '0': 'n', '1': 'y'}
    return values[x]


def _status(x):
    status = {0: 'active',
              1: 'terminated',
              2: 'administratively terminated',
              3: 'undetermined'}
    return status[int(x)]


def _lobbyist_indicator(x):
    indicator = {0: 'not covered',
                 1: 'covered',
                 2: 'undetermined'}
    return indicator[int(x)]


# xml.dom.pulldom-specific code

def _filing_elements(doc):
    """The sequence of all Filing elements in a lobbyist database.

    doc - The XML document. Can be a filename, a URL or anything else
    that xml.dom.pulldom.parse takes as an argument.

    Yields a sequence of expanded DOM Filing elements.

    """
    dom = xml.dom.pulldom.parse(doc)
    for event, node in dom:
        if event == 'START_ELEMENT' and node.nodeName == 'Filing':
            dom.expandNode(node)
            yield node


def _child_elements(elt):
    """Yield a sequence of child elements of the given DOM element."""
    for child in elt.childNodes:
        if child.nodeType == xml.dom.Node.ELEMENT_NODE:
            yield child


def _attr_of(elt, attrname):
    """Get the value of an attribute of an element.

    Returns the value of an attribute of the specified DOM element, or
    None if no such attribute exists in the element.

    elt - The DOM element.

    attrname - The name of the attribute to retrieve.

    """
    val = elt.getAttribute(attrname)
    if val == '':
        return None
    else:
        return val


def _element_name(elt):
    """The name of the given element."""
    return elt.tagName


# Parsers for DOM elements and their child elements.

def _parse_attrs(elt, attrs):
    """Parse the attributes of a DOM element into a sequence of pairs.

    elt - The DOM element.

    attrs - A sequence of tuples of 3 items each. The first item is
    the attribute name (a string). The second is the identifier
    associated with the parsed attribute value in the yielded
    pair. The third is the parsing function. It's applied to the
    attribute's DOM value, and its output is stored with the
    identifier in the yielded pair.

    Note that the value of an attribute which doesn't appear in the
    element is None.

    Yields a sequence of (identifier, value) pairs.

    """
    for name, id, parse in attrs:
        yield (id, parse(_attr_of(elt, name)))


def _parse_element(elt, id, attrs):
    return (id, dict(_parse_attrs(elt, attrs)))


def _parse_list(list_elt, id, subelt_parser):
    lst = list()
    for subelt in _child_elements(list_elt):
        lst.append(dict([subelt_parser(subelt)]))
    return (id, lst)


_client_attrs = [('ClientCountry', 'country', _optional),
                 ('ClientID', 'senate_id', int),
                 ('ClientName', 'name', _identity),
                 ('ClientPPBCountry', 'ppb_country', _identity),
                 ('ClientPPBState', 'ppb_state', _optional),
                 ('ClientState', 'state', _optional),
                 ('ClientStatus', 'status', _status),
                 ('ContactFullname', 'contact_name', _optional),
                 ('GeneralDescription', 'description', _optional),
                 ('IsStateOrLocalGov', 'state_or_local_gov', _is_gov)]


def _parse_client(elt):
    """Parse a Client DOM element.

    elt - The Client DOM element.

    Returns a pair whose first item is the string 'client' and whose
    second item is the dictionary of parsed attributes.

    """
    return _parse_element(elt, 'client', _client_attrs)


_registrant_attrs = [('Address', 'address', _optional),
                     ('GeneralDescription', 'description', _optional),
                     ('RegistrantCountry', 'country', _identity),
                     ('RegistrantID', 'senate_id', int),
                     ('RegistrantName', 'name', _identity),
                     ('RegistrantPPBCountry', 'ppb_country', _identity)]


def _parse_registrant(elt):
    """Parse a Registrant DOM element.

    elt - The Registrant DOM element.

    Returns a pair whose first item is the string 'registrant' and
    whose second item is the dictionary of parsed attributes.

    """
    return _parse_element(elt, 'registrant', _registrant_attrs)


# LobbyistName uses the '_optional' parser. This is intentional; there
# are a handful of records in the XML documents (see 2008_3_7.xml)
# where the attribute's value is the empty string "".

_lobbyist_attrs = [('LobbyistName', 'name', _optional),
                   ('LobbyistStatus', 'status', _status),
                   ('LobbyisteIndicator', 'indicator', _lobbyist_indicator),
                   ('OfficialPosition', 'official_position', _optional)]


def _parse_lobbyist(elt):
    """Parse a Lobbyist DOM element.

    elt - The Lobbyist DOM element.

    Returns a pair whose first item is the string 'lobbyist' and whose
    second item is the dictionary of parsed attributes.

    """
    return _parse_element(elt, 'lobbyist', _lobbyist_attrs)


def _parse_lobbyists(elt):
    """Parse a Lobbyists DOM element.

    elt - The Lobbyists DOM element.

    Returns a pair whose first item is the string 'lobbyists' and
    whose second item is a list of parsed Lobbyist DOM elements, one
    for each Lobbyist sub-element of this Lobbyists element.

    """
    return _parse_list(elt, 'lobbyists', _parse_lobbyist)


_govt_entity_attrs = [('GovEntit
yName', 'name', _identity)]


def _parse_govt_entity(elt):
    """Parse a GovernmentEntity DOM element.

    elt - The GovernmentEntity DOM element.

    Returns a pair whose first item is the string 'govt_entity' and
 
   whose second item is the dictionary of parsed attributes.

    """
    return _parse_element(elt, 'govt_entity', _govt_entity_attrs)


def _parse_govt_entities(elt):
    """Parse a GovernmentEntities DOM element.

    elt - The GovernmentEntities DOM element.

    Returns a pair whose first item is the string 'govt_entities' and
    whose second item is a list of parsed GovernmentEntity DOM
    elements, one for each GovernmentEntity sub-element of this
    GovernmentEntities element.

    """
    return _parse_list(elt, 'govt_entities', _parse_govt_entity)


_issue_attrs = [('Code', 'code', _identity),
                ('SpecificIssue', 'specific_issue', _optional)]


def _parse_issue(elt):
    """Parse an Issue DOM element.

    elt - The Issue DOM element.

    Returns a pair whose first item is the string 'issue' and whose
    second item is the dictionary of parsed attributes.

    """
    return _parse_element(elt, 'issue', _issue_attrs)


d