"""
Utility functions for NetLogger modules and command-line programs
"""
__rcsid__ = "$Id: util.py 27069 2011-02-08 20:09:10Z dang $"
__author__ = "Dan Gunter (dkgunter (at) lbl.gov)"
import glob
import os
import queue
import re
import signal
import sys
import time
import traceback
from asyncore import compact_traceback
from copy import copy
from optparse import Option, OptionParser, OptionValueError, make_option

#
import Pegasus.netlogger
from Pegasus.netlogger import configobj

try:
    from hashlib import md5

    md5_new = md5
except ImportError:
    import md5

    md5_new = md5.new

## Globals

NL_HOME = "NETLOGGER_HOME"

EOF_EVENT = "netlogger.EOF"

MAGICDATE_EXAMPLES = ", ".join(
    "%s" % s
    for s in (
        "<N> weeks|days|hours|minutes|seconds time ago",
        "Today",
        "Now",
        "Tomorrow",
        "Yesterday",
        "4th [[Jan] 2003]",
        "Jan 4th 2003",
        "mm/dd/yyyy (first)",
        "dd/mm/yyyy (second)",
        "yyyy-mm-dd",
        "yyyymmdd",
        "next Tuesday",
        "last Tuesday",
    )
)

DATA_DIR = os.path.join(os.path.dirname(Pegasus.netlogger.__file__), "data")

## Exceptions


class ConfigError(Exception):
    """Use this exception for configuration-time error handling."""


class DBConnectError(Exception):
    """Used for failed database connections."""


## Classes


class ScriptOptionParser(OptionParser):
    standard_option_list = (
        make_option(
            "-q",
            "--quiet",
            action="store_true",
            dest="quiet",
            help="only report errors",
        ),
        make_option(
            "-v",
            "--verbosity",
            default=0,
            action="count",
            dest="verbosity",
            help="show verbose status information, " "repeat for even more detail",
        ),
    )


class CountingQueue(queue.Queue):
    """Wrapper around Queue that counts how many items
    were processed, for accounting purposes.
    """

    def __init__(self, type_, *args):
        queue.Queue.__init__(self, *args)
        self.n = 0
        if type_ == "get":
            self._gets, self._puts = True, False
        else:
            self._gets, self._puts = False, True

    def get(self, *args, **kwargs):
        if self._gets:
            self.n += 1
        return queue.Queue.get(self, *args, **kwargs)

    def put(self, *args, **kwargs):
        if self._puts:
            self.n += 1
        return queue.Queue.put(self, *args, **kwargs)

    def getNumProcessed(self):
        return self.n


class FIFODict:
    """A container that is limited to a fixed size.
    When that size is exceeded, every new item inserted
    results in an item removed (in FIFO order).
    Any 'old' items do not replace newer ones.
    The interface is specialized for adding an item and checking
    that it is new at the same time.
    """

    class CircularBuffer:
        """Specialized circular buffer"""

        def __init__(self, size):
            self._data = []
            self._size = size
            self._p = 0

        def getput(self, item):
            """Put item in buffer.
            If the buffer is full, return the oldest element,
            otherwise return None.
            """
            if len(self._data) == self._size:
                oldest = self._data[self._p]
                self._data[self._p] = item
                self._p = (self._p + 1) % self._size
            else:
                oldest = None
                self._data.append(item)
            return oldest

    def __init__(self, size):
        self._q = self.CircularBuffer(size)
        self._data = {}

    def add(self, key):
        """Add a key.
        If it is new, return True otherwise False.
        """
        if key in self._data:
            return False
        self._data[key] = 1
        removed = self._q.getput(key)
        if removed is not None:
            del self._data[removed]
        return True


def traceback():
    """Traceback as a string with no newlines."""
    return str(compact_traceback())


def parse_nvp(args):
    d = {}
    for arg in args:
        try:
            name, value = arg.split("=")
        except ValueError:
            pass
        d[name] = value
    return d


def tzstr():
    return "%s%02d:%02d" % (
        ("+", "-")[time.timezone > 0],
        time.timezone / 3600,
        (time.timezone - int(time.timezone / 3600) * 3600) / 60,
    )


def parseDatetime(d, utc=False):
    """Parse a datetime object, or anything that formats itself
    with isoformat(), to number of seconds since epoch.
    """
    from Pegasus.netlogger.parsers.base import parseDate

    if d is None:
        raise ValueError("date is None")
    iso = d.isoformat()
    # a
dd 'midnight' time if none given
    if "T" not in iso:
        iso += "T00:00:00"
    # only append timezone if none is there already
    if not iso.endswith("Z") 
and not re.match(r".*[+-]\d\d:\d\d$", iso):
        if utc:
            iso += "Z"
        else:
            iso += tzstr()
    return parseDate(iso)


class ProgressMeter:
    """A simple textual progress meter."""

    REPORT_INTERVAL = 1000

    def __init__(self, ofile, units="lines"):
        self.ofile = ofile
        self.units = units
        self._counter = 0
        self.reset(0)

    def reset(self, n):
        self.t0 = time.time()
        self.last_report = n

    def advance(self, num=0, inc=1):
        if num == 0:
            self._counter += inc
            num = self._counter
        if num - self.last_report >= self.REPORT_INTERVAL:
            n = num - self.last_report
            dt = time.time() - self.t0
            rate = n / dt
            if rate < 1:
                dig = 2
            elif rate < 10:
                dig = 1
            else:
                dig = 0
            fmt = "[ %%5d ] %%s, rate = %%.%df %%s/sec     \r" % dig
            self.ofile.write(fmt % (num, self.units, rate, self.units))
            self.ofile.flush()
            self.reset(num)


class NullProgressMeter:
    """Substitute for ProgressMeter when you don't want anything to
    actually be printed.
    """

    def __init__(self, ofile=None, units=None):
        return

    def reset(self, n):
        pass

    def advance(self, num=0):
        pass


def mostRecentFile(dir, file_pattern, after_time=None):
    """Search 'dir' for all files matching glob 'file_pattern',
    and return the mostRecent one(s). If 'after_time' is given,
    it should be a number of seconds since the epoch UTC; no files
    will be returned if none is on or after this time.

    Returns a list of the full paths to file(s), or an empty list.
    More than one file may be returned, in the case that they have
    the same modification time.

    """
    if not os.path.isdir(dir):
        return []
    search_path = os.path.join(dir, file_pattern)
    # make a sortable list of filenames and modification times
    timed_files = [
        (os.stat_result(os.stat(fname)).st_mtime, fname)
        for fname in glob.glob(search_path)
    ]
    # if the list is empty, stop
    if not timed_files:
        return []
    # reverse sort so most-recent is first
    timed_files.sort(reverse=True)
    most_recent_time = timed_files[0][0]
    # return nothing if the most recent time is not
    # after the cutoff
    if after_time is not None and most_recent_time < int(after_time):
        return []
    # start with most recent, then append all 'ties'
    result = [timed_files[0][1]]
    i = 1
    try:
        while timed_files[i][0] == most_recent_time:
            result.append(timed_files[i][1])
            i += 1
    except IndexError:
        pass  # ran off end of list. all ties (!)
    # return all 'most recent' files
    return result


def daemonize(log=None, root_log=None, close_fds=True):
    """Make current process into a daemon."""
    # Do a double-fork so that the daemon process is completely
    # detached from its parent (it becomes a child of init).
    # For details the classic text is:
    # W. Richard Stevens, "Advanced Programming in the UNIX Environment"
    log.info("daemonize.start")
    log.debug("