"""Fork of urllib2.

When reading this, don't assume that all code in here is reachable.  Code in
the rest of mechanize may be used instead.

Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Python
Software Foundation; All Rights Reserved

Copyright 2002-2009 John J Lee <jjl@pobox.com>

This code is free software; y
ou can redistribute it and/or modify it
under the terms 
of the BSD or ZPL 2.1 licenses (see the file
COPYING.txt included with the distribution).

"""

# XXX issues:
# If an authentication error handler that tries to perform
# authentication for some reason but fails, how should the error be
# signalled?  The client needs to know the HTTP error code.  But if
# the handler knows that the problem was, e.g., that it didn't know
# that hash algo that requested in the challenge, it would be good to
# pass that information along to the client, too.
# ftp errors aren't handled cleanly
# check digest against correct (i.e. non-apache) implementation

# Possible extensions:
# complex proxies  XXX not sure what exactly was meant by this
# abstract factory for opener

import base64
import bisect
import copy
import httplib
import logging
import mimetools
import os
import posixpath
import random
import re
import socket
import sys
import time
from urllib import (unwrap, unquote, splittype, quote,
     addinfourl, splitport,
     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
from urllib import localhost, url2pathname, getproxies  # @UnusedImport
import urllib
from urllib2 import HTTPError, URLError
import urlparse

from _clientcookie import CookieJar
import _request
from _response import closeable_response
import _rfc3986
import _sockettimeout


try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

try:
    import hashlib
except ImportError:
    # python 2.4
    import md5
    import sha
    def sha1_digest(bytes):  # @ReservedAssignment
        return sha.new(bytes).hexdigest()
    def md5_digest(bytes):  # @ReservedAssignment
        return md5.new(bytes).hexdigest()
else:
    def sha1_digest(bytes):  # @ReservedAssignment
        return hashlib.sha1(bytes).hexdigest()
    def md5_digest(bytes):  # @ReservedAssignment
        return hashlib.md5(bytes).hexdigest()


try:
    socket._fileobject("fake socket", close=True)
except TypeError:
    # python <= 2.4
    create_readline_wrapper = socket._fileobject
else:
    def create_readline_wrapper(fh):
        return socket._fileobject(fh, close=True)


# python 2.4 splithost has a bug in empty path component case
_hostprog = None
def splithost(url):
    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
    global _hostprog
    if _hostprog is None:
        import re  # @Reimport
        _hostprog = re.compile('^//([^/?]*)(.*)$')

    match = _hostprog.match(url)
    if match: return match.group(1, 2)
    return None, url



# support for FileHandler, proxies via environment variables





# used in User-Agent header sent
__version__ = sys.version[:3]

_opener = None
def urlopen(url, data=None, timeout=_sockettimeout._GLOBAL_DEFAULT_TIMEOUT):
    global _opener
    if _opener is None:
        _opener = build_opener()
    return _opener.open(url, data, timeout)

def install_opener(opener):
    global _opener
    _opener = opener

# copied from cookielib.py
_cut_port_re = re.compile(r":\d+$")
def request_host(request):
    """Return request-host, as defined by RFC 2965.

    Variation from RFC: returned value is lowercased, for convenient
    comparison.

    """
    url = request.get_full_url()
    host = urlparse.urlparse(url)[1]
    if host == "":
        host = request.get_header("Host", "")

    # remove port, if present
    host = _cut_port_re.sub("", host, 1)
    return host.lower()

class Request:

    def __init__(self, url, data=None, headers={},
                 origin_req_host=None, unverifiable=False):
        # unwrap('<URL:type://host/path>') --> 'type://host/path'
        self.__original = unwrap(url)
        self.type = None
        # self.__r_type is what's left after doing the splittype
        self.host = None
        self.port = None
        self._tunnel_host = None
        self.data = data
        self.headers = {}
        for key, value in headers.items():
            self.add_header(key, value)
        self.unredirected_hdrs = {}
        if origin_req_host is None:
            origin_req_host = request_host(self)
        self.origin_req_host = origin_req_host
        self.unverifiable = unverifiable

    def __getattr__(self, attr):
        # XXX this is a fallback mechanism to guard against these
        # methods getting called in a non-standard order.  this may be
        # too complicated and/or unnecessary.
        # XXX should the __r_XXX attributes be public?
        if attr[:12] == '_Request__r_':
            name = attr[12:]
            if hasattr(Request, 'get_' + name):
                getattr(self, 'get_' + name)()
                return getattr(self, attr)
        raise AttributeError, attr

    def get_method(self):
        if self.has_data():
            return "POST"
        else:
            return "GET"

    # XXX these helper methods are lame

    def add_data(self, data):
        self.data = data

    def has_data(self):
        return self.data is not None

    def get_data(self):
        return self.data

    def get_full_url(self):
        return self.__original

    def get_type(self):
        if self.type is None:
            self.type, self.__r_type = splittype(self.__original)
            if self.type is None:
                raise ValueError, "unknown url type: %s" % self.__original
        return self.type

    def get_host(self):
        if self.host is None:
            self.host, self.__r_host = splithost(self.__r_type)
            if self.host:
                self.host = unquote(self.host)
        return self.host

    def get_selector(self):
        scheme, authority, path, query, fragment = _rfc3986.urlsplit(  # @UnusedVariable
            self.__r_host)
        if path == "":
            path = "/"  # RFC 2616, section 3.2.2
        fragment = None  # RFC 3986, section 3.5
        return _rfc3986.urlunsplit([scheme, authority, path, query, fragment])

    def set_proxy(self, host, type):  # @ReservedAssignment
        orig_host = self.get_host()
        if self.get_type() == 'https' and not self._tunnel_host:
            self._tunnel_host = orig_host
        else:
            self.type = type
            self.__r_host = self.__original

        self.host = host

    def has_proxy(self):
        """Private method."""
        # has non-HTTPS proxy
        return self.__r_host == self.__original

    def get_origin_req_host(self):
        return self.origin_req_host

    def is_unverifiable(self):
        return self.unverifiable

    def add_header(self, key, val):
        # useful for something like authentication
        self.headers[key.capitalize()] = val

    def add_unredirected_header(self, key, val):
        # will not be added to a redirected request
        self.unredirected_hdrs[key.capitalize()] = val

    def has_header(self, header_name):
        return (header_name in self.headers or
                header_name in self.unredirected_hdrs)

    def get_header(self, header_name, default=None):
        return self.headers.get(
            header_name,
            self.unredirected_hdrs.get(header_name, default))

    def header_items(self):
        hdrs = self.unredirected_hdrs.copy()
        hdrs.update(self.headers)
        return hdrs.items()

class OpenerDirector:
    def __init__(self):
        client_version = "Python-urllib/%s" % __version__
        self.addheaders = [('User-agent', client_version)]
        # manage the individual handlers
        self.handlers = []
        self.handle_open = {}
        self.handle_error = {}
        self.process_response = {}
        self.process_request = {}

    def add_handler(self, handler):
        if not hasattr(handler, "add_parent"):
            raise TypeError("expected BaseHandler instance, got %r" %
                            type(handler))

        added = False
        for me