"""HTML form handling for web clients.

HTML form handling for web clients: useful for parsing HTML forms, filling them
in and returning the completed forms to the server.  This code developed from a
port of Gisle Aas' Perl module HTML::Form, from the libwww-perl library, but
the interface is not the same.

The most useful docstring is the one for HTMLForm.

RFC 1866: HTML 2.0
RFC 1867: Form-based File Upload in HTML
RFC 2388: Returning Values from Forms: multipart/form-data
HTML 3.2 Specification, W3C Recommendation 14 January 1997 (for ISINDEX)
HTML 4.01 Specification, W3C Recommendation 24 December 1999


Copyright 2002-2007 John J. Lee <jjl@pobox.com>
Copyright 2005 Gary Poster
Copyright 2005 Zope Corporation
Copyright 1998-2000 Gisle Aas.

This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses (see the file
COPYING.txt included with the distribution).

"""

# TODO:
# Clean up post the merge into mechanize
#  * Remove code that was duplicated in ClientForm and mechanize
#  * Remove 
weird import stuff
#  * Remove pre-Python 2.4 compatibility cruft
#  * Clean up tests
#  * Later release: Remove the ClientForm 0.1 backwards-compatibility switch
# Remove parser testing hack
# Clean action URI
# Switch to unicode throughout
#  See Wichert Akkerman's 2004-01-22 message to c.l.py.
# Apply recommendations from google code project CURLIES
# Apply recommendations from HTML 5 spec
# A
dd charset parameter to Content-type headers?  How to find value??
# Functional tests to add:
#  Single and multiple file upload
#  File upload with missing name (check standards)
# mailto: submission & enctype text/plain??

# Replace by_label etc. with moniker / selector concept.  Allows, e.g., a
#  choice between selection by value / id / label / element contents.  Or
#  choice between matching labels exactly or by substring.  etc.


__all__ = ['AmbiguityError', 'CheckboxControl', 'Control',
           'ControlNotFoundError', 'FileControl', 'FormParser', 'HTMLForm',
           'HiddenControl', 'IgnoreControl', 'ImageControl', 'IsindexControl',
           'Item', 'ItemCountError', 'ItemNotFoundError', 'Label',
           'ListControl', 'LocateError', 'Missing', 'ParseError', 'ParseFile',
           'ParseFileEx', 'ParseResponse', 'ParseResponseEx','PasswordControl',
           'RadioControl', 'ScalarControl', 'SelectControl',
           'SubmitButtonControl', 'SubmitControl', 'TextControl',
           'TextareaControl', 'XHTMLCompatibleFormParser']

import HTMLParser
from cStringIO import StringIO
import inspect
import logging
import random
import re
import sys
import urllib
import urlparse
import warnings

import _beautifulsoup
import _request

# from Python itself, for backwards compatibility of raised exceptions
import sgmllib
# bundled copy of sgmllib
import _sgmllib_copy


VERSION = "0.2.11"

CHUNK = 1024  # size of chunks fed to parser, in bytes

DEFAULT_ENCODING = "latin-1"

_logger = logging.getLogger("mechanize.forms")
OPTIMIZATION_HACK = True

def debug(msg, *args, **kwds):
    if OPTIMIZATION_HACK:
        return

    caller_name = inspect.stack()[1][3]
    extended_msg = '%%s %s' % msg
    extended_args = (caller_name,)+args
    _logger.debug(extended_msg, *extended_args, **kwds)

def _show_debug_messages():
    global OPTIMIZATION_HACK
    OPTIMIZATION_HACK = False
    _logger.setLevel(logging.DEBUG)
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.DEBUG)
    _logger.addHandler(handler)


def deprecation(message, stack_offset=0):
    warnings.warn(message, DeprecationWarning, stacklevel=3+stack_offset)


class Missing: pass

_compress_re = re.compile(r"\s+")
def compress_text(text): return _compress_re.sub(" ", text.strip())

def normalize_line_endings(text):
    return re.sub(r"(?:(?<!\r)\n)|(?:\r(?!\n))", "\r\n", text)


def unescape(data, entities, encoding=DEFAULT_ENCODING):
    if data is None or "&" not in data:
        return data

    def replace_entities(match, entities=entities, encoding=encoding):
        ent = match.group()
        if ent[1] == "#":
            return unescape_charref(ent[2:-1], encoding)

        repl = entities.get(ent)
        if repl is not None:
            if type(repl) != type(""):
                try:
                    repl = repl.encode(encoding)
                except UnicodeError:
                    repl = ent
        else:
            repl = ent

        return repl

    return re.sub(r"&#?[A-Za-z0-9]+?;", replace_entities, data)

def unescape_charref(data, encoding):
    name, base = data, 10
    if name.startswith("x"):
        name, base= name[1:], 16
    uc = unichr(int(name, base))
    if encoding is None:
        return uc
    else:
        try:
            repl = uc.encode(encoding)
        except UnicodeError:
            repl = "&#%s;" % data
        return repl

def get_entitydefs():
    import htmlentitydefs
    from codecs import latin_1_decode
    entitydefs = {}
    try:
        htmlentitydefs.name2codepoint
    except AttributeError:
        entitydefs = {}
        for name, char in htmlentitydefs.entitydefs.items():
            uc = latin_1_decode(char)[0]
            if uc.startswith("&#") and uc.endswith(";"):
                uc = unescape_charref(uc[2:-1], None)
            entitydefs["&%s;" % name] = uc
    else:
        for name, codepoint in htmlentitydefs.name2codepoint.items():
            entitydefs["&%s;" % name] = unichr(codepoint)
    return entitydefs


def issequence(x):
    try:
        x[0]
    except (TypeError, KeyError):
        return False
    except IndexError:
        pass
    return True

def isstringlike(x):
    try: x+""
    except: return False
    else: return True


def choose_boundary():
    """Return a string usable as a multipart boundary."""
    # follow IE and firefox
    nonce = "".join([str(random.randint(0, sys.maxint-1)) for i in 0,1,2])
    return "-"*27 + nonce

# This cut-n-pasted MimeWriter from standard library is here so can add
# to HTTP headers rather than message body when appropriate.  It also uses
# \r\n in place of \n.  This is a bit nasty.
class MimeWriter:

    """Generic MIME writer.

    Methods:

    __init__()
    addheader()
    flushheaders()
    startbody()
    startmultipartbody()
    nextpart()
    lastpart()

    A MIME writer is much more primitive than a MIME parser.  It
    doesn't seek around on the output file, and it doesn't use large
    amounts of buffer space, so you have to write the parts in the
    order they should occur on the output file.  It does buffer the
    headers you add, allowing you to rearrange their order.

    General usage is:

    f = <open the output file>
    w = MimeWriter(f)
    ...call w.addheader(key, value) 0 or more times...

    followed by either:

    f = w.startbody(content_type)
    ...call f.write(data) for body data...

    or:

    w.startmultipartbody(subtype)
    for each part:
        subwriter = w.nextpart()
        ...use the subwriter's methods to create the subpart...
    w.lastpart()

    The subwriter is another MimeWriter instance, and should be
    treated in the same way as the toplevel MimeWriter.  This way,
    writing recursive body parts is easy.

    Warning: don't forget to call lastpart()!

    XXX There should be more state so calls made in the wrong order
    are detected.

    Some special cases:

    - startbody() just returns the file passed to the constructor;
      but don't use this knowledge, as it may be changed.

    - startmultipartbody() actually returns a file as well;
      this can be used to write the initial 'if you can read this your
      mailer is not MIME-aware' message.

    - If you call flushheaders(), the headers accumulated so far are
      written out (and forgotten); this is useful if you don't need a
      body part at all, e.g. for a subpart of type message/rfc822
      that's (mis)