# -*- coding: utf-8 -*-
# Copyright: 2011, Grigoriy Petukhov
# Author: Grigoriy Petukhov (http://lorien.name)
# License: BSD
"""
The core of grab package: the Grab class.
"""
from __future__ import absolute_import
import logging
import os
#import urllib
from random import randint, choice
from copy import copy
import threading
import itertools
import collections
try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin
import time
import re
import json
import email
from datetime import datetime

from .proxylist import ProxyList, parse_proxyline
from .tools.html import find_refresh_url, find_base_url
from .response import Response
from . import error
from .upload import UploadContent, UploadFile
from .tools.http import normalize_http_values, normalize_url
from .extension import register_extensions

from grab.util.py2old_support import *
from grab.util.py3k_support import *

# This counter will used in enumerating network queries.
# Its value wil
l be displayed in logging messages and also used
# in names of dumps
# I
 use mutable module variable to allow different
# instances of Grab maintain single counter
# This could be helpful in debuggin when your script
# creates multiple Grab instances - in case of shared counter
# grab instances do not overwrite dump logs
GLOBAL_STATE = {
    'dom_build_time': 0,
    'selector_time': 0,
}
REQUEST_COUNTER = itertools.count(1)

# Some extensions need GLOBAL_STATE variable
# what's why they go after GLOBAL_STATE definition
from .ext.lxml import LXMLExtension
from .ext.form import FormExtension
from .ext.django import DjangoExtension
from .ext.text import TextExtension
from .ext.rex import RegexpExtension
from .ext.pquery import PyqueryExtension
from .ext.ftp import FTPExtension
from .ext.doc import DocExtension
from .ext.kit import KitExtension

__all__ = ('Grab', 'UploadContent', 'UploadFile')

MUTABLE_CONFIG_KEYS = ['post', 'multipart_post', 'headers', 'cookies',
                       'hammer_timeouts']

logger = logging.getLogger('grab.base')

# Logger to handle network activity
# It is separate logger to allow you easily
# control network logging separately from other grab logs
logger_network = logging.getLogger('grab.network')

def copy_config(config, mutable_config_keys=MUTABLE_CONFIG_KEYS):
    """
    Copy grab config ojection with correct handling
    of mutable config values.
    """

    cloned_config = copy(config)
    # Apply ``copy`` function to mutable config values
    for key in mutable_config_keys:
        cloned_config[key] = copy(config[key])
    return cloned_config


def default_config():
    # TODO: it seems config should be splitted into two entities:
    # 1) config which is not changed during request
    # 2) changable config
    return dict(
        # Common
        url = None,

        # Debugging
        log_file = None,
        log_dir = False,
        debug_post = False,
        # Only for curl transport
        debug = False,
        verbose_logging = False,

        # Only for selenium transport
        webdriver = 'firefox',
        selenium_wait = 1, # in seconds

        # Proxy
        proxy = None,
        proxy_type = None,
        proxy_userpwd = None,
        proxy_auto_change = True,

        # Method, Post
        method = None,
        post = None,
        multipart_post = None,

        # Headers, User-Agent, Referer
        headers = {},
        common_headers = {},
        user_agent = None,
        user_agent_file = None,
        referer = None,
        reuse_referer = True,

        # Cookies
        cookies = {},
        reuse_cookies = True,
        cookiefile = None,

        # Timeouts
        timeout = 15,
        connect_timeout = 10,
        hammer_mode = False,
        hammer_timeouts = ((2, 5), (5, 10), (10, 20), (15, 30)),

        # Response processing
        nobody = False,
        body_maxsize = None,
        body_inmemory = True,
        body_storage_dir = None,
        body_storage_filename = None,

        # Content compression
        encoding = 'gzip',

        # Network interface
        interface = None,

        # Redirects
        follow_refresh = False,
        follow_location = True,
        refresh_redirect_count = 0,
        redirect_limit = 10,

        # Authentication
        userpwd = None,

        # Character set to which any unicode data should be encoded
        # before get placed in request
        # This setting is overwritten after each request with
        # charset of rertreived document
        charset = 'utf-8',

        # Charset to use for converting content of response
        # into unicode, by default it is detected automatically
        document_charset = None,

        # Conent type control how DOM are built
        # For html type HTML DOM builder is used
        # For xml type XML DOM builder is used
        content_type = 'html',

        # Fix &#X; entities, where X between 128 and 160
        # Such entities are parsed by modern browsers as
        # windows-1251 entities independently of the real charset of
        # the document, If this option is True then such entities
        # will be replaced with correct unicode entitites e.g.:
        # &#151; ->  &#8212;
        fix_special_entities = True,

        # Convert document body to lower case before bulding LXML tree
        # It does not affect `response.body`
        lowercased_tree = False,

        # Strip null bytes from document body before building lXML tree
        # It does not affect `response.body`
        strip_null_bytes = True,
    )


class Grab(
           LXMLExtension, FormExtension, PyqueryExtension,
           DjangoExtension, TextExtension, RegexpExtension,
           FTPExtension, DocExtension, KitExtension,
           ):

    __slots__ = ('request_head', 'request_log', 'request_body',
                 'proxylist', 'config', '_request_prepared',
                 'clone_counter', 'response', 'transport',
                 'transport_param', 'request_method', 'request_counter',
                 '__weakref__',

                 # Dirst hack to make it possbile to inherit Grab from
                 # multiple base classes with __slots__
                 '_lxml_form', '_file_fields',
                 '_lxml_tree', '_strict_lxml_tree',
                 '_pyquery', '_doc', '_kit',
                 )

    # Points which could be handled in extension classes
    extension_points = ('config', 'init', 'reset')

    # Attributes which should be processed when clone
    # of Grab instance is creating
    clonable_attributes = ('request_head', 'request_log', 'request_body',
                           'proxylist')

    # Complex config items which points to mutable objects
    mutable_config_keys = copy(MUTABLE_CONFIG_KEYS)

    """
    Public methods
    """

    def __init__(self, response_body=None, transport='grab.transport.curl.CurlTransport',
                 **kwargs):
        """
        Create Grab instance
        """

        self.config = default_config()
        self.config['common_headers'] = self.common_headers()
        self.trigger_extensions('config')
        self.trigger_extensions('init')
        self._request_prepared = False

        self.setup_transport(transport)

        self.reset()
        self.proxylist = None
        if kwargs:
            self.setup(**kwargs)
        self.clone_counter = 0
        if response_body is not None:
            self.fake_response(response_body)

    def setup_transport(self, transport_param):
        self.transport_param = transport_param
        if isinstance(transport_param, basestring):
            mod_path, cls_name = transport_param.rsplit('.', 1)
            mod = __import__(mod_path, globals(), locals(), ['foo'])
            self.transport = getattr(mod, cls_name)()
        elif isinstance(transport_param, collections.Callable):
            self.transport = transport_param()
        else:
            raise GrabMisuseError('Option `transport` should be string or callable. '\
                                  'Got %s' % type(transport_param))

    def reset(self):
        """
        Reset all attributes which could be modified d