import re
import sys
import logging
import asyncio
import textwrap

import validators
from itertools import product
from more_itertools import always_iterable

from hypotonic import request
from hypotonic.context import Context

logger = logging.getLogger('hypotonic')


async def get(session, context, data, urls, params=None):
  def resolve_params(params):
    """Resolve parameters given as callables."""
    if params:
      for key, param in params.items():
        if callable(param):
          params[key] = param(context, data)
      return params
    return None

  def explode_params(params):
    """Explode parameters specified in array notation similarly to curl.
    For reference see https://curl.haxx.se/docs/manpage.html"""
    if params:
      for values in product(*map(always_iterable, params.values())):
        yield dict(zip(params.keys(), values))
    else:
      yield None

  for url in always_iterable(urls):
    for params in explode_params(resolve_params(params)):
      context = Context.create(*await request.get(session, url, params))
      yield context, data


async def post(session, _, data, urls, payload=None):
  for url in always_iterable(urls):
    context = Context.create(*await request.post(session, url, payload))
    yield context, data


async def find(_, context, data, selector):
  for result in context.select(selector):
    yield result, data


async def set(_, context, data, descriptor):
  if isinstance(descriptor, str):
    data = {**data, descriptor: context.text()}
  else:
    for key, selector in descriptor.items():
      if isinstance(selector, str):
        results = context.select(selector)
        data = {**data, key: results[0].text()}
      elif isinstance(selector, list):
        results = context.select(selector[0])
        values = [result.text() for result in results]
        data = {**data, key: values}
      elif callable(selector):
        data = {**data, key: selector(context, data)}
  yield context, data


async def then(_, context, data, func):
  context, data = func(context, data)
  yield context, data


async def follow(session, context, data, selector):
  for result in context.select(selector):
    url = result.text()
    context = Context.create(*await request.get(session, url))
    yield context, data


async def paginate(session, context, data, selector, limit=sys.maxsize, param=None):
  page = 1
  while True:
    yield context, data

    page += 1
    results = context.select(selector)
    if page > limit or len(results) == 0:
      break

    result = results[0].text()
    if isinstance(result, str) and validators.url(result):
      url = result
    else:
      url = str(context.url % {param: page})

    context = Context.create(*await request.get(session, url))


async def filter(_, context, data, selector):
  if len(context.select(selector)) > 0:
    yield context, data


async def match(_, context, data, regex, flags=0):
  if re.search(regex, context.text(), flags=flags):
    yield context, data


async def delay(_, context, data, secs):
  await asyncio.sleep(secs)
  yield context, data


async def log(_, context, data, level=logging.INFO):
  logger.log(level, f"{len(data)}:{textwrap.shorten(str(context), width=72)}")
  yield context, data

import re
import sys
import logging
import asyncio
import textwrap

import validators
from itertools import product
from more_itertools import always_iterable

from hypotonic import request
from hypotonic.context import Context

logger = logging.getLogger('hypotonic')


async def get(session, context, data, urls, params=None):
  def resolve_params(params):
    """Resolve parameters given as callables."""
    if params:
      for key, param in params.items():
        if callable(param):
          params[key] = param(context, data)
      return params
    return None

  def explode_params(params):
    """Explode parameters specified in array notation similarly to curl.
    For reference see https://curl.haxx.se/docs/manpage.html"""
    if params:
      for values in product(*map(always_iterable, params.values())):
        yield dict(zip(params.keys(), values))
    else:
      yield None

  for url in always_iterable(urls):
    for params in explode_params(resolve_params(params)):
      context = Context.create(*await request.get(session, url, params))
      yield context, data


async def post(session, _, data, urls, payload=None):
  for url in always_iterable(urls):
    context = Context.create(*await request.post(session, url, payload))
    yield context, data


async def find(_, context, data, selector):
  for result in context.select(selector):
    yield result, data


async def set(_, context, data, descriptor):
  if isinstance(descriptor, str):
    data = {**data, descriptor: context.text()}
  else:
    for key, selector in descriptor.items():
      if isinstance(selector, str):
        results = context.select(selector)
        data = {**data, key: results[0].text()}
      elif isinstance(selector, list):
        results = context.select(selector[0])
        values = [result.text() for result in results]
        data = {**data, key: values}
      elif callable(selector):
        data = {**data, key: selector(context, data)}
  yield context, data


async def then(_, context, data, func):
  context, data = func(context, data)
  yield context, data


async def follow(session, context, data, selector):
  for result in context.select(selector):
    url = result.text()
    context = Context.create(*await request.get(session, url))
    yield context, data


async def paginate(session, context, data, selector, limit=sys.maxsize, param=None):
  page = 1
  while True:
    yield context, data

    page += 1
    results = context.select(selector)
    if page > limit or len(results) == 0:
      break

    result = results[0].text()
    if isinstance(result, str) and validators.url(result):
      url = result
    else:
      url = str(context.url % {param: page})

    context = Context.create(*await request.get(session, url))


async def filter(_, context, data, selector):
  if len(context.select(selector)) > 0:
    yield context, data


async def match(_, context, data, regex, flags=0):
  if re.search(regex, context.text(), flags=flags):
    yield context, data


async def delay(_, context, data, secs):
  await asyncio.sleep(secs)
  yield context, data


async def log(_, context, data, level=logging.INFO):
  logger.log(level, f"{len(data)}:{textwrap.shorten(str(context), width=72)}")
  yield context, data

Resolve parameters given as callables. Explode parameters specified in array notation similarly to curl.     For reference see https://curl.haxx.se/docs/manpage.html