<gh_stars>1-10
import os
import time
import click
import logging
import requests

import datetime
import humanize
from tqdm.auto import tqdm
from functools import wraps


log = logging.getLogger("twarc")


def rate_limit(f, tries=30):
    """
    A decorator to handle rate limiting from the Twitter v2 API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.
    """

    @wraps(f)
    def new_f(*args, **kwargs):
        errors = 0
        while True:
            resp = f(*args, **kwargs)
            if resp.status_code in [200, 201]:
                errors = 0
                return resp
            elif resp.status_code == 429:

                # Check the headers, and try to infer why we're hitting the
                # rate limit. Because the search/all endpoints also have a
                # 1r/s rate limit that isn't obvious in the headers, we need
                # to infer the reason for the rate limit. Note that this is
                # included to help debug problems with multiple concurrent
                # clients - this shouldn't be hit in normal of operation of a
                # single twarc client.
                remaining = int(resp.headers["x-rate-limit-remaining"])

                # If we have a 429 rate limit, but there are remaining calls for
                # this endpoint, we've probably hit the 1r/s limit.
                if remaining:
                    log.warning(
                        "Hit the 1 request/second rate limit, sleeping for 10 seconds. "
                        "This shouldn't happen with normal usage of twarc, and may indicate "
                        "multiple clients interacting with the Twitter API at the "
                        "same time."
                    )
                    time.sleep(10)
                    continue

                # Just a regular 15 minute window rate limit.
                else:
                    reset = int(resp.headers["x-rate-limit-reset"])
                    now = time.time()

                    # The time to sleep depends on having an accurate system time,
                    # so check to see if there's something really bad happening
                    # to warn the user.
                    target_sleep_seconds = reset - now

                    # Never sleep longer than 15 minutes, as that is the basis for
                    # all of the read time based rate limits in the Twitter API
                    seconds = min(901, max(10, (target_sleep_seconds + 10)))

                    if target_sleep_seconds >= 900:
                        # If we need to sleep for more than a rate limit period, the
                        # system clock could be wrong.
                        log.warning(
                            "Detected overlong sleep interval - is your system clock accurate? "
                            "An accurate system time is needed to calculate how long to sleep for, "
                            "and data collection might be slowed. "
                            f"The rate limit resets at {reset} and the current time is {now}."
                        )
                    elif target_sleep_seconds < 0:
                        # If we need to sleep for negative time something weird might be up.
                        log.warning(
                            "Detected negative sleep interval - is your system clock accurate? "
                            "If your system time is running fast, rate limiting may not be "
                            "effective. "
                            f"The rate limit resets at {reset} and the current time is {now}."
                        )

                    log.warning("rate limit exceeded: sleeping %s secs", seconds)
                    time.sleep(seconds)

            elif resp.status_code >= 500:
                errors += 1
                if errors > tries:
                    log.warning(f"too many errors ({tries}) from Twitter, giving up")
                    resp.raise_for_status()
                seconds = errors**2
                log.warning(
                    "caught %s from Twitter API, sleeping %s", resp.status_code, seconds
                )
                time.sleep(seconds)
            else:
                log.error("Unexpected HTTP response: %s", resp)
                resp.raise_for_status()

    return new_f


def catch_request_exceptions(f, tries=30):
    """
    A decorator to handle all request exceptions. This decorator will catch
    *any* request level error, reconnect and try again. It does not handle
    HTTP protocol level errors (404, 500) etc.

    It will try up to tries times consecutively before giving up. A successful
    call to f will result in the try counter being reset to 0.
    """

    # pyOpenSSL has been known to throw these connection errors that need to be
    # caught separately: https://github.com/edsu/twarc/issues/72

    try:
        import OpenSSL

        ConnectionError = OpenSSL.SSL.SysCallError
    except:
        ConnectionError = requests.exceptions.ConnectionError

    @wraps(f)
    def new_f(self, *args, **kwargs):
        errors = 0
        while errors < tries:
            try:
                resp = f(self, *args, **kwargs)
                errors = 0
                return resp
            except (requests.exceptions.RequestException, ConnectionError) as e:

                # don't catch any HTTP errors since those are handled separately
                if isinstance(e, requests.exceptions.HTTPError):
                    raise e

                errors += 1
                log.warning("caught requests exception: %s", e)
                if errors > tries:
                    log.error(f"giving up, too many request exceptions: {tries}")
                    raise e
                seconds = errors**2
                log.info("sleeping %s", seconds)
                time.sleep(seconds)
                self.connect()

    return new_f


def interruptible_sleep(t, event=None):
    """
    Sleeps for a specified duration, optionally stopping early for event.

    Returns True if interrupted
    """
    log.info("sleeping %s", t)

    if event is None:
        time.sleep(t)
        return False
    else:
        return not event.wait(t)


class cli_api_error:
    """
    A decorator to catch HTTP errors for the command line.
    """

    def __init__(self, f):
        self.f = f
        # this is needed for click help docs to work properly
        self.__doc__ = f.__doc__

    def __call__(self, *args, **kwargs):
        try:
            return self.f(*args, **kwargs)
        except requests.exceptions.HTTPError as e:
            try:
                result = e.response.json()
                if "errors" in result:
                    for error in result["errors"]:
                        msg = error.get("message", "Unknown error")
                elif "title" in result:
                    msg = result["title"]
                else:
                    msg = "Unknown error"
            except ValueError:
                msg = f"Unable to parse {e.response.status_code} error as JSON: {e.response.text}"
        except InvalidAuthType as e:
            msg = "This command requires application authentication, try passing --app-auth"
        except ValueError as e:
            msg = str(e)
        click.echo(
            click.style("⚡ ", fg="yellow") + click.style(msg, fg="red"), err=True
        )


def requires_app_auth(f):
    """
    Ensure that application authentication is set for calls that only work in that mode.
    """

    @wraps(f)
    def new_f(self, *args, **kwargs):
        if self.auth_type != "application":
            raise InvalidAuthType(
                "This endpoint only works with application authentication"
            )

        else:
            return f(self, *args, **kwargs)

    return new_f


class InvalidAuthType(Exception):
    """
    Raised when the endpoint called is not supported by the current auth type.
    """


class FileLineProgressBar(tqdm):
    """
    A progress bar based on input file line count. Counts an input file by lines.
    This tries to read the entire file and count newlines in a robust way.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} lines of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"

        # Warn for large (> 1 GB) input files:
        if (os.stat(infile.name).st_size / (1024 * 1024 * 1024)) > 1:
            click.echo(
                click.style(
                    f"Input File Size is {os.stat(infile.name).st_size / (1024*1024):.2f} MB, it may take a while to process. CTRL+C to stop.",
                    fg="yellow",
                    bold=True,
                ),
                err=True,
            )

        with open(infile.name, "r", encoding="utf-8", errors="ignore") as f:
            total_lines = sum(1 for _ in f)

        kwargs["total"] = total_lines if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        and drop in compatibility with FileSizeProgressBar otherwise use tqdm's update().
        """

        try:
            if "data" in result:
                for item in result["data"]:
                    self.update()
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update()
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")


class FileSizeProgressBar(tqdm):
    """
    An input file size based progress bar. Counts an input file in bytes.
    This will also dig into the responses and add up the outputs to match the file size.
    Overrides `disable` parameter if file is a pipe.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["unit"] = "B"
        kwargs["unit_scale"] = True
        kwargs["unit_divisor"] = 1024
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
        kwargs["total"] = os.stat(infile.name).st_size if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        otherwise use twdm's own update() method.
        """
        try:
            if "data" in result:
                for item in result["data"]:
                    # Use the length of the id / name and a newline to match original file
                    self.update(len(item[field]) + len("\n"))
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update(len(error["value"]) + len("\n"))
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")


class TimestampProgressBar(tqdm):
    """
    A Timestamp based progress bar. Counts timestamp ranges in milliseconds.
    This can be used to display a progress bar for tweet ids and time ranges.
    """

    def __init__(self, since_id, until_id, start_time, end_time, **kwargs):
        self.early_stop = True
        self.tweet_count = 0

        disable = False if "disable" not in kwargs else kwargs["disable"]
        kwargs["disable"] = disable

        if start_time is None and (since_id is None and until_id is None):
            start_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(days=7)
        if end_time is None and (since_id is None and until_id is None):
            end_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(seconds=30)

        if since_id and not until_id:
            until_id = _millis2snowflake(
                _date2millis(datetime.datetime.now(datetime.timezone.utc))
            )

        if until_id and not since_id:
            since_id = 1

        total = (
            _snowflake2millis(until_id) - _snowflake2millis(since_id)
            if (since_id and until_id)
            else _date2millis(end_time) - _date2millis(start_time)
        )

        kwargs["miniters"] = 1
        kwargs["total"] = total
        tweets_timeline_format = "{l_bar}{bar}| Processed {n_time}/{total_time} [{elapsed}<{remaining}, {tweet_count} tweets total {postfix}]"
        kwargs["bar_format"] = (
            tweets_timeline_format
            if "bar_format" not in kwargs
            else kwargs["bar_format"]
        )
        super().__init__(**kwargs)

    def update_with_dates(self, start_span, end_span):
        """
        Update the progress bar with a start and end time span.
        """
        try:
            if isinstance(start_span, str):
                start_span = datetime.datetime.strptime(
                    start_span, "%Y-%m-%dT%H:%M:%S.%fZ"
                )
            if isinstance(end_span, str):
                end_span = datetime.datetime.strptime(end_span, "%Y-%m-%dT%H:%M:%S.%fZ")
            n = _date2millis(end_span) - _date2millis(start_span)
            if self.n + n > self.total:
                self.n = self.total
            else:
                self.update(n)
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    def update_with_result(self, result):
        """
        Update progress bar based on snowflake ids from an API response.
        """
        try:
            newest_id = result["meta"]["newest_id"]
            oldest_id = result["meta"]["oldest_id"]
            n = _snowflake2millis(int(newest_id)) - _snowflake2millis(int(oldest_id))
            self.update(n)
            self.tweet_count += len(result["data"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    @property
    def format_dict(self):
        d = super(TimestampProgressBar, self).format_dict  # original format dict
        tweets_per_second = int(self.tweet_count / d["elapsed"] if d["elapsed"] else 0)
        n_time = humanize.naturaldelta(datetime.timedelta(seconds=int(d["n"]) // 1000))
        total_time = humanize.naturaldelta(
            datetime.timedelta(seconds=int(d["total"]) // 1000)
        )
        d.update(n_time=n_time)
        d.update(total_time=total_time)
        d.update(tweet_count=self.tweet_count)
        d.update(tweets_per_second=tweets_per_second)
        return d

    def close(self):
        if not self.early_stop:
            # Finish the bar to 100% even if the last tweet ids do not cover the full time range
            self.update(self.total - self.n)
        super().close()


def _date2millis(dt):
    return int(dt.timestamp() * 1000)


def _millis2date(ms):
    return datetime.datetime.utcfromtimestamp(ms // 1000).replace(
        microsecond=ms % 1000 * 1000
    )


def _snowflake2millis(snowflake_id):
    return (snowflake_id >> 22) + 1288834974657


def _millis2snowflake(ms):
    return (int(ms) - 1288834974657) << 22

import os
import time
import click
import logging
import requests

import datetime
import humanize
from tqdm.auto import tqdm
from functools import wraps


log = logging.getLogger("twarc")


def rate_limit(f, tries=30):
    """
    A decorator to handle rate limiting from the Twitter v2 API. If
    a rate limit error is encountered we will sleep until we can
    issue the API call again.
    """

    @wraps(f)
    def new_f(*args, **kwargs):
        errors = 0
        while True:
            resp = f(*args, **kwargs)
            if resp.status_code in [200, 201]:
                errors = 0
                return resp
            elif resp.status_code == 429:

                # Check the headers, and try to infer why we're hitting the
                # rate limit. Because the search/all endpoints also have a
                # 1r/s rate limit that isn't obvious in the headers, we need
                # to infer the reason for the rate limit. Note that this is
                # included to help debug problems with multiple concurrent
                # clients - this shouldn't be hit in normal of operation of a
                # single twarc client.
                remaining = int(resp.headers["x-rate-limit-remaining"])

                # If we have a 429 rate limit, but there are remaining calls for
                # this endpoint, we've probably hit the 1r/s limit.
                if remaining:
                    log.warning(
                        "Hit the 1 request/second rate limit, sleeping for 10 seconds. "
                        "This shouldn't happen with normal usage of twarc, and may indicate "
                        "multiple clients interacting with the Twitter API at the "
                        "same time."
                    )
                    time.sleep(10)
                    continue

                # Just a regular 15 minute window rate limit.
                else:
                    reset = int(resp.headers["x-rate-limit-reset"])
                    now = time.time()

                    # The time to sleep depends on having an accurate system time,
                    # so check to see if there's something really bad happening
                    # to warn the user.
                    target_sleep_seconds = reset - now

                    # Never sleep longer than 15 minutes, as that is the basis for
                    # all of the read time based rate limits in the Twitter API
                    seconds = min(901, max(10, (target_sleep_seconds + 10)))

                    if target_sleep_seconds >= 900:
                        # If we need to sleep for more than a rate limit period, the
                        # system clock could be wrong.
                        log.warning(
                            "Detected overlong sleep interval - is your system clock accurate? "
                            "An accurate system time is needed to calculate how long to sleep for, "
                            "and data collection might be slowed. "
                            f"The rate limit resets at {reset} and the current time is {now}."
                        )
                    elif target_sleep_seconds < 0:
                        # If we need to sleep for negative time something weird might be up.
                        log.warning(
                            "Detected negative sleep interval - is your system clock accurate? "
                            "If your system time is running fast, rate limiting may not be "
                            "effective. "
                            f"The rate limit resets at {reset} and the current time is {now}."
                        )

                    log.warning("rate limit exceeded: sleeping %s secs", seconds)
                    time.sleep(seconds)

            elif resp.status_code >= 500:
                errors += 1
                if errors > tries:
                    log.warning(f"too many errors ({tries}) from Twitter, giving up")
                    resp.raise_for_status()
                seconds = errors**2
                log.warning(
                    "caught %s from Twitter API, sleeping %s", resp.status_code, seconds
                )
                time.sleep(seconds)
            else:
                log.error("Unexpected HTTP response: %s", resp)
                resp.raise_for_status()

    return new_f


def catch_request_exceptions(f, tries=30):
    """
    A decorator to handle all request exceptions. This decorator will catch
    *any* request level error, reconnect and try again. It does not handle
    HTTP protocol level errors (404, 500) etc.

    It will try up to tries times consecutively before giving up. A successful
    call to f will result in the try counter being reset to 0.
    """

    # pyOpenSSL has been known to throw these connection errors that need to be
    # caught separately: https://github.com/edsu/twarc/issues/72

    try:
        import OpenSSL

        ConnectionError = OpenSSL.SSL.SysCallError
    except:
        ConnectionError = requests.exceptions.ConnectionError

    @wraps(f)
    def new_f(self, *args, **kwargs):
        errors = 0
        while errors < tries:
            try:
                resp = f(self, *args, **kwargs)
                errors = 0
                return resp
            except (requests.exceptions.RequestException, ConnectionError) as e:

                # don't catch any HTTP errors since those are handled separately
                if isinstance(e, requests.exceptions.HTTPError):
                    raise e

                errors += 1
                log.warning("caught requests exception: %s", e)
                if errors > tries:
                    log.error(f"giving up, too many request exceptions: {tries}")
                    raise e
                seconds = errors**2
                log.info("sleeping %s", seconds)
                time.sleep(seconds)
                self.connect()

    return new_f


def interruptible_sleep(t, event=None):
    """
    Sleeps for a specified duration, optionally stopping early for event.

    Returns True if interrupted
    """
    log.info("sleeping %s", t)

    if event is None:
        time.sleep(t)
        return False
    else:
        return not event.wait(t)


class cli_api_error:
    """
    A decorator to catch HTTP errors for the command line.
    """

    def __init__(self, f):
        self.f = f
        # this is needed for click help docs to work properly
        self.__doc__ = f.__doc__

    def __call__(self, *args, **kwargs):
        try:
            return self.f(*args, **kwargs)
        except requests.exceptions.HTTPError as e:
            try:
                result = e.response.json()
                if "errors" in result:
                    for error in result["errors"]:
                        msg = error.get("message", "Unknown error")
                elif "title" in result:
                    msg = result["title"]
                else:
                    msg = "Unknown error"
            except ValueError:
                msg = f"Unable to parse {e.response.status_code} error as JSON: {e.response.text}"
        except InvalidAuthType as e:
            msg = "This command requires application authentication, try passing --app-auth"
        except ValueError as e:
            msg = str(e)
        click.echo(
            click.style("⚡ ", fg="yellow") + click.style(msg, fg="red"), err=True
        )


def requires_app_auth(f):
    """
    Ensure that application authentication is set for calls that only work in that mode.
    """

    @wraps(f)
    def new_f(self, *args, **kwargs):
        if self.auth_type != "application":
            raise InvalidAuthType(
                "This endpoint only works with application authentication"
            )

        else:
            return f(self, *args, **kwargs)

    return new_f


class InvalidAuthType(Exception):
    """
    Raised when the endpoint called is not supported by the current auth type.
    """


class FileLineProgressBar(tqdm):
    """
    A progress bar based on input file line count. Counts an input file by lines.
    This tries to read the entire file and count newlines in a robust way.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} lines of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"

        # Warn for large (> 1 GB) input files:
        if (os.stat(infile.name).st_size / (1024 * 1024 * 1024)) > 1:
            click.echo(
                click.style(
                    f"Input File Size is {os.stat(infile.name).st_size / (1024*1024):.2f} MB, it may take a while to process. CTRL+C to stop.",
                    fg="yellow",
                    bold=True,
                ),
                err=True,
            )

        with open(infile.name, "r", encoding="utf-8", errors="ignore") as f:
            total_lines = sum(1 for _ in f)

        kwargs["total"] = total_lines if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        and drop in compatibility with FileSizeProgressBar otherwise use tqdm's update().
        """

        try:
            if "data" in result:
                for item in result["data"]:
                    self.update()
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update()
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")


class FileSizeProgressBar(tqdm):
    """
    An input file size based progress bar. Counts an input file in bytes.
    This will also dig into the responses and add up the outputs to match the file size.
    Overrides `disable` parameter if file is a pipe.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["unit"] = "B"
        kwargs["unit_scale"] = True
        kwargs["unit_divisor"] = 1024
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
        kwargs["total"] = os.stat(infile.name).st_size if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        otherwise use twdm's own update() method.
        """
        try:
            if "data" in result:
                for item in result["data"]:
                    # Use the length of the id / name and a newline to match original file
                    self.update(len(item[field]) + len("\n"))
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update(len(error["value"]) + len("\n"))
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")


class TimestampProgressBar(tqdm):
    """
    A Timestamp based progress bar. Counts timestamp ranges in milliseconds.
    This can be used to display a progress bar for tweet ids and time ranges.
    """

    def __init__(self, since_id, until_id, start_time, end_time, **kwargs):
        self.early_stop = True
        self.tweet_count = 0

        disable = False if "disable" not in kwargs else kwargs["disable"]
        kwargs["disable"] = disable

        if start_time is None and (since_id is None and until_id is None):
            start_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(days=7)
        if end_time is None and (since_id is None and until_id is None):
            end_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(seconds=30)

        if since_id and not until_id:
            until_id = _millis2snowflake(
                _date2millis(datetime.datetime.now(datetime.timezone.utc))
            )

        if until_id and not since_id:
            since_id = 1

        total = (
            _snowflake2millis(until_id) - _snowflake2millis(since_id)
            if (since_id and until_id)
            else _date2millis(end_time) - _date2millis(start_time)
        )

        kwargs["miniters"] = 1
        kwargs["total"] = total
        tweets_timeline_format = "{l_bar}{bar}| Processed {n_time}/{total_time} [{elapsed}<{remaining}, {tweet_count} tweets total {postfix}]"
        kwargs["bar_format"] = (
            tweets_timeline_format
            if "bar_format" not in kwargs
            else kwargs["bar_format"]
        )
        super().__init__(**kwargs)

    def update_with_dates(self, start_span, end_span):
        """
        Update the progress bar with a start and end time span.
        """
        try:
            if isinstance(start_span, str):
                start_span = datetime.datetime.strptime(
                    start_span, "%Y-%m-%dT%H:%M:%S.%fZ"
                )
            if isinstance(end_span, str):
                end_span = datetime.datetime.strptime(end_span, "%Y-%m-%dT%H:%M:%S.%fZ")
            n = _date2millis(end_span) - _date2millis(start_span)
            if self.n + n > self.total:
                self.n = self.total
            else:
                self.update(n)
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    def update_with_result(self, result):
        """
        Update progress bar based on snowflake ids from an API response.
        """
        try:
            newest_id = result["meta"]["newest_id"]
            oldest_id = result["meta"]["oldest_id"]
            n = _snowflake2millis(int(newest_id)) - _snowflake2millis(int(oldest_id))
            self.update(n)
            self.tweet_count += len(result["data"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    @property
    def format_dict(self):
        d = super(TimestampProgressBar, self).format_dict  # original format dict
        tweets_per_second = int(self.tweet_count / d["elapsed"] if d["elapsed"] else 0)
        n_time = humanize.naturaldelta(datetime.timedelta(seconds=int(d["n"]) // 1000))
        total_time = humanize.naturaldelta(
            datetime.timedelta(seconds=int(d["total"]) // 1000)
        )
        d.update(n_time=n_time)
        d.update(total_time=total_time)
        d.update(tweet_count=self.tweet_count)
        d.update(tweets_per_second=tweets_per_second)
        return d

    def close(self):
        if not self.early_stop:
            # Finish the bar to 100% even if the last tweet ids do not cover the full time range
            self.update(self.total - self.n)
        super().close()


def _date2millis(dt):
    return int(dt.timestamp() * 1000)


def _millis2date(ms):
    return datetime.datetime.utcfromtimestamp(ms // 1000).replace(
        microsecond=ms % 1000 * 1000
    )


def _snowflake2millis(snowflake_id):
    return (snowflake_id >> 22) + 1288834974657


def _millis2snowflake(ms):
    return (int(ms) - 1288834974657) << 22
A decorator to handle rate limiting from the Twitter v2 API. If     a rate limit error is encountered we will sleep until we can     issue the API call again. # Check the headers, and try to infer why we're hitting the # rate limit. Because the search/all endpoints also have a # 1r/s rate limit that isn't obvious in the headers, we need # to infer the reason for the rate limit. Note that this is # included to help debug problems with multiple concurrent # clients - this shouldn't be hit in normal of operation of a # single twarc client. # If we have a 429 rate limit, but there are remaining calls for # this endpoint, we've probably hit the 1r/s limit. # Just a regular 15 minute window rate limit. # The time to sleep depends on having an accurate system time, # so check to see if there's something really bad happening # to warn the user. # Never sleep longer than 15 minutes, as that is the basis for # all of the read time based rate limits in the Twitter API # If we need to sleep for more than a rate limit period, the # system clock could be wrong. # If we need to sleep for negative time something weird might be up. A decorator to handle all request exceptions. This decorator will catch     *any* request level error, reconnect and try again. It does not handle     HTTP protocol level errors (404, 500) etc.      It will try up to tries times consecutively before giving up. A successful     call to f will result in the try counter being reset to 0. # pyOpenSSL has been known to throw these connection errors that need to be # caught separately: https://github.com/edsu/twarc/issues/72 # don't catch any HTTP errors since those are handled separately Sleeps for a specified duration, optionally stopping early for event.      Returns True if interrupted A decorator to catch HTTP errors for the command line. # this is needed for click help docs to work properly Ensure that application authentication is set for calls that only work in that mode. Raised when the endpoint called is not supported by the current auth type. A progress bar based on input file line count. Counts an input file by lines.     This tries to read the entire file and count newlines in a robust way. # Warn for large (> 1 GB) input files: Update the progress bar appropriately, with a full API response. For convenience,         and drop in compatibility with FileSizeProgressBar otherwise use tqdm's update(). # Account for deleted data # Errors have very inconsistent format, missing fields for different types of errors... # todo: hide or show this? # self.set_description( #    "Errors encountered, results may be incomplete" # ) # print(error["value"], error["resource_type"], error["parameter"]) An input file size based progress bar. Counts an input file in bytes.     This will also dig into the responses and add up the outputs to match the file size.     Overrides `disable` parameter if file is a pipe. Update the progress bar appropriately, with a full API response. For convenience,         otherwise use twdm's own update() method. # Use the length of the id / name and a newline to match original file # Account for deleted data # Errors have very inconsistent format, missing fields for different types of errors... # todo: hide or show this? # self.set_description( #    "Errors encountered, results may be incomplete" # ) # print(error["value"], error["resource_type"], error["parameter"]) A Timestamp based progress bar. Counts timestamp ranges in milliseconds.     This can be used to display a progress bar for tweet ids and time ranges. Update the progress bar with a start and end time span. Update progress bar based on snowflake ids from an API response. # original format dict # Finish the bar to 100% even if the last tweet ids do not cover the full time range