import os
import requests
import glob
import logging
import boto3
from botocore.exceptions import ClientError
from tqdm import tqdm
import datetime
from sub_units.utils import Region
from sub_units import post_analysis


def upload_file(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket

    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """

    # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name

    # Upload the file
    s3_client = boto3.client('s3')
    try:
        response = s3_client.upload_file(file_name, bucket, object_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True


#####
# Step 1b: Update load_data (this happens as soon as you import modules that use load_data)
#####

import datetime
import covid_moving_window as covid

#####
# Step 2: Run Update
#####

# get today's date
yesterdays_date_str = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
print(f'Yesterday: {yesterdays_date_str}')

covid.n_bootstraps = None
covid.n_likelihood_samples = None
covid.moving_window_size = 21  # three weeks
covid.override_max_date_str = yesterdays_date_str
covid.opt_force_calc = False
covid.opt_force_plot = False
covid.opt_plot = True
covid.opt_truncate = False
covid.opt_report = True
covid.opt_simplified = True  # set to True to just do statsmodels as a simplified daily service
covid.override_run_states = None

region_plot_subfolders = covid.run_everything()

print('region_plot_subfolders:')
print(region_plot_subfolders)

#####
# Step 3: Upload Figures to AWS
#####

from sub_units.utils import Region

date_str = yesterdays_date_str
region_plot_subfolders = {
    Region.US_counties: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_US_counties_region_statsmodels',
    Region.countries: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_countries_region_statsmodels',
    Region.US_states: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_US_states_region_statsmodels',
    # Region.provinces: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_provinces_region_statsmodels'
}

# Or... 
# run in bash...
# TODO: get this part working using these fast CLIs rather than boto3 as below
# git
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_states_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_counties_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_provinces_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/

for region, plot_subfolder in region_plot_subfolders.items():
    hyperparamater_str = os.path.basename(os.path.normpath(plot_subfolder))
    print(hyperparamater_str)

    first_level_files = glob.glob(plot_subfolder + '/*.*', recursive=True)
    second_level_files = glob.glob(plot_subfolder + '/*/*.*', recursive=True)
    files = first_level_files + second_level_files

    # This is really inefficient -- takes 20-30 minutes! Mainly issue is that 
    #   `aws s3 cp --recursive ...` uses parallel threads, and this is a serial upload
    for file in tqdm(list(files)):
        relative_filename = '/'.join(file.split('/')[1:])
        print(f'Uploading {relative_filename}...')
        upload_file(file, 'covid-figures', object_name=relative_filename)


######
# Step 5: Push to Github
######

# TODO: Update static_figures with most recent version
# TODO: Update README.md to link to new CSV files

# Do this by hand
# TODO: Figure out how to do this automatically instead of by hand
print('Now add, commit, and push to Github!')

import os
import requests
import glob
import logging
import boto3
from botocore.exceptions import ClientError
from tqdm import tqdm
import datetime
from sub_units.utils import Region
from sub_units import post_analysis


def upload_file(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket

    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """

    # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name

    # Upload the file
    s3_client = boto3.client('s3')
    try:
        response = s3_client.upload_file(file_name, bucket, object_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True


#####
# Step 1b: Update load_data (this happens as soon as you import modules that use load_data)
#####

import datetime
import covid_moving_window as covid

#####
# Step 2: Run Update
#####

# get today's date
yesterdays_date_str = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
print(f'Yesterday: {yesterdays_date_str}')

covid.n_bootstraps = None
covid.n_likelihood_samples = None
covid.moving_window_size = 21  # three weeks
covid.override_max_date_str = yesterdays_date_str
covid.opt_force_calc = False
covid.opt_force_plot = False
covid.opt_plot = True
covid.opt_truncate = False
covid.opt_report = True
covid.opt_simplified = True  # set to True to just do statsmodels as a simplified daily service
covid.override_run_states = None

region_plot_subfolders = covid.run_everything()

print('region_plot_subfolders:')
print(region_plot_subfolders)

#####
# Step 3: Upload Figures to AWS
#####

from sub_units.utils import Region

date_str = yesterdays_date_str
region_plot_subfolders = {
    Region.US_counties: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_US_counties_region_statsmodels',
    Region.countries: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_countries_region_statsmodels',
    Region.US_states: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_US_states_region_statsmodels',
    # Region.provinces: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_provinces_region_statsmodels'
}

# Or... 
# run in bash...
# TODO: get this part working using these fast CLIs rather than boto3 as below
# git
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_states_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_counties_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/
# HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_provinces_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/

for region, plot_subfolder in region_plot_subfolders.items():
    hyperparamater_str = os.path.basename(os.path.normpath(plot_subfolder))
    print(hyperparamater_str)

    first_level_files = glob.glob(plot_subfolder + '/*.*', recursive=True)
    second_level_files = glob.glob(plot_subfolder + '/*/*.*', recursive=True)
    files = first_level_files + second_level_files

    # This is really inefficient -- takes 20-30 minutes! Mainly issue is that 
    #   `aws s3 cp --recursive ...` uses parallel threads, and this is a serial upload
    for file in tqdm(list(files)):
        relative_filename = '/'.join(file.split('/')[1:])
        print(f'Uploading {relative_filename}...')
        upload_file(file, 'covid-figures', object_name=relative_filename)


######
# Step 5: Push to Github
######

# TODO: Update static_figures with most recent version
# TODO: Update README.md to link to new CSV files

# Do this by hand
# TODO: Figure out how to do this automatically instead of by hand
print('Now add, commit, and push to Github!')

Upload a file to an S3 bucket      :param file_name: File to upload     :param bucket: Bucket to upload to     :param object_name: S3 object name. If not specified then file_name is used     :return: True if file was uploaded, else False # If S3 object_name was not specified, use file_name # Upload the file ##### # Step 1b: Update load_data (this happens as soon as you import modules that use load_data) ##### ##### # Step 2: Run Update ##### # get today's date # three weeks # set to True to just do statsmodels as a simplified daily service ##### # Step 3: Upload Figures to AWS ##### # Region.provinces: f'state_plots/{date_str}_date_smoothed_moving_window_21_days_provinces_region_statsmodels' # Or... # run in bash... # TODO: get this part working using these fast CLIs rather than boto3 as below # git # HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_states_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/ # HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_US_counties_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/ # HYP_STR=2020_07_05_date_smoothed_moving_window_21_days_provinces_region_statsmodels; aws s3 cp --recursive state_plots/$HYP_STR s3://covid-figures/$HYP_STR/ # This is really inefficient -- takes 20-30 minutes! Mainly issue is that #   `aws s3 cp --recursive ...` uses parallel threads, and this is a serial upload ###### # Step 5: Push to Github ###### # TODO: Update static_figures with most recent version # TODO: Update README.md to link to new CSV files # Do this by hand # TODO: Figure out how to do this automatically instead of by hand