#!/usr/bin/python
import numpy as np
import os
import json
import argparse

try:
    import bnnbench
except:
    import sys
    sys.path.append(os.path.expandvars('$BNNBENCHPATH'))

import bnnbench.utils.data_utils
from bnnbench.models import model_types
from bnnbench.config import globalConfig
from bnnbench import _log as bnnbench_logger
from bnnbench.utils.attrDict import AttrDict
import bnnbench.utils.universal_utils as utils

config_top_level_keys = utils.config_top_level_keys

# ----------------------------------------------------------------------------------------------------------------------
# --------------------------------------Set up default experiment parameters--------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------
config = AttrDict()

config.OBJECTIVE_FUNC = None
config.SPLITS = None

config.model_params = {}
config.exp_params = globalConfig
config.mtype = model_types.mlp


# ----------------------------------------------------------------------------------------------------------------------
# ---------------------------------------------Check command line args--------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------

def handle_cli():
    print("Handling command line arguments.")
    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument('--model', type=str, default='mlp',
                        help='Case-insensitive string indicating the type of model to be used for this experiment. '
                             'Valid options are: ["mlp", "mcdropout", "mcbatchnorm", "ensemble", "dngo"]')
    parser.add_argument('--config', type=str, default=None,
                        help='Filename of JSON file containing experiment configuration. If not provided, default '
                             'configurations are used.')

    for argument, helptext in globalConfig.cli_arguments.items():
        argname = '--' + argument
        defaultval = globalConfig.defaults[argument]
        if type(defaultval) is bool:
            action = "store_false" if defaultval else "store_true"
        elif isinstance(defaultval, str):
            action = "store"

        parser.add_argument(argname, default=None, action=action, help=helptext)

    parser.add_argument('--plotdata', action='store_true', default=False, required=False,
                        help='When given, generates a plot of the training/test data. Only supported for 1D '
                             'datasets.')
    parser.add_argument('--summarize', action='store_true', default=False, required=False,
                        help='When given, generates a summary of the network generated by the model using '
                             'torchsummary.')

    args = parser.parse_args()

    config.plotdata = args.plotdata
    config.summarize = args.summarize

    mtype = str.lower(args.model)
    if mtype not in model_types:
        raise RuntimeError("Unknown model type %s specified." % mtype)
    else:
        config.mtype = model_types[mtype]

    default_model_params = model_types[mtype]._default_model_params._asdict()

    if args.config is not None:
        bnnbench_logger.info("--config flag detected.")
        config_file_path = utils.standard_pathcheck(args.config)
        with open(config_file_path, 'r') as fp:
            json_config = json.load(fp)

        if config_top_level_keys.obj_func in json_config:
            bnnbench_logger.debug("Attempting to fetch objective function %s" %
                               json_config[config_top_level_keys.obj_func])
            if isinstance(json_config[config_top_level_keys.obj_func], dict):
                utils.parse_objective(config=json_config[config_top_level_keys.obj_func], out=config)
            else:
                raise RuntimeError("This script is intended for use with datasets only and thus requires the dataset "
                                   "to be specified as a dict in the JSON config file.")
            bnnbench_logger.info("Fetched objective.")

        if config_top_level_keys.mparams in json_config:
            json_model_params = json_config[config_top_level_keys.mparams]
            bnnbench_logger.info("Using model parameters provided by config file.")
            for key, val in default_model_params.items():
                config.model_params[key] = val if json_model_params.get(key, None) is None else \
                    json_model_params[key]
            bnnbench_logger.info("Final model parameters: %s" % config.model_params)

        if config_top_level_keys.eparams in json_config:
            bnnbench_logger.info("Using experiment parameters provided by config file.")
            json_exp_params = json_config[config_top_level_keys.eparams]

            for key in globalConfig.cli_arguments:
                # Only handle those settings that can be modified using the CLI or JSON config file.
                # Priorities: 1. CLI, 2. Config file, 3. Defaults
                clival = getattr(args, key)
                jsonval = json_exp_params.get(key, None)
                if clival not in [None, '']:
                    setattr(config.exp_params, key, clival)
                elif jsonval not in [None, '']:
                    setattr(config.exp_params, key, jsonval)

            # TODO: Fix. Use the params property to display this properly.
            bnnbench_logger.info("Final experiment parameters: %s" % config.exp_params)

    else:
        bnnbench_logger.info("No config file detected, using default parameters.")
        config.model_params = default_model_params

    bnnbench_logger.info("Finished reading command line arguments.")


def perform_experiment():
    # -----------------------------------------------Generate data------------------------------------------------------

    if isinstance(config.OBJECTIVE_FUNC, AttrDict):
        data_splits = bnnbench.utils.data_utils.data_generator(config.OBJECTIVE_FUNC)
    else:
        raise RuntimeError("This script does not support the old-style interface for specifying 1D toy functions.")

    bnnbench_logger.debug("Finished generating dataset splits.")
    for idx, (Xtrain, ytrain, Xtest, ytest) in enumerate(data_splits):

        bnnbench_logger.info("Now conducting experiment on test split %d." % idx)

        Xtrain = Xtrain[:, None] if len(Xtrain.shape) == 1 else Xtrain
        ytrain = ytrain[:, None] if len(ytrain.shape) == 1 else ytrain
        Xtest = Xtest[:, None] if len(Xtest.shape) == 1 else Xtest
        ytest = ytest[:, None] if len(ytest.shape) == 1 else ytest
        bnnbench_logger.debug("Loaded split with training X, y of shapes %s, %s and test X, y of shapes %s, %s" %
                           (Xtrain.shape, ytrain.shape, Xtest.shape, ytest.shape))

        # ---------------------------------------------Generate model---------------------------------------------------

        model = config.mtype(model_params=config.model_params)
        # if config.exp_params['tbdir'] is None:
        if config.exp_params.tbdir in [None, '']:
            config.exp_params.tbdir = model.modeldir
            bnnbench_logger.info("Tensorboard directory set to: %s" % (config.exp_params.tbdir))
        globalConfig.params = config.exp_params

        rng: np.random.RandomState = model.rng
        mean_only = True if config.mtype is model_types.mlp else False

        bnnbench_logger.info("Saving new model to: %s" % config.model_params["model_path"])

        # -----------------------------------------------Let it roll----------------------------------------------------

        model.preprocess_training_data(Xtrain, ytrain)
        model.train_network()  # Don't save interim progress plots

        predicted_y = model.predict(Xtest)
        savedir = utils.ensure_path_exists(model.modeldir)

        if mean_only:
            # out = np.zeros((Xtest.shape[0], Xtest.shape[1] + 1))
            out = np.concatenate((Xtest, predicted_y), axis=1)
        else:
            # Assume the model predicted means and variances, returned as a tuple
            # Treat both elements of the tuple as individual numpy arrays
            out = np.concatenate((Xtest, predicted_y[0], predicted_y[1]), axis=1)

        # ------------------------------------If needed, generate visualizations----------------------------------------

        bnnbench_logger.info("Saving model performance results in %s " % savedir)

        if config.plotdata:
            from bnnbench.utils.universal_utils import simple_plotter
            import matplotlib.pyplot as plt
            traindata = np.concatenate((Xtrain, ytrain), axis=1)
            testdata = np.concatenate((Xtest, ytest), axis=1)
            bnnbench_logger.info("Displaying:\nTraining data of shape %s \nTest data of shape %s\n"
                              "Prediction data of shape %s" % (traindata.shape, testdata.shape, out.shape))
            _ = simple_plotter(
                pred=out,
                train=traindata,
                test=testdata,
                plot_variances=not mean_only
            )
            plt.show()

        # -----------------------------------------------Save results---------------------------------------------------

        np.save(file=os.path.join(savedir, 'trainset'), arr=np.concatenate((Xtrain, ytrain), axis=1), allow_pickle=True)
        np.save(file=os.path.join(savedir, 'testset'), arr=np.concatenate((Xtest, ytest), axis=1), allow_pickle=True)
        np.save(file=os.path.join(savedir, 'test_predictions'), arr=out, allow_pickle=True)

        utils.make_model_params_json_compatible(config.model_params)

        # TODO: Remove this function
        # utils.make_exp_params_json_compatible(config.exp_params)
        jdict = {
            config_top_level_keys.obj_func: str(config.OBJECTIVE_FUNC),
            config_top_level_keys.mparams: config.model_params,
            config_top_level_keys.eparams: config.exp_params.to_cli()
        }

        with open(os.path.join(savedir, 'config.json'), 'w') as fp:
            try:
                json.dump(jdict, fp, indent=4)
            except TypeError as e:
                print("Could not write configuration file for config:\n%s" % jdict)

        print("Finished experiment.")

        if config.summarize:
            model.network.to('cuda')
            from torchsummary import summary
            summary(model.network, input_size=(model.batch_size, model.input_dims))

        del model
        bnnbench_logger.info("Finished conducting experiment on test split %d." % idx)
    bnnbench_logger.info("Finished conducting all experiments on the given dataset.")


if __name__ == '__main__':
    handle_cli()
    perform_experiment()

#!/usr/bin/python
import numpy as np
import os
import json
import argparse

try:
    import bnnbench
except:
    import sys
    sys.path.append(os.path.expandvars('$BNNBENCHPATH'))

import bnnbench.utils.data_utils
from bnnbench.models import model_types
from bnnbench.config import globalConfig
from bnnbench import _log as bnnbench_logger
from bnnbench.utils.attrDict import AttrDict
import bnnbench.utils.universal_utils as utils

config_top_level_keys = utils.config_top_level_keys

# ----------------------------------------------------------------------------------------------------------------------
# --------------------------------------Set up default experiment parameters--------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------
config = AttrDict()

config.OBJECTIVE_FUNC = None
config.SPLITS = None

config.model_params = {}
config.exp_params = globalConfig
config.mtype = model_types.mlp


# ----------------------------------------------------------------------------------------------------------------------
# ---------------------------------------------Check command line args--------------------------------------------------
# ----------------------------------------------------------------------------------------------------------------------

def handle_cli():
    print("Handling command line arguments.")
    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument('--model', type=str, default='mlp',
                        help='Case-insensitive string indicating the type of model to be used for this experiment. '
                             'Valid options are: ["mlp", "mcdropout", "mcbatchnorm", "ensemble", "dngo"]')
    parser.add_argument('--config', type=str, default=None,
                        help='Filename of JSON file containing experiment configuration. If not provided, default '
                             'configurations are used.')

    for argument, helptext in globalConfig.cli_arguments.items():
        argname = '--' + argument
        defaultval = globalConfig.defaults[argument]
        if type(defaultval) is bool:
            action = "store_false" if defaultval else "store_true"
        elif isinstance(defaultval, str):
            action = "store"

        parser.add_argument(argname, default=None, action=action, help=helptext)

    parser.add_argument('--plotdata', action='store_true', default=False, required=False,
                        help='When given, generates a plot of the training/test data. Only supported for 1D '
                             'datasets.')
    parser.add_argument('--summarize', action='store_true', default=False, required=False,
                        help='When given, generates a summary of the network generated by the model using '
                             'torchsummary.')

    args = parser.parse_args()

    config.plotdata = args.plotdata
    config.summarize = args.summarize

    mtype = str.lower(args.model)
    if mtype not in model_types:
        raise RuntimeError("Unknown model type %s specified." % mtype)
    else:
        config.mtype = model_types[mtype]

    default_model_params = model_types[mtype]._default_model_params._asdict()

    if args.config is not None:
        bnnbench_logger.info("--config flag detected.")
        config_file_path = utils.standard_pathcheck(args.config)
        with open(config_file_path, 'r') as fp:
            json_config = json.load(fp)

        if config_top_level_keys.obj_func in json_config:
            bnnbench_logger.debug("Attempting to fetch objective function %s" %
                               json_config[config_top_level_keys.obj_func])
            if isinstance(json_config[config_top_level_keys.obj_func], dict):
                utils.parse_objective(config=json_config[config_top_level_keys.obj_func], out=config)
            else:
                raise RuntimeError("This script is intended for use with datasets only and thus requires the dataset "
                                   "to be specified as a dict in the JSON config file.")
            bnnbench_logger.info("Fetched objective.")

        if config_top_level_keys.mparams in json_config:
            json_model_params = json_config[config_top_level_keys.mparams]
            bnnbench_logger.info("Using model parameters provided by config file.")
            for key, val in default_model_params.items():
                config.model_params[key] = val if json_model_params.get(key, None) is None else \
                    json_model_params[key]
            bnnbench_logger.info("Final model parameters: %s" % config.model_params)

        if config_top_level_keys.eparams in json_config:
            bnnbench_logger.info("Using experiment parameters provided by config file.")
            json_exp_params = json_config[config_top_level_keys.eparams]

            for key in globalConfig.cli_arguments:
                # Only handle those settings that can be modified using the CLI or JSON config file.
                # Priorities: 1. CLI, 2. Config file, 3. Defaults
                clival = getattr(args, key)
                jsonval = json_exp_params.get(key, None)
                if clival not in [None, '']:
                    setattr(config.exp_params, key, clival)
                elif jsonval not in [None, '']:
                    setattr(config.exp_params, key, jsonval)

            # TODO: Fix. Use the params property to display this properly.
            bnnbench_logger.info("Final experiment parameters: %s" % config.exp_params)

    else:
        bnnbench_logger.info("No config file detected, using default parameters.")
        config.model_params = default_model_params

    bnnbench_logger.info("Finished reading command line arguments.")


def perform_experiment():
    # -----------------------------------------------Generate data------------------------------------------------------

    if isinstance(config.OBJECTIVE_FUNC, AttrDict):
        data_splits = bnnbench.utils.data_utils.data_generator(config.OBJECTIVE_FUNC)
    else:
        raise RuntimeError("This script does not support the old-style interface for specifying 1D toy functions.")

    bnnbench_logger.debug("Finished generating dataset splits.")
    for idx, (Xtrain, ytrain, Xtest, ytest) in enumerate(data_splits):

        bnnbench_logger.info("Now conducting experiment on test split %d." % idx)

        Xtrain = Xtrain[:, None] if len(Xtrain.shape) == 1 else Xtrain
        ytrain = ytrain[:, None] if len(ytrain.shape) == 1 else ytrain
        Xtest = Xtest[:, None] if len(Xtest.shape) == 1 else Xtest
        ytest = ytest[:, None] if len(ytest.shape) == 1 else ytest
        bnnbench_logger.debug("Loaded split with training X, y of shapes %s, %s and test X, y of shapes %s, %s" %
                           (Xtrain.shape, ytrain.shape, Xtest.shape, ytest.shape))

        # ---------------------------------------------Generate model---------------------------------------------------

        model = config.mtype(model_params=config.model_params)
        # if config.exp_params['tbdir'] is None:
        if config.exp_params.tbdir in [None, '']:
            config.exp_params.tbdir = model.modeldir
            bnnbench_logger.info("Tensorboard directory set to: %s" % (config.exp_params.tbdir))
        globalConfig.params = config.exp_params

        rng: np.random.RandomState = model.rng
        mean_only = True if config.mtype is model_types.mlp else False

        bnnbench_logger.info("Saving new model to: %s" % config.model_params["model_path"])

        # -----------------------------------------------Let it roll----------------------------------------------------

        model.preprocess_training_data(Xtrain, ytrain)
        model.train_network()  # Don't save interim progress plots

        predicted_y = model.predict(Xtest)
        savedir = utils.ensure_path_exists(model.modeldir)

        if mean_only:
            # out = np.zeros((Xtest.shape[0], Xtest.shape[1] + 1))
            out = np.concatenate((Xtest, predicted_y), axis=1)
        else:
            # Assume the model predicted means and variances, returned as a tuple
            # Treat both elements of the tuple as individual numpy arrays
            out = np.concatenate((Xtest, predicted_y[0], predicted_y[1]), axis=1)

        # ------------------------------------If needed, generate visualizations----------------------------------------

        bnnbench_logger.info("Saving model performance results in %s " % savedir)

        if config.plotdata:
            from bnnbench.utils.universal_utils import simple_plotter
            import matplotlib.pyplot as plt
            traindata = np.concatenate((Xtrain, ytrain), axis=1)
            testdata = np.concatenate((Xtest, ytest), axis=1)
            bnnbench_logger.info("Displaying:\nTraining data of shape %s \nTest data of shape %s\n"
                              "Prediction data of shape %s" % (traindata.shape, testdata.shape, out.shape))
            _ = simple_plotter(
                pred=out,
                train=traindata,
                test=testdata,
                plot_variances=not mean_only
            )
            plt.show()

        # -----------------------------------------------Save results---------------------------------------------------

        np.save(file=os.path.join(savedir, 'trainset'), arr=np.concatenate((Xtrain, ytrain), axis=1), allow_pickle=True)
        np.save(file=os.path.join(savedir, 'testset'), arr=np.concatenate((Xtest, ytest), axis=1), allow_pickle=True)
        np.save(file=os.path.join(savedir, 'test_predictions'), arr=out, allow_pickle=True)

        utils.make_model_params_json_compatible(config.model_params)

        # TODO: Remove this function
        # utils.make_exp_params_json_compatible(config.exp_params)
        jdict = {
            config_top_level_keys.obj_func: str(config.OBJECTIVE_FUNC),
            config_top_level_keys.mparams: config.model_params,
            config_top_level_keys.eparams: config.exp_params.to_cli()
        }

        with open(os.path.join(savedir, 'config.json'), 'w') as fp:
            try:
                json.dump(jdict, fp, indent=4)
            except TypeError as e:
                print("Could not write configuration file for config:\n%s" % jdict)

        print("Finished experiment.")

        if config.summarize:
            model.network.to('cuda')
            from torchsummary import summary
            summary(model.network, input_size=(model.batch_size, model.input_dims))

        del model
        bnnbench_logger.info("Finished conducting experiment on test split %d." % idx)
    bnnbench_logger.info("Finished conducting all experiments on the given dataset.")


if __name__ == '__main__':
    handle_cli()
    perform_experiment()

#!/usr/bin/python # ---------------------------------------------------------------------------------------------------------------------- # --------------------------------------Set up default experiment parameters-------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- # ---------------------------------------------Check command line args-------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- # Only handle those settings that can be modified using the CLI or JSON config file. # Priorities: 1. CLI, 2. Config file, 3. Defaults # TODO: Fix. Use the params property to display this properly. # -----------------------------------------------Generate data------------------------------------------------------ # ---------------------------------------------Generate model--------------------------------------------------- # if config.exp_params['tbdir'] is None: # -----------------------------------------------Let it roll---------------------------------------------------- # Don't save interim progress plots # out = np.zeros((Xtest.shape[0], Xtest.shape[1] + 1)) # Assume the model predicted means and variances, returned as a tuple # Treat both elements of the tuple as individual numpy arrays # ------------------------------------If needed, generate visualizations---------------------------------------- # -----------------------------------------------Save results--------------------------------------------------- # TODO: Remove this function # utils.make_exp_params_json_compatible(config.exp_params)