#!/usr/bin/env python3
"""
Prepare the ESM dataset and EUDEM for EnerMaps.

Note that the 25-m (EU-DEM) and 2.5-m (ESM) resolution files must be downloaded from Copernicus (requires log-in)
and extracted in the data/21 and data/35 directory.
This script expects that the original zip file from Copernicus is extracted
in multiple zip files, which will be then extracted and tiled.


@author: giuseppeperonato
"""

import glob
import json
import logging
import os
import pathlib
import shutil
import sys
import uuid

import geopandas as gpd
import pandas as pd
import utilities
from shapely.geometry import box

N_FILES = {21: 27, 35: 279}
COMPRESS = {21: "DEFLATE", 35: "DEFLATE"}
RESOLUTION = {21: 800, 35: 20000}
ISRASTER = True
logging.basicConfig(level=logging.INFO)

DB_URL = utilities.DB_URL
LEGENDS = {
    35: {
        "vis_id": uuid.uuid4(),
        "legend": {
            "name": "Land use",
            "type": "custom",
            "symbology": [
                {
                    "red": 112,
                    "green": 162,
                    "blue": 255,
                    "opacity": 1.0,
                    "value": "1",
                    "label": "Water",
                },
                {
                    "red": 102,
                    "green": 102,
                    "blue": 102,
                    "opacity": 1.0,
                    "value": "2",
                    "label": "Railways",
                },
                {
                    "red": 242,
                    "green": 242,
                    "blue": 242,
                    "opacity": 1.0,
                    "value": "10",
                    "label": "Non-built area - Open Space",
                },
                {
                    "red": 221,
                    "green": 230,
                    "blue": 207,
                    "opacity": 1.0,
                    "value": "20",
                    "label": "Non-built area - Green ndvix",
                },
                {
                    "red": 225,
                    "green": 225,
                    "blue": 225,
                    "opacity": 1.0,
                    "value": "30",
                    "label": "Built area - Open space",
                },
                {
                    "red": 181,
                    "green": 204,
                    "blue": 142,
                    "opacity": 1.0,
                    "value": "40",
                    "label": "Built area - Green ndvix",
                },
                {
                    "red": 200,
                    "green": 230,
                    "blue": 161,
                    "opacity": 1.0,
                    "value": "41",
                    "label": "Built area - Green Urban Atlas",
                },
                {
                    "red": 128,
                    "green": 125,
                    "blue": 121,
                    "opacity": 1.0,
                    "value": "50",
                    "label": "Built area - Built up",
                },
            ],
        },
    }
}

RECORD_METADATA = {
    "variable": {21: "Elevation", 35: "Land use"},
    "unit": {21: "m", 35: ""},
    "vis_id": {21: None, 35: LEGENDS[35]["vis_id"]},
}


# Settings for the query metadata
# these are the fields that are used to construct a query
QUERY_FIELDS = {
    21: None,
    35: None,
}  # empty list means all; None means do not use query fields.
# these are parameters that added to those automatically generated by the pipeline
QUERY_PARAMETERS = {
    21: {"is_tiled": True, "is_raster": True, "min_zoom_level": 12},
    35: {"is_tiled": True, "is_raster": True, "min_zoom_level": 10},
}


def convertZip(directory: str, ds_id=int, Remove=True):
    """Convert files downloaded from Copernicus."""
    files_list = glob.glob(os.path.join(directory, "*.zip"))
    if len(files_list) > 0:
        logging.info("Extracting zip files")
        if not os.path.exists(os.path.join(directory, "orig_tiles")):
            os.mkdir(os.path.join(directory, "orig_tiles"))
        for zipfile in files_list:
            extract_dir = os.path.join(
                os.path.dirname(zipfile), pathlib.Path(zipfile).stem
            )
            extracted = utilities.extractZip(zipfile, extract_dir)
            source_file = [x for x in extracted if x.endswith("TIF")][0]

            logging.info(source_file)

            dest_file = os.path.join(
                os.path.dirname(extract_dir),
                "orig_tiles",
                os.path.basename(extract_dir) + ".tif",
            )
            os.system(  # nosec
                "gdal_translate {source_file} {dest_file}  -of GTIFF --config"
                " GDAL_PAM_ENABLED NO -co COMPRESS={compress} -co BIGTIFF=YES".format(
                    source_file=source_file,
                    dest_file=dest_file,
                    compress=COMPRESS[ds_id],
                )
            )
            if Remove:
                shutil.rmtree(extract_dir)
                os.remove(zipfile)
    else:
        logging.info("There are no zip files to extract")


def tiling(directory: str, ds_id=int, Remove=True):
    """Tile data from Copernicus."""
    files_list = glob.glob(os.path.join(directory, "orig_tiles", "*.tif"))
    if len(files_list) > 0:
        logging.info("Tiling")
        for file in files_list:
            target_dir = os.path.join(directory, os.path.basename(file))[:-4]
            os.mkdir(target_dir)
            os.system(  # nosec
                "gdal_retile.py -ps {resolution} {resolution} -co COMPRESS={compress}"
                " -co BIGTIFF=YES -targetDir {target_dir} -csv tiles.csv -csvDelim ,"
                " {source_file} ".format(
                    resolution=RESOLUTION[ds_id],
                    compress=COMPRESS[ds_id],
                    target_dir=target_dir,
                    source_file=file,
                )
            )
        if Remove:
            shutil.rmtree(os.path.join(directory, "orig_tiles"))
    else:
        logging.info("There are no files to tile")


def get(directory):
    """Prepare df and gdf with ESM data."""
    files_list = glob.glob(os.path.join(directory, "*", "*.csv"))
    data = []
    for file in files_list:
        logging.info(file)
        tiles = pd.read_csv(file, header=None)
        tiles.columns = ["tilename", "minx", "maxx", "miny", "maxy"]
        tiles["extentBox"] = tiles.apply(
            lambda x: box(x.minx, x.miny, x.maxx, x.maxy), axis=1
        )
        tiles["tilename"] = (
            os.path.basename(os.path.dirname(file)) + "/" + tiles["tilename"]
        )
        data.append(tiles)
    data = pd.concat(data, ignore_index=True)

    enermaps_data = utilities.ENERMAPS_DF
    enermaps_data["fid"] = data["tilename"]
    enermaps_data["israster"] = ISRASTER

    spatial = gpd.GeoDataFrame(
        geometry=data["extentBox"],
        crs="EPSG:3035",
    )
    spatial["fid"] = data["tilename"]

    return enermaps_data, spatial


def addRecordMetadata(data: pd.DataFrame, metadata: dict):
    """Add custom metadata to each record in the data table."""
    for ds_id in data["ds_id"].unique():
        for column in metadata.keys():
            if isinstance(metadata[column][ds_id], dict):
                data.loc[data["ds_id"] == ds_id, column] = json.dumps(
                    metadata[column][ds_id]
                )
            else:
                data.loc[data["ds_id"] == ds_id, column] = metadata[column][ds_id]
    return data


if __name__ == "__main__":
    datasets = pd.read_csv("datasets.csv", index_col=[0])
    script_name = os.path.basename(sys.argv[0])
    ds_ids, isForced = utilities.parser(script_name, datasets)

    for ds_id in ds_ids:
        directory = "data/{}".format(ds_id)

        if (
            os.path.exists(directory)
            and os.path.isdir(directory)
            and len(os.listdir(directory)) == N_FILES[ds_id]
        ):
            # Dezip
            convertZip(directory, ds_id, Remove=True)

            # Retile
            tiling(directory, ds_id, Remove=True)

            data, spatial = get(directory)

            # Add metadata
            data["ds_id"] = ds_id
            data = addRecordMetadata(data, RECORD_METADATA)

            # Remove existing dataset
            if utilities.datasetExists(ds_id, DB_URL) and not isForced:
                raise FileExistsError("Use --force to replace the existing dataset.")
            elif utilities.datasetExists(ds_id, DB_URL) and isForced:
                utilities.removeDataset(ds_id, DB_URL)
                logging.info("Removed existing dataset")
            else:
                pass

            # Create dataset table
            metadata = datasets.loc[ds_id].fillna("").to_dict()
            # Add parameters as metadata
            (
                metadata["parameters"],
                metadata["default_parameters"],
            ) = utilities.get_query_metadata(
                data, QUERY_FIELDS[ds_id], QUERY_PARAMETERS[ds_id]
            )
            metadata = json.dumps(metadata)
            dataset = pd.DataFrame(
                [
                    {
                        "ds_id": ds_id,
                        "metadata": metadata,
                        "shared_id": datasets.loc[ds_id, "shared_id"],
                    }
                ]
            )
            utilities.toPostgreSQL(
                dataset,
                DB_URL,
                schema="datasets",
            )

            # Add legends
            legends = []
            for key in LEGENDS.keys():
                legends.append(
                    (LEGENDS[key]["vis_id"], json.dumps(LEGENDS[key]["legend"]))
                )
            legends_df = pd.DataFrame(legends, columns=["vis_id", "legend"])
            utilities.toPostgreSQL(
                legends_df,
                DB_URL,
                schema="visualization",
            )

            # Create data table
            utilities.toPostgreSQL(
                data,
                DB_URL,
                schema="data",
            )

            # Create spatial table
            spatial["ds_id"] = ds_id
            utilities.toPostGIS(
                spatial,
                DB_URL,
                schema="spatial",
            )
        else:
            logging.error(
                "The {} directory must exist and contain {} files from Copernicus."
                .format(directory, N_FILES)
            )

#!/usr/bin/env python3
"""
Prepare the ESM dataset and EUDEM for EnerMaps.

Note that the 25-m (EU-DEM) and 2.5-m (ESM) resolution files must be downloaded from Copernicus (requires log-in)
and extracted in the data/21 and data/35 directory.
This script expects that the original zip file from Copernicus is extracted
in multiple zip files, which will be then extracted and tiled.


@author: giuseppeperonato
"""

import glob
import json
import logging
import os
import pathlib
import shutil
import sys
import uuid

import geopandas as gpd
import pandas as pd
import utilities
from shapely.geometry import box

N_FILES = {21: 27, 35: 279}
COMPRESS = {21: "DEFLATE", 35: "DEFLATE"}
RESOLUTION = {21: 800, 35: 20000}
ISRASTER = True
logging.basicConfig(level=logging.INFO)

DB_URL = utilities.DB_URL
LEGENDS = {
    35: {
        "vis_id": uuid.uuid4(),
        "legend": {
            "name": "Land use",
            "type": "custom",
            "symbology": [
                {
                    "red": 112,
                    "green": 162,
                    "blue": 255,
                    "opacity": 1.0,
                    "value": "1",
                    "label": "Water",
                },
                {
                    "red": 102,
                    "green": 102,
                    "blue": 102,
                    "opacity": 1.0,
                    "value": "2",
                    "label": "Railways",
                },
                {
                    "red": 242,
                    "green": 242,
                    "blue": 242,
                    "opacity": 1.0,
                    "value": "10",
                    "label": "Non-built area - Open Space",
                },
                {
                    "red": 221,
                    "green": 230,
                    "blue": 207,
                    "opacity": 1.0,
                    "value": "20",
                    "label": "Non-built area - Green ndvix",
                },
                {
                    "red": 225,
                    "green": 225,
                    "blue": 225,
                    "opacity": 1.0,
                    "value": "30",
                    "label": "Built area - Open space",
                },
                {
                    "red": 181,
                    "green": 204,
                    "blue": 142,
                    "opacity": 1.0,
                    "value": "40",
                    "label": "Built area - Green ndvix",
                },
                {
                    "red": 200,
                    "green": 230,
                    "blue": 161,
                    "opacity": 1.0,
                    "value": "41",
                    "label": "Built area - Green Urban Atlas",
                },
                {
                    "red": 128,
                    "green": 125,
                    "blue": 121,
                    "opacity": 1.0,
                    "value": "50",
                    "label": "Built area - Built up",
                },
            ],
        },
    }
}

RECORD_METADATA = {
    "variable": {21: "Elevation", 35: "Land use"},
    "unit": {21: "m", 35: ""},
    "vis_id": {21: None, 35: LEGENDS[35]["vis_id"]},
}


# Settings for the query metadata
# these are the fields that are used to construct a query
QUERY_FIELDS = {
    21: None,
    35: None,
}  # empty list means all; None means do not use query fields.
# these are parameters that added to those automatically generated by the pipeline
QUERY_PARAMETERS = {
    21: {"is_tiled": True, "is_raster": True, "min_zoom_level": 12},
    35: {"is_tiled": True, "is_raster": True, "min_zoom_level": 10},
}


def convertZip(directory: str, ds_id=int, Remove=True):
    """Convert files downloaded from Copernicus."""
    files_list = glob.glob(os.path.join(directory, "*.zip"))
    if len(files_list) > 0:
        logging.info("Extracting zip files")
        if not os.path.exists(os.path.join(directory, "orig_tiles")):
            os.mkdir(os.path.join(directory, "orig_tiles"))
        for zipfile in files_list:
            extract_dir = os.path.join(
                os.path.dirname(zipfile), pathlib.Path(zipfile).stem
            )
            extracted = utilities.extractZip(zipfile, extract_dir)
            source_file = [x for x in extracted if x.endswith("TIF")][0]

            logging.info(source_file)

            dest_file = os.path.join(
                os.path.dirname(extract_dir),
                "orig_tiles",
                os.path.basename(extract_dir) + ".tif",
            )
            os.system(  # nosec
                "gdal_translate {source_file} {dest_file}  -of GTIFF --config"
                " GDAL_PAM_ENABLED NO -co COMPRESS={compress} -co BIGTIFF=YES".format(
                    source_file=source_file,
                    dest_file=dest_file,
                    compress=COMPRESS[ds_id],
                )
            )
            if Remove:
                shutil.rmtree(extract_dir)
                os.remove(zipfile)
    else:
        logging.info("There are no zip files to extract")


def tiling(directory: str, ds_id=int, Remove=True):
    """Tile data from Copernicus."""
    files_list = glob.glob(os.path.join(directory, "orig_tiles", "*.tif"))
    if len(files_list) > 0:
        logging.info("Tiling")
        for file in files_list:
            target_dir = os.path.join(directory, os.path.basename(file))[:-4]
            os.mkdir(target_dir)
            os.system(  # nosec
                "gdal_retile.py -ps {resolution} {resolution} -co COMPRESS={compress}"
                " -co BIGTIFF=YES -targetDir {target_dir} -csv tiles.csv -csvDelim ,"
                " {source_file} ".format(
                    resolution=RESOLUTION[ds_id],
                    compress=COMPRESS[ds_id],
                    target_dir=target_dir,
                    source_file=file,
                )
            )
        if Remove:
            shutil.rmtree(os.path.join(directory, "orig_tiles"))
    else:
        logging.info("There are no files to tile")


def get(directory):
    """Prepare df and gdf with ESM data."""
    files_list = glob.glob(os.path.join(directory, "*", "*.csv"))
    data = []
    for file in files_list:
        logging.info(file)
        tiles = pd.read_csv(file, header=None)
        tiles.columns = ["tilename", "minx", "maxx", "miny", "maxy"]
        tiles["extentBox"] = tiles.apply(
            lambda x: box(x.minx, x.miny, x.maxx, x.maxy), axis=1
        )
        tiles["tilename"] = (
            os.path.basename(os.path.dirname(file)) + "/" + tiles["tilename"]
        )
        data.append(tiles)
    data = pd.concat(data, ignore_index=True)

    enermaps_data = utilities.ENERMAPS_DF
    enermaps_data["fid"] = data["tilename"]
    enermaps_data["israster"] = ISRASTER

    spatial = gpd.GeoDataFrame(
        geometry=data["extentBox"],
        crs="EPSG:3035",
    )
    spatial["fid"] = data["tilename"]

    return enermaps_data, spatial


def addRecordMetadata(data: pd.DataFrame, metadata: dict):
    """Add custom metadata to each record in the data table."""
    for ds_id in data["ds_id"].unique():
        for column in metadata.keys():
            if isinstance(metadata[column][ds_id], dict):
                data.loc[data["ds_id"] == ds_id, column] = json.dumps(
                    metadata[column][ds_id]
                )
            else:
                data.loc[data["ds_id"] == ds_id, column] = metadata[column][ds_id]
    return data


if __name__ == "__main__":
    datasets = pd.read_csv("datasets.csv", index_col=[0])
    script_name = os.path.basename(sys.argv[0])
    ds_ids, isForced = utilities.parser(script_name, datasets)

    for ds_id in ds_ids:
        directory = "data/{}".format(ds_id)

        if (
            os.path.exists(directory)
            and os.path.isdir(directory)
            and len(os.listdir(directory)) == N_FILES[ds_id]
        ):
            # Dezip
            convertZip(directory, ds_id, Remove=True)

            # Retile
            tiling(directory, ds_id, Remove=True)

            data, spatial = get(directory)

            # Add metadata
            data["ds_id"] = ds_id
            data = addRecordMetadata(data, RECORD_METADATA)

            # Remove existing dataset
            if utilities.datasetExists(ds_id, DB_URL) and not isForced:
                raise FileExistsError("Use --force to replace the existing dataset.")
            elif utilities.datasetExists(ds_id, DB_URL) and isForced:
                utilities.removeDataset(ds_id, DB_URL)
                logging.info("Removed existing dataset")
            else:
                pass

            # Create dataset table
            metadata = datasets.loc[ds_id].fillna("").to_dict()
            # Add parameters as metadata
            (
                metadata["parameters"],
                metadata["default_parameters"],
            ) = utilities.get_query_metadata(
                data, QUERY_FIELDS[ds_id], QUERY_PARAMETERS[ds_id]
            )
            metadata = json.dumps(metadata)
            dataset = pd.DataFrame(
                [
                    {
                        "ds_id": ds_id,
                        "metadata": metadata,
                        "shared_id": datasets.loc[ds_id, "shared_id"],
                    }
                ]
            )
            utilities.toPostgreSQL(
                dataset,
                DB_URL,
                schema="datasets",
            )

            # Add legends
            legends = []
            for key in LEGENDS.keys():
                legends.append(
                    (LEGENDS[key]["vis_id"], json.dumps(LEGENDS[key]["legend"]))
                )
            legends_df = pd.DataFrame(legends, columns=["vis_id", "legend"])
            utilities.toPostgreSQL(
                legends_df,
                DB_URL,
                schema="visualization",
            )

            # Create data table
            utilities.toPostgreSQL(
                data,
                DB_URL,
                schema="data",
            )

            # Create spatial table
            spatial["ds_id"] = ds_id
            utilities.toPostGIS(
                spatial,
                DB_URL,
                schema="spatial",
            )
        else:
            logging.error(
                "The {} directory must exist and contain {} files from Copernicus."
                .format(directory, N_FILES)
            )

#!/usr/bin/env python3 Prepare the ESM dataset and EUDEM for EnerMaps.  Note that the 25-m (EU-DEM) and 2.5-m (ESM) resolution files must be downloaded from Copernicus (requires log-in) and extracted in the data/21 and data/35 directory. This script expects that the original zip file from Copernicus is extracted in multiple zip files, which will be then extracted and tiled.   @author: giuseppeperonato # Settings for the query metadata # these are the fields that are used to construct a query # empty list means all; None means do not use query fields. # these are parameters that added to those automatically generated by the pipeline Convert files downloaded from Copernicus. # nosec Tile data from Copernicus. # nosec Prepare df and gdf with ESM data. Add custom metadata to each record in the data table. # Dezip # Retile # Add metadata # Remove existing dataset # Create dataset table # Add parameters as metadata # Add legends # Create data table # Create spatial table