diff --git a/cli_081_india_andhra_tn_climate_projections/README.md b/cli_081_india_andhra_tn_climate_projections/README.md new file mode 100644 index 0000000..3743e90 --- /dev/null +++ b/cli_081_india_andhra_tn_climate_projections/README.md @@ -0,0 +1,14 @@ +## Ted Wong created dataset for long term climate projections for two states in India + +This dataset is provided by the source as a series of tif files, which were uploaded to Google Earth Engine. + +You can view the processed dataset for display on Agriadapt at the links below. +https://agriadapt.org/explore/cli081crt0-Projected-minimum-temperature-mean?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli082crt0-Projected-maximum-temperature-mean?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli083crt0-Projected-mean-temperature?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli084crt0-Projected-Extreme-rainy-days?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 +https://agriadapt.org/explore/cli085crt0-Projected-total-rainfall?zoom=3&lat=7.2752923363727575&lng=65.69850221673482&pitch=0&bearing=0&basemap=dark&labels=light&tab=layers&emission_scenario=rcp4.5 + +You can also download the original datasets at https://wri-agriadapt-datasets.s3.us-east-1.amazonaws.com/cli_081/climate_data_proj_india_ap_tn_ted.zip + +###### Note: This dataset processing was done by [Raghuram BK] diff --git a/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py b/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py new file mode 100644 index 0000000..20af4d7 --- /dev/null +++ b/cli_081_india_andhra_tn_climate_projections/cli_081_india_andhra_tn_climate_projections.py @@ -0,0 +1,133 @@ +import ee +import os +import sys +from google.cloud import storage +from google_drive_downloader import GoogleDriveDownloader as gdd +import logging +from zipfile import ZipFile +from pathlib import Path + +utils_path = os.path.join(os.path.abspath(os.getenv("PROCESSING_DIR")), "utils") +if utils_path not in sys.path: + sys.path.append(utils_path) +import util_files +import util_cloud + +# Get the top-level logger object +logger = logging.getLogger() +for handler in logger.handlers: + logger.removeHandler(handler) +logger.setLevel(logging.DEBUG) +# make it print to the console. +console = logging.StreamHandler() +logger.addHandler(console) +logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") + +# name of asset on GEE where you want to upload data +# this should be an asset name that is not currently in use +dataset_name = "cli_081_india_andhra_tn_climate_projections" +logger.info("Executing script for dataset: " + dataset_name) + +data_dir = util_files.prep_dirs(dataset_name) +logger.debug("Data directory relative path: " + data_dir) +logger.debug("Data directory absolute path: " + os.path.abspath(data_dir)) + +dataset = "climate_data_proj_india_ap_tn_ted" + +# Extract the datafiles from the zip +with ZipFile(os.path.join(f"{os.path.abspath(data_dir)}", f"{dataset}.zip"), "r") as zf: + zf.extractall(os.path.abspath(data_dir)) + +""" +Process data and upload processed data to Google Earth Engine +""" + +# initialize ee and eeUtil modules for uploading to Google Earth Engine + +auth = ee.ServiceAccountCredentials( + os.getenv("GEE_SERVICE_ACCOUT"), os.getenv("GOOGLE_APPLICATION_CREDENTIALS") +) +ee.Initialize(auth) + +# set up Google Cloud Storage project and bucket objects +gcs_client = storage.Client(os.environ.get("CLOUDSDK_CORE_PROJECT")) +gcs_bucket = gcs_client.bucket(os.environ.get("GEE_STAGING_BUCKET")) + +indicators = { + "total_rainfall": "TotalRainfall/Annual", + "extreme_rainy_days": "ExtremeRainyDays", + "mean_temp": "MeanTemp", + "max_temp": "MaxTemp/Annual", + "min_temp": "MinTemp/Annual", +} +print("creating folder") + ee.data.createAsset( + {"type": "Folder"}, "projects/resource-watch-gee/climate_data_proj_india_ap_tn_ted" + ) +print("created folder") +measures = ["mean"] +for indicator, indicator_dir in indicators.items(): + for measure in measures: + # print(f"projects/resource-watch-gee/{dataset}/{dataset}_{indicator}_{measure}") + # print(f"{os.path.abspath(data_dir)}/{dataset}/{indicator_dir}") + + image_collection = ( + f"projects/resource-watch-gee/{dataset}/{dataset}_{indicator}_{measure}" + ) + print(image_collection) + # create IC + # Already created by Chris so skip creating the image collection + ee.data.createAsset({"type": "ImageCollection"}, image_collection) + # set dataset privacy to public + acl = {"all_users_can_read": True} + ee.data.setAssetAcl(image_collection, acl) + logger.info("Privacy set to public.") + + for path in Path( + f"{os.path.abspath(data_dir)}/{dataset}/{indicator_dir}" + ).rglob(f"*_{measure}*"): + logger.debug(f"Processing {path}") + + image_name = os.path.splitext(os.path.basename(path))[0] + + """ + Upload processed data to Google Earth Engine + """ + logger.info("Uploading processed data to Google Cloud Storage.") + + # upload local files to Google Cloud Storage + gs_uris = util_cloud.gcs_upload( + str(path), os.path.join(dataset, f"{indicator}_{measure}") + ) + + logger.info("Uploading processed data to Google Earth Engine.") + + # set pyramiding policy for GEE upload + pyramiding_policy = "MEAN" # check + + # create manifests for upload to GEE + band_manifest = [ + { + "id": "Band1", + "tileset_band_index": 0, + "tileset_id": os.path.basename(gs_uris[0]).split(".")[0], + "pyramidingPolicy": pyramiding_policy, + } + ] + + asset_name = f"{image_collection}/{image_name}" + manifest = util_cloud.gee_manifest_complete( + asset_name, gs_uris[0], band_manifest + ) + + properties = { + "SCP": "ssp245.ssp585", + "change_vs_absolute": "abs", + } + manifest["properties"] = properties + # upload from GCS to GEE + util_cloud.gee_ingest(manifest, public=True) + # delete files from GCS + util_cloud.gcs_remove(gs_uris, gcs_bucket=gcs_bucket) + + logger.info("Files deleted from Google Cloud Storage.")