New input layer

pyproject.toml

@@ -34,7 +34,9 @@ dependencies = [
     "xlrd",
     "mypy-extensions",
     "pypubsub",
-    "tomlkit"
+    "tomlkit",
+    "duckdb",
+    "fsspec"
 ]
 dynamic = ["version"]
 

src/muse/data/example/default_new_input/agent_objectives.csv

@@ -0,0 +1,3 @@
+agent_id,objective,objective_data,objective_sort
+Agent1,LCOE,1,TRUE
+Agent2,LCOE,1,TRUE

src/muse/data/example/default_new_input/agent_pairs.csv

@@ -0,0 +1,2 @@
+id,new_agent_id,retrofit_agent_id,quantity
+A1,Agent1,Agent2,1

src/muse/data/example/default_new_input/agent_regions.csv

@@ -0,0 +1,2 @@
+agent_pair_id,region_id
+A1,R1

src/muse/data/example/default_new_input/agents.csv

@@ -0,0 +1,3 @@
+id,description,type,maturity_threshold,annual_cost_limit,search_rule,decision_rule
+Agent1,New agent for A1,new,-1,inf,all,single
+Agent2,Retrofit agent for A1,retrofit,-1,inf,all,single

src/muse/data/example/default_new_input/assets.csv

@@ -0,0 +1,8 @@
+agent_id,process_id,region_id,year,capacity
+Agent2,gassupply1,R1,2020,15
+Agent2,gassupply1,R1,2025,15
+Agent2,gassupply1,R1,2030,7.5
+Agent2,gasCCGT,R1,2020,1
+Agent2,gasCCGT,R1,2025,1
+Agent2,gasboiler,R1,2020,10
+Agent2,gasboiler,R1,2025,5

src/muse/data/example/default_new_input/commodities.csv

@@ -0,0 +1,6 @@
+id,description,type,unit
+electricity,Electricity,energy,PJ
+gas,Gas,energy,PJ
+heat,Heat,energy,PJ
+wind,Wind,energy,PJ
+C02f,Carbon dioxide,energy,kt

src/muse/data/example/default_new_input/commodity_costs.csv

@@ -0,0 +1,58 @@
+commodity_id,region_id,year,value
+electricity,R1,2010,14.81481472
+electricity,R1,2015,17.89814806
+electricity,R1,2020,19.5
+electricity,R1,2025,21.93518528
+electricity,R1,2030,26.50925917
+electricity,R1,2035,26.51851861
+electricity,R1,2040,23.85185194
+electricity,R1,2045,23.97222222
+electricity,R1,2050,24.06481472
+electricity,R1,2055,25.3425925
+electricity,R1,2060,25.53703694
+electricity,R1,2065,25.32407417
+electricity,R1,2070,23.36111111
+electricity,R1,2075,22.27777778
+electricity,R1,2080,22.25925917
+electricity,R1,2085,22.17592583
+electricity,R1,2090,22.03703694
+electricity,R1,2095,21.94444444
+electricity,R1,2100,21.39814806
+gas,R1,2010,6.6759
+gas,R1,2015,6.914325
+gas,R1,2020,7.15275
+gas,R1,2025,8.10645
+gas,R1,2030,9.06015
+gas,R1,2035,9.2191
+gas,R1,2040,9.37805
+gas,R1,2045,9.193829337
+gas,R1,2050,9.009608674
+gas,R1,2055,8.832625604
+gas,R1,2060,8.655642534
+gas,R1,2065,8.485612708
+gas,R1,2070,8.315582883
+gas,R1,2075,8.152233126
+gas,R1,2080,7.988883368
+gas,R1,2085,7.831951236
+gas,R1,2090,7.675019103
+gas,R1,2095,7.524252461
+gas,R1,2100,7.373485819
+CO2f,R1,2010,0
+CO2f,R1,2015,0.052913851
+CO2f,R1,2020,0.08314119
+CO2f,R1,2025,0.120069795
+CO2f,R1,2030,0.156998399
+CO2f,R1,2035,0.214877567
+CO2f,R1,2040,0.272756734
+CO2f,R1,2045,0.35394801
+CO2f,R1,2050,0.435139285
+CO2f,R1,2055,0.542365578
+CO2f,R1,2060,0.649591871
+CO2f,R1,2065,0.780892624
+CO2f,R1,2070,0.912193378
+CO2f,R1,2075,1.078321687
+CO2f,R1,2080,1.244449995
+CO2f,R1,2085,1.4253503
+CO2f,R1,2090,1.606250604
+CO2f,R1,2095,1.73877515
+CO2f,R1,2100,1.871299697

src/muse/data/example/default_new_input/commodity_trade.csv

@@ -0,0 +1 @@
+commodity_id,region_id,year,import,export

src/muse/data/example/default_new_input/demand.csv

@@ -0,0 +1,3 @@
+commodity_id,region_id,year,demand
+heat,R1,2020,10
+heat,R1,2050,30

src/muse/data/example/default_new_input/demand_slicing.csv

@@ -0,0 +1,7 @@
+commodity_id,region_id,year,timeslice,fraction
+heat,R1,,night,0.1
+heat,R1,,morning,0.15
+heat,R1,,afternoon,0.1
+heat,R1,,early-peak,0.15
+heat,R1,,late-peak,0.3
+heat,R1,,evening,0.2

src/muse/data/example/default_new_input/process_availabilities.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,timeslice,lim_type,value
+gassupply1,,,ALL,UP,0.9
+gasCCGT,,,ALL,UP,0.9
+windturbine,,,ALL,UP,0.4
+gasboiler,,,ALL,UP,1
+heatpump,,,ALL,UP,1

src/muse/data/example/default_new_input/process_flows.csv

@@ -0,0 +1,12 @@
+process_id,commodity_id,region_id,year,flow
+gassupply1,gas,,,1
+gasCCGT,gas,,,-1.67
+gasCCGT,electricity,,,1
+gasCCGT,CO2f,,,91.67
+windturbine,wind,,,-1
+windturbine,electricity,,,1
+gasboiler,gas,,,-1.16
+gasboiler,heat,,,1
+gasboiler,CO2f,,,64.71
+heatpump,electricity,,,-0.4
+heatpump,heat,,,1

src/muse/data/example/default_new_input/process_parameters.csv

@@ -0,0 +1,6 @@
+process_id,region_id,year,cap_par,cap_exp,fix_par,fix_exp,var_par,var_exp,max_capacity_addition,max_capacity_growth,total_capacity_limit,life,scaling_size,efficiency,discount_rate
+gassupply1,,,0,1,0,1,2.55,1,5,1,60,35,0.00000189,86,0.1
+gasCCGT,,,23.78234399,1,0,1,0,1,2,1,60,35,0.00000189,86,0.1
+windturbine,,,36.30771182,1,0,1,0,1,2,1,60,25,0.00000189,86,0.1
+gasboiler,,,3.8,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1
+heatpump,,,8.866667,1,0,1,0,1,10,0.02,60,10,0.00000189,86,0.1

src/muse/data/example/default_new_input/process_regions.csv

@@ -0,0 +1,6 @@
+process_id,region_id
+gassupply1,R1
+gasCCGT,R1
+windturbine,R1
+gasboiler,R1
+heatpump,R1

src/muse/data/example/default_new_input/processes.csv

@@ -0,0 +1,6 @@
+id,sector_id,type,fuel,end_use,level
+gassupply1,gas,energy,gas,gas,fixed
+gasCCGT,power,energy,gas,electricity,fixed
+windturbine,power,energy,wind,electricity,fixed
+gasboiler,residential,energy,gas,heat,fixed
+heatpump,residential,energy,electricity,heat,fixed

src/muse/data/example/default_new_input/regions.csv

@@ -0,0 +1,2 @@
+id,description
+R1,Region 1

src/muse/data/example/default_new_input/sectors.csv

@@ -0,0 +1,4 @@
+id,description
+gas,Gas sector
+power,Power sector
+residential,Residential sector

src/muse/data/example/default_new_input/time_slices.csv

@@ -0,0 +1,7 @@
+season,day,time_of_day,fraction
+all,all,night,0.1667
+all,all,morning,0.1667
+all,all,afternoon,0.1667
+all,all,early-peak,0.1667
+all,all,late-peak,0.1667
+all,all,evening,0.1667

src/muse/examples.py

@@ -48,7 +48,8 @@ def example_data_dir() -> Path:
 
 def available_examples() -> list[str]:
     """List examples available in the examples folder."""
-    return [d.stem for d in example_data_dir().iterdir() if d.is_dir()]
+    # temporary skip for default_new_input as this is not yet working
+    return [d.stem for d in example_data_dir().iterdir()]
 
 
 def model(name: str = "default") -> MCA:
@@ -111,6 +112,8 @@ def copy_model(
         _copy_minimum_service(path)
     elif name.lower() == "trade":
         _copy_trade(path)
+    elif name.lower() == "default_new_input":
+        _copy_default_new_input(path)
     return path
 
 
@@ -287,6 +290,12 @@ def _copy_default(path: Path):
     copyfile(example_data_dir() / "default" / "settings.toml", path / "settings.toml")
 
 
+def _copy_default_new_input(path: Path):
+    from shutil import copytree
+
+    copytree(example_data_dir() / "default_new_input", path)
+
+
 def _copy_default_retro(path: Path):
     from shutil import copyfile, copytree
 

tests/test_fullsim_regression.py

@@ -4,11 +4,15 @@
 
 from muse.examples import available_examples
 
+# hack to skip regression tests for new input schema
+MODELS = available_examples()
+MODELS.pop(MODELS.index("default_new_input"))
+
 
 @mark.usefixtures("save_timeslice_globals")
 @mark.regression
 @mark.example
-@mark.parametrize("model", available_examples())
+@mark.parametrize("model", MODELS)
 def test_fullsim_regression(model, tmpdir, compare_dirs):
     from warnings import simplefilter
 

tests/test_readers.py

@@ -5,7 +5,7 @@
 import numpy as np
 import toml
 import xarray as xr
-from pytest import fixture, mark, raises
+from pytest import approx, fixture, mark, raises
 
 
 @fixture
@@ -856,3 +856,148 @@ def test_check_utilization_and_minimum_service_factors_fail_missing_utilization(
 
     with raises(ValueError):
         check_utilization_and_minimum_service_factors(df, "file.csv")
+
+
+@fixture
+def default_new_input(tmp_path):
+    from muse.examples import copy_model
+
+    copy_model("default_new_input", tmp_path)
+    return tmp_path
+
+
+@mark.xfail
+def test_read_new_global_commodities(default_new_input):
+    from muse.new_input.readers import read_inputs
+
+    all_data = read_inputs(default_new_input)
+    data = all_data["global_commodities"]
+
+    assert isinstance(data, xr.Dataset)
+    assert set(data.dims) == {"commodity"}
+    assert dict(data.dtypes) == dict(
+        type=np.dtype("str"),
+        unit=np.dtype("str"),
+    )
+
+    assert list(data.coords["commodity"].values) == [
+        "electricity",
+        "gas",
+        "heat",
+        "wind",
+        "CO2f",
+    ]
+    assert list(data.data_vars["type"].values) == ["energy"] * 5
+    assert list(data.data_vars["unit"].values) == ["PJ"] * 4 + ["kt"]
+
+
+@mark.xfail
+def test_read_demand(default_new_input):
+    from muse.new_input.readers import read_inputs
+
+    all_data = read_inputs(default_new_input)
+    data = all_data["demand"]
+
+    assert isinstance(data, xr.DataArray)
+    assert data.dtype == np.float64
+
+    assert set(data.dims) == {"year", "commodity", "region", "timeslice"}
+    assert list(data.coords["region"].values) == ["R1"]
+    assert list(data.coords["timeslice"].values) == list(range(1, 7))
+    assert list(data.coords["year"].values) == [2020, 2050]
+    assert set(data.coords["commodity"].values) == {
+        "electricity",
+        "gas",
+        "heat",
+        "wind",
+        "CO2f",
+    }
+
+    assert data.sel(year=2020, commodity="electricity", region="R1", timeslice=0) == 1
+
+
+@mark.xfail
+def test_new_read_initial_market(default_new_input):
+    from muse.new_input.readers import read_inputs
+
+    all_data = read_inputs(default_new_input)
+    data = all_data["initial_market"]
+
+    assert isinstance(data, xr.Dataset)
+    assert set(data.dims) == {"region", "year", "commodity", "timeslice"}
+    assert dict(data.dtypes) == dict(
+        prices=np.float64,
+        exports=np.float64,
+        imports=np.float64,
+        static_trade=np.float64,
+    )
+    assert list(data.coords["region"].values) == ["R1"]
+    assert list(data.coords["year"].values) == list(range(2010, 2105, 5))
+    assert list(data.coords["commodity"].values) == [
+        "electricity",
+        "gas",
+        "heat",
+        "CO2f",
+        "wind",
+    ]
+    month_values = ["all-year"] * 6
+    day_values = ["all-week"] * 6
+    hour_values = [
+        "night",
+        "morning",
+        "afternoon",
+        "early-peak",
+        "late-peak",
+        "evening",
+    ]
+
+    assert list(data.coords["timeslice"].values) == list(
+        zip(month_values, day_values, hour_values)
+    )
+    assert list(data.coords["month"]) == month_values
+    assert list(data.coords["day"]) == day_values
+    assert list(data.coords["hour"]) == hour_values
+
+    assert all(var.coords.equals(data.coords) for var in data.data_vars.values())
+
+    prices = data.data_vars["prices"]
+    assert approx(
+        prices.sel(
+            year=2010,
+            region="R1",
+            commodity="electricity",
+            timeslice=("all-year", "all-week", "night"),
+        )
+        - 14.81481,
+        abs=1e-4,
+    )
+
+    exports = data.data_vars["exports"]
+    assert (
+        exports.sel(
+            year=2010,
+            region="R1",
+            commodity="electricity",
+            timeslice=("all-year", "all-week", "night"),
+        )
+    ) == 0
+
+    imports = data.data_vars["imports"]
+    assert (
+        imports.sel(
+            year=2010,
+            region="R1",
+            commodity="electricity",
+            timeslice=("all-year", "all-week", "night"),
+        )
+    ) == 0
+
+    static_trade = data.data_vars["static_trade"]
+    assert (
+        static_trade.sel(
+            year=2010,
+            region="R1",
+            commodity="electricity",
+            timeslice=("all-year", "all-week", "night"),
+        )
+    ) == 0