Merge pull request #1637 from gammasim/use-acada-telescope-ids

Add common array elements id. Add to reduced event data tables.

Author: GernotMaier

docs/changes/1637.feature.md

@@ -0,0 +1 @@
+Add common array elements ID. Add list with triggered common telescope IDs to reduced event data tables.

src/simtools/constants.py

@@ -21,3 +21,5 @@
     "https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/"
     "/model_parameters"
 )
+# Path to resource files
+RESOURCE_PATH = files("simtools") / "resources"

src/simtools/resources/array-element-ids.json

@@ -0,0 +1,126 @@
+{
+  "metadata": {
+    "CTA DATA MODEL NAME": "ctao.common.identifiers.array_elements",
+    "CTA DATA MODEL URL": "https://gitlab.cta-observatory.org/cta-computing/common",
+    "CTA DATA MODEL VERSION": 2,
+    "CTA DATA PRODUCT DESCRIPTION": "This file contains a mapping of array element ID number to human-readable name. The ids define the telescopes and other array elements at CTAO-North and CTAO-South",
+    "CTA PRODUCT ID": "350917c3-fb77-453d-b1bc-296471f5e7fb"
+  },
+
+  "array_elements": [
+    {"id": 1, "name": "LSTN-01"},
+    {"id": 2, "name": "LSTN-02"},
+    {"id": 3, "name": "LSTN-03"},
+    {"id": 4, "name": "LSTN-04"},
+    {"id": 5, "name": "MSTN-01"},
+    {"id": 6, "name": "MSTN-02"},
+    {"id": 7, "name": "MSTN-03"},
+    {"id": 8, "name": "MSTN-04"},
+    {"id": 9, "name": "MSTN-05"},
+    {"id": 10, "name": "MSTN-06"},
+    {"id": 11, "name": "MSTN-07"},
+    {"id": 12, "name": "MSTN-08"},
+    {"id": 13, "name": "MSTN-09"},
+    {"id": 14, "name": "MSTN-10"},
+    {"id": 15, "name": "MSTN-11"},
+    {"id": 16, "name": "MSTN-12"},
+    {"id": 17, "name": "MSTN-13"},
+    {"id": 18, "name": "MSTN-14"},
+    {"id": 19, "name": "MSTN-15"},
+    {"id": 20, "name": "RLDN-01"},
+    {"id": 21, "name": "STPN-01"},
+    {"id": 22, "name": "MSPN-01"},
+    {"id": 23, "name": "ILLN-01"},
+    {"id": 24, "name": "ILLN-02"},
+    {"id": 25, "name": "CEIN-01"},
+    {"id": 26, "name": "WSTN-01"},
+    {"id": 27, "name": "WSTN-02"},
+    {"id": 28, "name": "WSTN-03"},
+    {"id": 29, "name": "ASCN-01"},
+    {"id": 30, "name": "DUSN-01"},
+    {"id": 31, "name": "LISN-01"},
+    {"id": 101, "name": "LSTS-01"},
+    {"id": 102, "name": "LSTS-02"},
+    {"id": 103, "name": "LSTS-03"},
+    {"id": 104, "name": "LSTS-04"},
+    {"id": 105, "name": "MSTS-01"},
+    {"id": 106, "name": "MSTS-02"},
+    {"id": 107, "name": "MSTS-03"},
+    {"id": 108, "name": "MSTS-04"},
+    {"id": 109, "name": "MSTS-05"},
+    {"id": 110, "name": "MSTS-06"},
+    {"id": 111, "name": "MSTS-07"},
+    {"id": 112, "name": "MSTS-08"},
+    {"id": 113, "name": "MSTS-09"},
+    {"id": 114, "name": "MSTS-10"},
+    {"id": 115, "name": "MSTS-11"},
+    {"id": 116, "name": "MSTS-12"},
+    {"id": 117, "name": "MSTS-13"},
+    {"id": 118, "name": "MSTS-14"},
+    {"id": 119, "name": "SSTS-01"},
+    {"id": 120, "name": "SSTS-02"},
+    {"id": 121, "name": "SSTS-03"},
+    {"id": 122, "name": "SSTS-04"},
+    {"id": 123, "name": "SSTS-05"},
+    {"id": 124, "name": "SSTS-06"},
+    {"id": 125, "name": "SSTS-07"},
+    {"id": 126, "name": "SSTS-08"},
+    {"id": 127, "name": "SSTS-09"},
+    {"id": 128, "name": "SSTS-10"},
+    {"id": 129, "name": "SSTS-11"},
+    {"id": 130, "name": "SSTS-12"},
+    {"id": 131, "name": "SSTS-13"},
+    {"id": 132, "name": "SSTS-14"},
+    {"id": 133, "name": "SSTS-15"},
+    {"id": 134, "name": "SSTS-16"},
+    {"id": 135, "name": "SSTS-17"},
+    {"id": 136, "name": "SSTS-18"},
+    {"id": 137, "name": "SSTS-19"},
+    {"id": 138, "name": "SSTS-20"},
+    {"id": 139, "name": "SSTS-21"},
+    {"id": 140, "name": "SSTS-22"},
+    {"id": 141, "name": "SSTS-23"},
+    {"id": 142, "name": "SSTS-24"},
+    {"id": 143, "name": "SSTS-25"},
+    {"id": 144, "name": "SSTS-26"},
+    {"id": 145, "name": "SSTS-27"},
+    {"id": 146, "name": "SSTS-28"},
+    {"id": 147, "name": "SSTS-29"},
+    {"id": 148, "name": "SSTS-30"},
+    {"id": 149, "name": "SSTS-31"},
+    {"id": 150, "name": "SSTS-32"},
+    {"id": 151, "name": "SSTS-33"},
+    {"id": 152, "name": "SSTS-34"},
+    {"id": 153, "name": "SSTS-35"},
+    {"id": 154, "name": "SSTS-36"},
+    {"id": 155, "name": "SSTS-37"},
+    {"id": 156, "name": "SSTS-38"},
+    {"id": 157, "name": "SSTS-39"},
+    {"id": 158, "name": "SSTS-40"},
+    {"id": 159, "name": "SSTS-41"},
+    {"id": 160, "name": "SSTS-42"},
+    {"id": 161, "name": "SSTS-43"},
+    {"id": 162, "name": "SSTS-44"},
+    {"id": 163, "name": "SSTS-45"},
+    {"id": 164, "name": "SSTS-46"},
+    {"id": 165, "name": "SSTS-47"},
+    {"id": 166, "name": "SSTS-48"},
+    {"id": 167, "name": "SSTS-49"},
+    {"id": 168, "name": "SSTS-50"},
+    {"id": 169, "name": "CEIS-01"},
+    {"id": 170, "name": "RLDS-01"},
+    {"id": 171, "name": "RLDS-02"},
+    {"id": 172, "name": "STPS-01"},
+    {"id": 173, "name": "STPS-02"},
+    {"id": 174, "name": "MSPS-01"},
+    {"id": 175, "name": "ILLS-01"},
+    {"id": 176, "name": "ILLS-02"},
+    {"id": 177, "name": "ILLS-03"},
+    {"id": 178, "name": "ILLS-04"},
+    {"id": 179, "name": "WSTS-01"},
+    {"id": 180, "name": "WSTS-02"},
+    {"id": 181, "name": "WSTS-03"},
+    {"id": 182, "name": "ASCS-01"},
+    {"id": 183, "name": "DUSS-01"}
+  ]
+}

src/simtools/schemas/array_elements.yml renamed to src/simtools/resources/array_elements.yml

@@ -22,6 +22,7 @@
     get_sim_telarray_telescope_id_to_telescope_name_mapping,
 )
 from simtools.utils.geometry import calculate_circular_mean
+from simtools.utils.names import get_common_identifier_from_array_element_name
 
 
 @dataclass
@@ -47,6 +48,7 @@ class TableSchemas:
         "array_altitude": (np.float64, u.rad),
         "array_azimuth": (np.float64, u.rad),
         "telescope_list": (str, None),  # Store as comma-separated string
+        "telescope_list_common_id": (str, None),  # Store as comma-separated string
     }
 
     file_info_schema = {
@@ -271,6 +273,12 @@ def _fill_array_event(self, telescopes, tracking_positions, event_id, file_id):
                 "array_altitude": float(np.mean(altitudes)),
                 "array_azimuth": float(calculate_circular_mean(azimuths)),
                 "telescope_list": ",".join(map(str, telescopes)),
+                "telescope_list_common_id": ",".join(
+                    [
+                        str(get_common_identifier_from_array_element_name(tel, 0))
+                        for tel in telescopes
+                    ]
+                ),
             }
         )
 

src/simtools/simtel/simtel_io_event_writer.py

@@ -12,6 +12,7 @@
 
 """
 
+import json
 import logging
 import re
 from functools import cache
@@ -22,7 +23,7 @@
 from simtools.constants import (
     MODEL_PARAMETER_DESCRIPTION_METASCHEMA,
     MODEL_PARAMETER_SCHEMA_PATH,
-    SCHEMA_PATH,
+    RESOURCE_PATH,
 )
 
 _logger = logging.getLogger(__name__)
@@ -59,10 +60,30 @@ def array_elements():
     dict
         Array elements.
     """
-    with open(Path(SCHEMA_PATH) / "array_elements.yml", encoding="utf-8") as file:
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    with open(Path(RESOURCE_PATH) / "array_elements.yml", encoding="utf-8") as file:
         return yaml.safe_load(file)["data"]
 
 
+@cache
+def array_element_common_identifiers():
+    """
+    Get array element IDs from CTAO common identifier.
+
+    Returns
+    -------
+    dict, dict
+        Dictionary mapping array element names to their IDs and vice versa.
+    """
+    # for efficiency reason, no functions from simtools.utils.general are used here
+    id_to_name = {}
+    with open(Path(RESOURCE_PATH) / "array-element-ids.json", encoding="utf-8") as file:
+        data = json.load(file)
+    id_to_name = {e["id"]: e["name"] for e in data["array_elements"]}
+    name_to_id = {e["name"]: e["id"] for e in data["array_elements"]}
+    return id_to_name, name_to_id
+
+
 @cache
 def simulation_software():
     """
@@ -415,6 +436,54 @@ def get_array_element_id_from_name(array_element_name):
         raise ValueError(f"Invalid name {array_element_name}") from exc
 
 
+def get_common_identifier_from_array_element_name(array_element_name, default_return=None):
+    """
+    Get numerical common identifier from array element name as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    array_element_name: str
+        Array element name (e.g. LSTN-01)
+
+    Returns
+    -------
+    int
+        Common identifier.
+    """
+    _, name_to_id = array_element_common_identifiers()
+    try:
+        return name_to_id[array_element_name]
+    except KeyError as exc:
+        if default_return is not None:
+            return default_return
+        raise ValueError(f"Unknown array element name {array_element_name}") from exc
+
+
+def get_array_element_name_from_common_identifier(common_identifier):
+    """
+    Get array element name from common identifier as used by CTAO.
+
+    Common identifiers are numerical IDs used by the CTAO ACADA and DPPS systems.
+
+    Parameters
+    ----------
+    common_identifier: int
+        Common identifier.
+
+    Returns
+    -------
+    str
+        Array element name.
+    """
+    id_to_name, _ = array_element_common_identifiers()
+    try:
+        return id_to_name[common_identifier]
+    except KeyError as exc:
+        raise ValueError(f"Unknown common identifier {common_identifier}") from exc
+
+
 def get_list_of_array_element_types(
     array_element_class="telescopes", site=None, observatory="CTAO"
 ):

src/simtools/utils/names.py

@@ -15,7 +15,6 @@
 
 OUTPUT_FILE_NAME = "output.fits"
 one_two_three = "LSTN-01,LSTN-02,MSTN-01"
-unknown_one_two_three = "Unknown_1,Unknown_2,Unknown_3"
 
 
 @pytest.fixture
@@ -224,13 +223,16 @@ def test_process_array_event(lookup_table_generator):
 
     lookup_table_generator.shower_data.append({"shower_id": 1, "event_id": 42, "file_id": 0})
 
-    lookup_table_generator._process_array_event(mock_array_event, 0)
+    with patch.object(
+        lookup_table_generator, "_map_telescope_names", return_value=one_two_three.split(",")
+    ):
+        lookup_table_generator._process_array_event(mock_array_event, 0)
 
     assert len(lookup_table_generator.trigger_data) == 1
     trigger_event = lookup_table_generator.trigger_data[0]
     assert trigger_event["shower_id"] == 1
     assert trigger_event["event_id"] == 42
-    assert trigger_event["telescope_list"] == unknown_one_two_three
+    assert trigger_event["telescope_list"] == one_two_three
 
 
 def test_process_array_event_empty(lookup_table_generator):
@@ -251,13 +253,16 @@ def test_process_array_event_with_trigger_data(lookup_table_generator):
     mock_array_event = create_array_event()
     lookup_table_generator.shower_data.append({"shower_id": 1, "event_id": 42, "file_id": 0})
 
-    lookup_table_generator._process_array_event(mock_array_event, 0)
+    with patch.object(
+        lookup_table_generator, "_map_telescope_names", return_value=one_two_three.split(",")
+    ):
+        lookup_table_generator._process_array_event(mock_array_event, 0)
 
     assert len(lookup_table_generator.trigger_data) == 1
     trigger_event = lookup_table_generator.trigger_data[0]
     assert trigger_event["shower_id"] == 1
     assert trigger_event["event_id"] == 42
-    assert trigger_event["telescope_list"] == unknown_one_two_three
+    assert trigger_event["telescope_list"] == one_two_three
 
 
 def test_get_preliminary_nsb_level(lookup_table_generator):

tests/unit_tests/simtel/test_simtel_io_event_writer.py

@@ -507,6 +507,12 @@ def test_generate_file_name_ray_tracing():
     )
 
 
+def test_simulation_software():
+    software = names.simulation_software()
+    assert isinstance(software, list)
+    assert "sim_telarray" in software
+
+
 def test_get_simulation_software_name_from_parameter_name():
     sim_telarray = "sim_telarray"
     assert (
@@ -597,3 +603,44 @@ def test_is_design_type():
     assert names.is_design_type("MSTS-FlashCam")
     assert names.is_design_type("MSTS-NectarCam")
     assert not names.is_design_type("MSTS-22")
+
+
+def test_array_element_common_identifiers():
+    id_to_name, name_to_id = names.array_element_common_identifiers()
+    assert isinstance(id_to_name, dict)
+    assert isinstance(name_to_id, dict)
+    assert len(id_to_name) > 0
+    assert len(name_to_id) > 0
+
+    # Check that the dictionaries are consistent
+    for name, id_ in name_to_id.items():
+        assert id_ in id_to_name
+        assert id_to_name[id_] == name
+
+    for id_, name in id_to_name.items():
+        assert name in name_to_id
+        assert name_to_id[name] == id_
+
+
+def test_get_common_identifier_from_array_element_name():
+    assert names.get_common_identifier_from_array_element_name("LSTN-01") == 1
+    assert names.get_common_identifier_from_array_element_name("MSTN-08") == 12
+    assert names.get_common_identifier_from_array_element_name("SSTS-03") == 121
+
+    with pytest.raises(ValueError, match="Unknown array element name Not_a_name"):
+        names.get_common_identifier_from_array_element_name("Not_a_name")
+
+    assert names.get_common_identifier_from_array_element_name("Not_a_name", default_return=0) == 0
+
+
+def test_get_array_element_name_from_common_identifier():
+    id_to_name, _ = names.array_element_common_identifiers()
+
+    # Check some known identifiers
+    assert names.get_array_element_name_from_common_identifier(1) == "LSTN-01"
+    assert names.get_array_element_name_from_common_identifier(12) == "MSTN-08"
+    assert names.get_array_element_name_from_common_identifier(121) == "SSTS-03"
+
+    # Check that the function raises an error for an unknown identifier
+    with pytest.raises(ValueError, match="Unknown common identifier 9999"):
+        names.get_array_element_name_from_common_identifier(9999)