ENH: Python derived/accum interface

darshan-util/pydarshan/darshan/backend/api_def_c.py

@@ -8,6 +8,30 @@
 
 
 header = """/* from darshan-logutils.h */
+
+struct darshan_file_category_counters {
+    int64_t count;                   /* number of files in this category */
+    int64_t total_read_volume_bytes; /* total read traffic volume */
+    int64_t total_write_volume_bytes;/* total write traffic volume */
+    int64_t max_read_volume_bytes;   /* maximum read traffic volume to 1 file */
+    int64_t max_write_volume_bytes;  /* maximum write traffic volume to 1 file */
+    int64_t total_max_offset_bytes;  /* summation of max_offsets */
+    int64_t max_offset_bytes;        /* largest max_offset */
+    int64_t nprocs;                  /* how many procs accessed (-1 for "all") */
+};
+
+struct darshan_derived_metrics {
+    int64_t total_bytes;
+    double unique_io_total_time_by_slowest;
+    double unique_rw_only_time_by_slowest;
+    double unique_md_only_time_by_slowest;
+    int unique_io_slowest_rank;
+    double shared_io_total_time_by_slowest;
+    double agg_perf_by_slowest;
+    double agg_time_by_slowest;
+    struct darshan_file_category_counters category_counters[7];
+};
+
 struct darshan_mnt_info
 {
     char mnt_type[3015];
@@ -23,6 +47,20 @@
     int partial_flag;
 };
 
+/* opaque accumulator reference */
+struct darshan_accumulator_st;
+typedef struct darshan_accumulator_st* darshan_accumulator;
+
+/* NOTE: darshan_module_id is technically an enum in the C API, but we'll
+ * just use an int for now (equivalent type) to avoid warnings from cffi
+ * that we have not defined explicit enum values.  We don't need that
+ * functionality.
+ */
+int darshan_accumulator_create(int darshan_module_id, int64_t, darshan_accumulator*);
+int darshan_accumulator_inject(darshan_accumulator, void*, int);
+int darshan_accumulator_emit(darshan_accumulator, struct darshan_derived_metrics*, void* aggregation_record);
+int darshan_accumulator_destroy(darshan_accumulator);
+
 /* from darshan-log-format.h */
 typedef uint64_t darshan_record_id;
 

darshan-util/pydarshan/darshan/backend/cffi_backend.py

@@ -657,3 +657,72 @@ def _log_get_heatmap_record(log):
     libdutil.darshan_free(buf[0])
 
     return rec
+
+
+@functools.lru_cache()
+def log_get_derived_metrics(log_path: str, mod_name: str):
+    """
+    Returns the darshan_derived_metrics struct from CFFI/C accumulator code.
+
+    Parameters:
+        log_path: Path to the darshan log file
+        mod_name: The name of the module to retrieve derived metrics for
+
+    Returns:
+        darshan_derived_metrics struct (cdata object)
+    """
+    # TODO: eventually add support for i.e., a regex filter on the records
+    # the user wants to get derived metrics for--like filtering to records
+    # with a single filename involved before accumulating the data?
+    log_handle = log_open(log_path)
+    jobrec = ffi.new("struct darshan_job *")
+    libdutil.darshan_log_get_job(log_handle['handle'], jobrec)
+    modules = log_get_modules(log_handle)
+
+    if mod_name not in modules:
+        raise ValueError(f"{mod_name} is not in the available log file "
+                         f"modules: {modules.keys()}")
+
+    mod_type = _structdefs[mod_name]
+    darshan_accumulator = ffi.new("darshan_accumulator *")
+    r = libdutil.darshan_accumulator_create(modules[mod_name]['idx'],
+                                            jobrec[0].nprocs,
+                                            darshan_accumulator)
+    if r != 0:
+        raise RuntimeError("A nonzero exit code was received from "
+                           "darshan_accumulator_create() at the C level. "
+                           f"This could mean that the {mod_name} module does not "
+                           "support derived metric calculation, or that "
+                           "another kind of error occurred. It may be possible "
+                           "to retrieve additional information from the stderr "
+                           "stream.")
+
+    buf = ffi.new("void **")
+    r = 1
+    while r >= 1:
+        r = libdutil.darshan_log_get_record(log_handle['handle'], modules[mod_name]['idx'], buf)
+        if r < 1:
+            break
+        rbuf = ffi.cast(mod_type, buf)
+        r_i = libdutil.darshan_accumulator_inject(darshan_accumulator[0], rbuf[0], 1)
+        if r_i != 0:
+            libdutil.darshan_free(buf[0])
+            raise RuntimeError("A nonzero exit code was received from "
+                               "darshan_accumulator_inject() at the C level. "
+                               "It may be possible "
+                               "to retrieve additional information from the stderr "
+                               "stream.")
+    darshan_derived_metrics = ffi.new("struct darshan_derived_metrics *")
+    r = libdutil.darshan_accumulator_emit(darshan_accumulator[0],
+                                          darshan_derived_metrics,
+                                          rbuf[0])
+    libdutil.darshan_free(buf[0])
+    libdutil.darshan_accumulator_destroy(darshan_accumulator[0])
+    log_close(log_handle)
+    if r != 0:
+        raise RuntimeError("A nonzero exit code was received from "
+                           "darshan_accumulator_emit() at the C level. "
+                           "It may be possible "
+                           "to retrieve additional information from the stderr "
+                           "stream.")
+    return darshan_derived_metrics

darshan-util/pydarshan/darshan/cli/base.html

@@ -49,7 +49,8 @@ <h3>${fig_title}</h3>
               <figcaption>${fig.fig_description}</figcaption>
             % else:
               <!-- temporary handling for DXT-disabled cases -->
-              <figcaption style="font-weight: bold; color: red; width: 400px;">
+              <!-- now also handles the bandwidth text... -->
+              <figcaption style="font-weight: bold; color: ${fig.text_only_color}; width: 400px;">
                 ${fig.fig_description}
               </figcaption>
             % endif

darshan-util/pydarshan/darshan/cli/summary.py

@@ -17,6 +17,7 @@
 
 import darshan
 import darshan.cli
+from darshan.lib.accum import log_get_bytes_bandwidth
 from darshan.experimental.plots import (
     plot_dxt_heatmap,
     plot_io_cost,
@@ -56,6 +57,11 @@ def __init__(
         fig_args: dict,
         fig_description: str = "",
         fig_width: int = 500,
+        # when there is no HTML data generated
+        # for the figure (i.e., no image/plot),
+        # we have the option of changing the caption
+        # text color for a warning/important standalone text
+        text_only_color: str = "red",
     ):
         self.section_title = section_title
         if not fig_title:
@@ -68,7 +74,11 @@ def __init__(
         # temporary handling for DXT disabled cases
         # so special error message can be passed
         # in place of an encoded image
+        # NOTE: this code path is now also
+        # being used for adding the bandwidth
+        # text, which doesn't really have an image...
         self.fig_html = None
+        self.text_only_color = text_only_color
         if self.fig_func:
             self.generate_fig()
 
@@ -489,6 +499,30 @@ def register_figures(self):
                 )
                 self.figures.append(opcount_fig)
 
+            try:
+                # this is really just some text
+                # so using ReportFigure feels awkward...
+                bandwidth_fig = ReportFigure(
+                        section_title=sect_title,
+                        fig_title="",
+                        fig_func=None,
+                        fig_args=None,
+                        fig_description=log_get_bytes_bandwidth(log_path=self.log_path,
+                                                                mod_name=mod),
+                        text_only_color="blue")
+                self.figures.append(bandwidth_fig)
+            except (RuntimeError, KeyError):
+                # the module probably doesn't support derived metrics
+                # calculations, but the C code doesn't distinguish other
+                # types of errors
+
+                # the KeyError appears to be needed for a subset of logs
+                # for which _structdefs lacks APMPI or APXC entries;
+                # for example `e3sm_io_heatmap_only.darshan` in logs
+                # repo
+                pass
+
+
         #########################
         # Data Access by Category
         if not {"POSIX", "STDIO"}.isdisjoint(set(self.report.modules)):

darshan-util/pydarshan/darshan/lib/accum.py

@@ -0,0 +1,53 @@
+from darshan.backend.cffi_backend import log_get_derived_metrics
+
+
+def log_get_bytes_bandwidth(log_path: str, mod_name: str) -> str:
+    """
+    Summarize I/O performance for a given darshan module.
+
+    Parameters
+    ----------
+    log_path : str
+            Path to the darshan binary log file.
+    mod_name : str
+            Name of the darshan module to summarize the I/O
+            performance for.
+
+    Returns
+    -------
+    out: str
+        A short string summarizing the performance of the given module
+        in the provided log file, including bandwidth and total data
+        transferred.
+
+    Raises
+    ------
+    RuntimeError
+        When a provided module name is not supported for the accumulator
+        interface for provision of the summary data, or for any other
+        error that occurs in the C/CFFI interface.
+    ValueError
+        When a provided module name does not exist in the log file.
+
+    Examples
+    --------
+
+    >>> from darshan.log_utils import get_log_path
+    >>> from darshan.lib.accum import log_get_bytes_bandwidth
+
+    >>> log_path = get_log_path("imbalanced-io.darshan")
+    >>> log_get_bytes_bandwidth(log_path, "POSIX")
+    I/O performance estimate (at the POSIX layer): transferred 101785.8 MiB at 164.99 MiB/s
+
+    >>> log_get_bytes_bandwidth(log_path, "MPI-IO")
+    I/O performance estimate (at the MPI-IO layer): transferred 126326.8 MiB at 101.58 MiB/s
+    """
+    # get total bytes (in MiB) and bandwidth (in MiB/s) for
+    # a given module -- this information was commonly reported
+    # in the old perl-based summary reports
+    darshan_derived_metrics = log_get_derived_metrics(log_path=log_path,
+                                                      mod_name=mod_name)
+    total_mib = darshan_derived_metrics.total_bytes / 2 ** 20
+    total_bw = darshan_derived_metrics.agg_perf_by_slowest
+    ret_str = f"I/O performance estimate (at the {mod_name} layer): transferred {total_mib:.1f} MiB at {total_bw:.2f} MiB/s"
+    return ret_str

darshan-util/pydarshan/darshan/tests/test_lib_accum.py

@@ -0,0 +1,93 @@
+from darshan.lib.accum import log_get_bytes_bandwidth
+from darshan.log_utils import get_log_path
+
+import pytest
+
+
+@pytest.mark.parametrize("log_path, mod_name, expected_str", [
+    # the expected bytes/bandwidth strings are pasted
+    # directly from the old perl summary reports;
+    # exceptions noted below
+    # in some cases we defer to darshan-parser for the expected
+    # values; see discussion in gh-839
+    ("imbalanced-io.darshan",
+     "STDIO",
+     "I/O performance estimate (at the STDIO layer): transferred 1.1 MiB at 0.01 MiB/s"),
+    ("imbalanced-io.darshan",
+     "MPI-IO",
+     "I/O performance estimate (at the MPI-IO layer): transferred 126326.8 MiB at 101.58 MiB/s"),
+    # imbalanced-io.darshan does have LUSTRE data,
+    # but it doesn't support derived metrics at time
+    # of writing
+    ("imbalanced-io.darshan",
+     "LUSTRE",
+     "RuntimeError"),
+    ("imbalanced-io.darshan",
+     "POSIX",
+     "I/O performance estimate (at the POSIX layer): transferred 101785.8 MiB at 164.99 MiB/s"),
+    ("laytonjb_test1_id28730_6-7-43012-2131301613401632697_1.darshan",
+     "STDIO",
+     "I/O performance estimate (at the STDIO layer): transferred 0.0 MiB at 4.22 MiB/s"),
+    ("runtime_and_dxt_heatmaps_diagonal_write_only.darshan",
+     "POSIX",
+     "I/O performance estimate (at the POSIX layer): transferred 0.0 MiB at 0.02 MiB/s"),
+    ("treddy_mpi-io-test_id4373053_6-2-60198-9815401321915095332_1.darshan",
+     "STDIO",
+     "I/O performance estimate (at the STDIO layer): transferred 0.0 MiB at 16.47 MiB/s"),
+    ("e3sm_io_heatmap_only.darshan",
+     "STDIO",
+     "I/O performance estimate (at the STDIO layer): transferred 0.0 MiB at 3.26 MiB/s"),
+    ("e3sm_io_heatmap_only.darshan",
+     "MPI-IO",
+     "I/O performance estimate (at the MPI-IO layer): transferred 73880.2 MiB at 105.69 MiB/s"),
+    ("partial_data_stdio.darshan",
+     "MPI-IO",
+     "I/O performance estimate (at the MPI-IO layer): transferred 32.0 MiB at 2317.98 MiB/s"),
+    ("partial_data_stdio.darshan",
+     "STDIO",
+     "I/O performance estimate (at the STDIO layer): transferred 16336.0 MiB at 2999.14 MiB/s"),
+    # the C derived metrics code can't distinguish
+    # between different kinds of errors at this time,
+    # but we can still intercept in some cases...
+    ("partial_data_stdio.darshan",
+     "GARBAGE",
+     "ValueError"),
+    # TODO: determine if the lack of APMPI and
+    # any other "add-ons" in _structdefs is a bug
+    # in the control flow for `log_get_derived_metrics()`?
+    ("e3sm_io_heatmap_only.darshan",
+     "APMPI",
+     "KeyError"),
+    ("skew-app.darshan",
+     "POSIX",
+     "I/O performance estimate (at the POSIX layer): transferred 41615.8 MiB at 157.49 MiB/s"),
+    ("skew-app.darshan",
+     "MPI-IO",
+     "I/O performance estimate (at the MPI-IO layer): transferred 41615.8 MiB at 55.22 MiB/s"),
+])
+def test_derived_metrics_bytes_and_bandwidth(log_path, mod_name, expected_str):
+    # test the basic scenario of retrieving
+    # the total data transferred and bandwidth
+    # for all records in a given module; the situation
+    # of accumulating drived metrics with filtering
+    # (i.e., for a single filename) is not tested here
+
+    log_path = get_log_path(log_path)
+    if expected_str == "RuntimeError":
+        with pytest.raises(RuntimeError,
+                           match=f"{mod_name} module does not support derived"):
+            log_get_bytes_bandwidth(log_path=log_path,
+                                    mod_name=mod_name)
+    elif expected_str == "ValueError":
+        with pytest.raises(ValueError,
+                           match=f"{mod_name} is not in the available log"):
+            log_get_bytes_bandwidth(log_path=log_path,
+                                    mod_name=mod_name)
+    elif expected_str == "KeyError":
+        with pytest.raises(KeyError, match=f"{mod_name}"):
+            log_get_bytes_bandwidth(log_path=log_path,
+                                    mod_name=mod_name)
+    else:
+        actual_str = log_get_bytes_bandwidth(log_path=log_path,
+                                             mod_name=mod_name)
+        assert actual_str == expected_str

darshan-util/pydarshan/darshan/tests/test_summary.py

@@ -236,6 +236,12 @@ def test_main_all_logs_repo_files(tmpdir, log_filepath):
                     else:
                         assert actual_runtime_heatmap_titles == 0
 
+                    # check for presence of bandwidth summary strings
+                    # (more detailed per-module probes are present
+                    # in test_derived_metrics_bytes_and_bandwidth())
+                    assert "I/O performance estimate" in report_str
+                    assert "color: blue" in report_str
+
 
 class TestReportData: