MAINT: PR 839 revisions

* needed to expose `darshan_file_category_counters` so that its
size is available for usage in `darshan_derived_metrics` struct

* some improvements to the renamed `log_get_derived_metrics()` function,
including first draft of error handling; the renamed
`test_derived_metrics_basic` test was also adjusted to expect an
error for `LUSTRE` module

Author: tylerjereddy

darshan-util/pydarshan/darshan/backend/api_def_c.py

@@ -9,6 +9,17 @@
 
 header = """/* from darshan-logutils.h */
 
+struct darshan_file_category_counters {
+    int64_t count;                   /* number of files in this category */
+    int64_t total_read_volume_bytes; /* total read traffic volume */
+    int64_t total_write_volume_bytes;/* total write traffic volume */
+    int64_t max_read_volume_bytes;   /* maximum read traffic volume to 1 file */
+    int64_t max_write_volume_bytes;  /* maximum write traffic volume to 1 file */
+    int64_t total_max_offset_bytes;  /* summation of max_offsets */
+    int64_t max_offset_bytes;        /* largest max_offset */
+    int64_t nprocs;                  /* how many procs accessed (-1 for "all") */
+};
+
 struct darshan_derived_metrics {
     int64_t total_bytes;
     double unique_io_total_time_by_slowest;

darshan-util/pydarshan/darshan/backend/cffi_backend.py

@@ -652,26 +652,68 @@ def _log_get_heatmap_record(log):
     return rec
 
 
-def log_get_accumulator(log, mod_name: str):
-    log = log_open(log)
+def log_get_derived_metrics(log_path: str, mod_name: str):
+    """
+    Returns the darshan_derived_metrics struct from CFFI/C accumulator code.
+
+    Parameters:
+        log_path: Path to the darshan log file
+        mod_name: The name of the module to retrieve derived metrics for
+
+    Returns:
+        darshan_derived_metrics struct (cdata object)
+    """
+    # TODO: eventually add support for i.e., a regex filter on the records
+    # the user wants to get derived metrics for--like filtering to records
+    # with a single filename involved before accumulating the data?
+    log_handle = log_open(log_path)
     jobrec = ffi.new("struct darshan_job *")
-    libdutil.darshan_log_get_job(log['handle'], jobrec)
-    modules = log_get_modules(log)
+    libdutil.darshan_log_get_job(log_handle['handle'], jobrec)
+    modules = log_get_modules(log_handle)
 
     if mod_name not in modules:
         return None
     mod_type = _structdefs[mod_name]
 
     buf = ffi.new("void **")
-    r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
+    r = libdutil.darshan_log_get_record(log_handle['handle'], modules[mod_name]['idx'], buf)
     rbuf = ffi.cast(mod_type, buf)
 
     darshan_accumulator = ffi.new("darshan_accumulator *")
+    print("before create")
     r = libdutil.darshan_accumulator_create(modules[mod_name]['idx'],
                                             jobrec[0].nprocs,
                                             darshan_accumulator)
-
-    # TODO: fix the segfault on the inject call below
+    if r != 0:
+        raise RuntimeError("A nonzero exit code was received from "
+                           "darshan_accumulator_create() at the C level. "
+                           f"This could mean that the {mod_name} module does not "
+                           "support derived metric calculation, or that "
+                           "another kind of error occurred. It may be possible "
+                           "to retrieve additional information from the stderr "
+                           "stream.")
+    print("after create")
+
+    print("before inject")
     r = libdutil.darshan_accumulator_inject(darshan_accumulator[0], rbuf[0], 1)
-    # TODO: darshan_accumulator_emit and darshan_accumulator_destroy
-    return darshan_accumulator
+    if r != 0:
+        raise RuntimeError("A nonzero exit code was received from "
+                           "darshan_accumulator_inject() at the C level. "
+                           "It may be possible "
+                           "to retrieve additional information from the stderr "
+                           "stream.")
+    print("after inject")
+    darshan_derived_metrics = ffi.new("struct darshan_derived_metrics *")
+    print("before emit")
+    r = libdutil.darshan_accumulator_emit(darshan_accumulator[0],
+                                          darshan_derived_metrics,
+                                          rbuf[0])
+    if r != 0:
+        raise RuntimeError("A nonzero exit code was received from "
+                           "darshan_accumulator_emit() at the C level. "
+                           "It may be possible "
+                           "to retrieve additional information from the stderr "
+                           "stream.")
+    print("after emit")
+    #libdutil.darshan_accumulator_destroy(darshan_accumulator)
+    return darshan_derived_metrics

darshan-util/pydarshan/darshan/tests/test_cffi_misc.py

@@ -164,10 +164,26 @@ def test_log_get_generic_record(dtype):
 @pytest.mark.parametrize("log_path", [
     "imbalanced-io.darshan",
 ])
-def test_accumulator_emit(log_path):
+def test_derived_metrics_basic(log_path):
+    # test the basic scenario of retrieving
+    # the derived metrics from all records for a given
+    # module; the situation where you'd like to
+    # retrieve derived metrics for a subset of records (i.e.,
+    # a particular filename) is not tested here
     log_path = get_log_path(log_path)
     report = darshan.DarshanReport(log_path, read_all=True)
     for mod_name in report.modules:
-        acc = backend.log_get_accumulator(log=log_path,
-                                          mod_name=mod_name)
+        # if support is added for accumulator work on these
+        # modules later on, the test will fail to raise an error,
+        # causing the test to ultimately fail; that is good, it will
+        # force us to acknowledge that the support was added intentionally
+        # under the hood
+        print("testing mod_name:", mod_name)
+        if mod_name in {"LUSTRE"}:
+            with pytest.raises(RuntimeError):
+                derived_metrics = backend.log_get_derived_metrics(log_path=log_path,
+                                                  mod_name=mod_name)
+        else:
+            derived_metrics = backend.log_get_derived_metrics(log_path=log_path,
+                                              mod_name=mod_name)
         # TODO: assert against values from i.e., perl reports