WIP: Python derived/accum interface

* early draft of Python/CFFI interface to
derived metrics/accumulators described in:
- gh-642
- gh-677

* for now this definitely doesn't work, and feels like I'm basically
reconstituting a C control flow in CFFI/Python instead of using a sensible
exposure point between C and Python to pull out populated structs from
a single entry point

* perhaps folks can just help me sort out the current issues noted
in the source changes rather than providing a convenient API, though
once thorough regression tests are in place that might be something
to consider in the future... (or even just maintaining it in
`pandas`/Python someday if the perf is ~similar)

Author: tylerjereddy

darshan-util/pydarshan/darshan/backend/api_def_c.py

@@ -8,6 +8,34 @@
 
 
 header = """/* from darshan-logutils.h */
+
+struct darshan_derived_metrics {
+    int64_t total_bytes;
+    double unique_io_total_time_by_slowest;
+    double unique_rw_only_time_by_slowest;
+    double unique_md_only_time_by_slowest;
+    int unique_io_slowest_rank;
+    double shared_io_total_time_by_slowest;
+    double agg_perf_by_slowest;
+    double agg_time_by_slowest;
+    struct darshan_file_category_counters;
+};
+
+
+
+struct darshan_accumulator {
+    int64_t module_id;
+    int64_t job_nprocs;
+    void* agg_record;
+    int num_records;
+    void *file_hash_table;
+    double shared_io_total_time_by_slowest;
+    int64_t total_bytes;
+    double *rank_cumul_io_total_time;
+    double *rank_cumul_rw_only_time;
+    double *rank_cumul_md_only_time;
+};
+
 struct darshan_mnt_info
 {
     char mnt_type[3031];
@@ -23,6 +51,11 @@
     int partial_flag;
 };
 
+int darshan_accumulator_emit(struct darshan_accumulator, struct darshan_derived_metrics*, void* aggregation_record);
+int darshan_accumulator_destroy(struct darshan_accumulator);
+int darshan_accumulator_create(enum darshan_module_id, int64_t, struct darshan_accumulator*);
+int darshan_accumulator_inject(struct darshan_accumulator, void*, int);
+
 /* from darshan-log-format.h */
 typedef uint64_t darshan_record_id;
 

darshan-util/pydarshan/darshan/backend/cffi_backend.py

@@ -650,3 +650,28 @@ def _log_get_heatmap_record(log):
     libdutil.darshan_free(buf[0])
 
     return rec
+
+
+def log_get_accumulator(log, mod_name: str):
+    log = log_open(log)
+    jobrec = ffi.new("struct darshan_job *")
+    libdutil.darshan_log_get_job(log['handle'], jobrec)
+    modules = log_get_modules(log)
+
+    if mod_name not in modules:
+        return None
+    mod_type = _structdefs[mod_name]
+
+    darshan_accumulator = ffi.new("struct darshan_accumulator *")
+    buf = ffi.new("void **")
+    r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
+    rbuf = ffi.cast(mod_type, buf)
+
+    libdutil.darshan_accumulator_create(modules[mod_name]['idx'],
+                                        jobrec[0].nprocs,
+                                        darshan_accumulator)
+
+    # TODO: fix the segfault on the inject call below
+    r = libdutil.darshan_accumulator_inject(darshan_accumulator[0], rbuf[0], 1)
+    # TODO: darshan_accumulator_emit and darshan_accumulator_destroy
+    return darshan_accumulator

darshan-util/pydarshan/darshan/tests/test_cffi_misc.py

@@ -159,3 +159,15 @@ def test_log_get_generic_record(dtype):
         # make sure the returned key/column names agree
         assert actual_counter_names == expected_counter_names
         assert actual_fcounter_names == expected_fcounter_names
+
+
+@pytest.mark.parametrize("log_path", [
+    "imbalanced-io.darshan",
+])
+def test_accumulator_emit(log_path):
+    log_path = get_log_path(log_path)
+    report = darshan.DarshanReport(log_path, read_all=True)
+    for mod_name in report.modules:
+        acc = backend.log_get_accumulator(log=log_path,
+                                          mod_name=mod_name)
+        # TODO: assert against values from i.e., perl reports