Updated physical decoding logic to always group by data length before processing

Author: Martin

examples/data-processing/README.md

@@ -44,7 +44,6 @@ To use the Transport Protocol functionality in other scripts, you need to make m
 1. Ensure that you import the `MultiFrameDecoder` class from `utils.py` 
 2. Specify the type via the `tp_type` variable e.g. to `j1939` 
 3. After you've extract the normal raw dataframe, parse it to the `tp.combine_tp_frames` function as below
-4. Ensure you include `tp_type` in the `extract_phys` function
 
 See below example:
 
@@ -53,7 +52,6 @@ tp_type = "j1939"
 df_raw, device_id = proc.get_raw_data(log_file)
 tp = MultiFrameDecoder(tp_type)
 df_raw = tp.combine_tp_frames(df_raw)
-df_phys = proc.extract_phys(df_raw, tp_type=tp_type)
 ```
 
 

examples/data-processing/utils.py

@@ -57,12 +57,10 @@ def list_log_files(fs, devices, start_times, verbose=True):
             if len(log_files_device) > 0:
                 with fs.open(log_files_device[0], "rb") as handle:
                     mdf_file = mdf_iter.MdfFile(handle)
-
                     df_raw_lin = mdf_file.get_data_frame_lin()
                     df_raw_lin["IDE"] = 0
                     df_raw_can = mdf_file.get_data_frame()
                     df_raw = df_raw_can.append(df_raw_lin)
-
                     end_time = df_raw.index[-1]
 
                 if end_time < start:
@@ -129,7 +127,7 @@ def __init__(self, fs, db_list, signals=[], days_offset=None, verbose=True):
         self.verbose = verbose
         return
 
-    def extract_phys(self, df_raw, tp_type=None):
+    def extract_phys(self, df_raw):
         """Given df of raw data and list of decoding databases, create new def with
         physical values (no duplicate signals and optionally filtered/rebaselined)
         """
@@ -140,15 +138,12 @@ def extract_phys(self, df_raw, tp_type=None):
         for db in self.db_list:
             df_decoder = can_decoder.DataFrameDecoder(db)
 
-            if tp_type != None:
-                df_phys_tp = pd.DataFrame()
-                for length, group in df_raw.groupby("DataLength"):
-                    df_phys_group = df_decoder.decode_frame(group)
-                    df_phys_tp = df_phys_tp.append(df_phys_group)
+            df_phys_temp = pd.DataFrame()
+            for length, group in df_raw.groupby("DataLength"):
+                df_phys_group = df_decoder.decode_frame(group)
+                df_phys_temp = df_phys_temp.append(df_phys_group)
 
-                df_phys = df_phys.append(df_phys_tp.sort_index())
-            else:
-                df_phys = df_phys.append(df_decoder.decode_frame(df_raw))
+            df_phys = df_phys.append(df_phys_temp.sort_index())
 
         # remove duplicates in case multiple DBC files contain identical signals
         df_phys["datetime"] = df_phys.index
@@ -157,20 +152,21 @@ def extract_phys(self, df_raw, tp_type=None):
 
         # optionally filter and rebaseline the data
         df_phys = self.filter_signals(df_phys)
-        df_phys = self.rebaseline_data(df_phys)
+
+        if not df_phys.empty and type(self.days_offset) == int:
+            df_phys = self.rebaseline_data(df_phys)
 
         return df_phys
 
     def rebaseline_data(self, df_phys):
         """Given a df of physical values, this offsets the timestamp
         to be equal to today, minus a given number of days.
         """
-        if not df_phys.empty and type(self.days_offset) == int:
-            from datetime import datetime, timezone
-            import pandas as pd
+        from datetime import datetime, timezone
+        import pandas as pd
 
-            delta_days = (datetime.now(timezone.utc) - df_phys.index.min()).days - self.days_offset
-            df_phys.index = df_phys.index + pd.Timedelta(delta_days, "day")
+        delta_days = (datetime.now(timezone.utc) - df_phys.index.min()).days - self.days_offset
+        df_phys.index = df_phys.index + pd.Timedelta(delta_days, "day")
 
         return df_phys