Merge pull request #56 from ooipy/bug-fixes

John-Ragland · web-flow · commit 24d9b673b814 · 2021-04-12T14:09:52.000-07:00
Adding all bug fixes for 1.1.1 release
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -0,0 +1,14 @@
+
+name: Lint code
+
+on: [pull_request, push]
+
+jobs:
+  lint-code:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/action@v2.0.2
+      # allow to error for now
+      continue-on-error: true
diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
@@ -4,40 +4,55 @@ on: [push, pull_request]
 
 jobs:
   unit_test:
-    runs-on: ubuntu-latest
+    name: ${{ matrix.python-version }}-unit-test
+    runs-on: ubuntu-20.04
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+        experimental: [false]
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+    - name: Cache conda
+      uses: actions/cache@v2
+      env:
+        # Increase this value to reset cache if conda_envs/py${{ matrix.python-version }}.yml has not changed
+        CACHE_NUMBER: 0
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda_envs/py${{ matrix.python-version }}.yml') }}
     - name: Install conda
       uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: ooi
-        environment-file: conda_envs/ooi_linux.yml
-        auto-update-conda: true
-        python-version: 3.8
-        miniconda-version: "latest"
+        environment-file: conda_envs/py${{ matrix.python-version }}.yml
+        python-version: ${{ matrix.python-version }}
         auto-activate-base: false
-    - shell: bash -l {0}
+        use-only-tar-bz2: true
+    - name: Print conda env
+      shell: bash -l {0}
       run: |
         conda info
         conda list
-    - name: install pytest
+    - name: Install pytest
       shell: bash -l {0}
-      run: conda install --yes pytest
-    - name: install ooipy 
+      run: mamba install --yes pytest
+    - name: Install ooipy 
       shell: bash -l {0}
       run: pip install .
-    - name: run unit tests
+    - name: Run unit tests
       shell: bash -l {0}
       env:
         FORCE_COLOR: 3
-      run: pytest -vv tests
-    - name: save unit tests in file
-      if: ${{ always() }}
-      shell: bash -l {0}
-      run: py.test tests &> unit_test_log.log
+      run: pytest -vv tests |& tee unit_test_log.log
     - name: Upload unit test log
-      if: ${{ always() }}
+      if: ${{ success() || failure() }}
       uses: actions/upload-artifact@v2
       with:
         name: unit_test_log
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,45 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.1.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-docstring-first
+      - id: check-json
+      - id: check-yaml
+      - id: pretty-format-json
+        args: ["--autofix", "--indent=2", "--no-sort-keys"]
+
+  - repo: https://github.com/ambv/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+        args: ["--line-length", "100"]
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
+        args:
+          [
+            "--ignore",
+            "E999,N802,N806,W503,W504",
+            "--exclude=setup.py,tests,__init__.py,ooipy/__init__.py,ooipy/request/__init__.py,ooipy/hydrophone/__init__.py,,ooipy/ctd/__init__.py,ooipy/surface_buoy/__init__.py,ooipy/visualize/__init__.py,docs",
+            "--tee",
+            "--output-file=flake8_log.txt",
+          ]
+
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
+    hooks:
+      - id: seed-isort-config
+
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.2.0
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/myint/rstcheck
+    rev: master
+    hooks:
+      - id: rstcheck
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,4 @@
 include README.md
 include LICENSE
 include *.txt
+include ooipy/hydrophone/calibration_by_assetID.csv
diff --git a/conda_envs/py3.7.yml b/conda_envs/py3.7.yml
@@ -0,0 +1,11 @@
+name: ooi
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - obspy
+  - fsspec
+  - aiohttp
+  - pandas
+  - mamba
diff --git a/conda_envs/py3.8.yml b/conda_envs/py3.8.yml
@@ -0,0 +1,11 @@
+name: ooi
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.8
+  - obspy
+  - fsspec
+  - aiohttp
+  - pandas
+  - mamba
diff --git a/features_since_release.txt b/features_since_release.txt
@@ -14,3 +14,7 @@ Updates in resealse v1.1.0
 - added request/ctd_request.py module
 - added ctd/basic.py module for extracting CTD data and computing parameter profiles over the water column
 
+Updates in release v1.1.1
+-------------------------
+
+- bug fixes in multiprocessing functions and hydrophone sensitivity correction
diff --git a/ooipy/hydrophone/basic.py b/ooipy/hydrophone/basic.py
@@ -82,35 +82,26 @@ def frequency_calibration(self, N):
         output_array : np.array
             array with correction coefficient for every frequency
         """
-        # Calibration for Low Frequency Hydrophones
+        # Load calibation file and get appropriate calibration info
+        filename = os.path.dirname(ooipy.__file__) + \
+            '/hydrophone/calibration_by_assetID.csv'
+        # Use deployment CSV to determine asset_ID
+        assetID = self.get_asset_ID()
+
+        # load calibration data as pandas dataframe
+        cal_by_assetID = pd.read_csv(filename, header=[0, 1])
+
+        f_calib = cal_by_assetID[assetID]['Freq (kHz)'].to_numpy() * 1000
+        sens_calib_0 = cal_by_assetID[assetID]['0 phase'].to_numpy()
+        sens_calib_90 = cal_by_assetID[assetID]['90 phase'].to_numpy()
+        sens_calib = 0.5 * (sens_calib_0 + sens_calib_90)
+        f = np.linspace(0, round(self.stats.sampling_rate / 2), N)
+
+        # Convert calibration to correct units
         if round(self.stats.sampling_rate) == 200:
-            lf_cal = {
-                'HYSB1': 2311.11, 'HYS14': 2499.09, 'AXBA1': 2257.3,
-                'AXEC2': 2421.0, 'AXCC1': 2480.98}
-            f = np.linspace(0, 100, N)
-            f_calib = np.array([0, 100])
-            sens_calib_db = 20 * np.log10(lf_cal[self.stats.station] * 1e-6)
-            sens_calib = np.array([sens_calib_db, sens_calib_db])
-
-        # Calibratino for Broadband Hydrophones
+            sens_calib = 20 * np.log10(sens_calib * 1e-6)
         elif round(self.stats.sampling_rate) == 64000:
-            filename = os.path.dirname(ooipy.__file__) + \
-                '/hydrophone/calibration_by_assetID.pkl'
-            # Use deployment CSV to determine asset_ID
-            assetID = self.get_asset_ID()
-
-            # load calibration data as pandas dataframe
-            with open(filename, 'rb') as f:
-                cal_by_assetID = pickle.load(f)
-
-            f_calib = cal_by_assetID[assetID]['Freq (kHz)'].to_numpy() * 1000
-            sens_calib_0 = cal_by_assetID[assetID]['0 phase'].to_numpy()
-            sens_calib_90 = cal_by_assetID[assetID]['90 phase'].to_numpy()
-            # Average 0 and 90 degree phase response
-            sens_calib = 0.5 * (sens_calib_0 + sens_calib_90)
             sens_calib = sens_calib + 128.9
-            f = np.linspace(0, 32000, N)
-
         else:
             raise Exception('Invalid sampling rate')
 
@@ -306,14 +297,16 @@ def compute_spectrogram_mp(self, n_process=None, win='hann', L=4096,
             temp_slice = self.slice(starttime=UTCDateTime(starttime),
                                     endtime=UTCDateTime(endtime))
             tmp_obj = HydrophoneData(data=temp_slice.data,
-                                     header=temp_slice.stats)
+                                     header=temp_slice.stats,
+                                     node=self.stats.location)
             ooi_hyd_data_list.append((tmp_obj, win, L, avg_time, overlap))
 
         starttime = self.stats.starttime + datetime.timedelta(
             seconds=(N - 1) * seconds_per_process)
         temp_slice = self.slice(starttime=UTCDateTime(starttime),
                                 endtime=UTCDateTime(self.stats.endtime))
-        tmp_obj = HydrophoneData(data=temp_slice.data, header=temp_slice.stats)
+        tmp_obj = HydrophoneData(data=temp_slice.data, header=temp_slice.stats,
+                                 node=self.stats.location)
         ooi_hyd_data_list.append((tmp_obj, win, L, avg_time, overlap))
 
         with mp.get_context("spawn").Pool(n_process) as p:
@@ -488,7 +481,8 @@ def compute_psd_welch_mp(self, split, n_process=None, win='hann', L=4096,
                 temp_slice = self.slice(starttime=UTCDateTime(starttime),
                                         endtime=UTCDateTime(endtime))
                 tmp_obj = HydrophoneData(data=temp_slice.data,
-                                         header=temp_slice.stats)
+                                         header=temp_slice.stats,
+                                         node=self.stats.location)
                 ooi_hyd_data_list.append(
                     (tmp_obj, win, L, overlap, avg_method, interpolate, scale))
 
@@ -499,7 +493,8 @@ def compute_psd_welch_mp(self, split, n_process=None, win='hann', L=4096,
             temp_slice = self.slice(starttime=UTCDateTime(starttime),
                                     endtime=UTCDateTime(self.stats.endtime))
             tmp_obj = HydrophoneData(data=temp_slice.data,
-                                     header=temp_slice.stats)
+                                     header=temp_slice.stats,
+                                     node=self.stats.location)
             ooi_hyd_data_list.append(
                 (tmp_obj, win, L, overlap, avg_method, interpolate, scale))
         # use segmentation specified by split
@@ -509,7 +504,8 @@ def compute_psd_welch_mp(self, split, n_process=None, win='hann', L=4096,
                 temp_slice = self.slice(starttime=UTCDateTime(row[0]),
                                         endtime=UTCDateTime(row[1]))
                 tmp_obj = HydrophoneData(data=temp_slice.data,
-                                         header=temp_slice.stats)
+                                         header=temp_slice.stats,
+                                         node=self.stats.location)
                 ooi_hyd_data_list.append(
                     (tmp_obj, win, L, overlap, avg_method, interpolate, scale))
 
@@ -570,38 +566,61 @@ def wav_write(self, filename, norm=False, new_sample_rate=None):
         wavfile.write(filename, int(sampling_rate), data)
 
     def get_asset_ID(self):
-        url = 'https://raw.githubusercontent.com/OOI-CabledArray/' \
-            'deployments/main/HYDBBA_deployments.csv'
-        hyd_df = pd.read_csv(url)
-
-        if self.stats.location == 'LJ01D':  # LJ01D'Oregon Shelf Base Seafloor
-            ref = 'CE02SHBP-LJ01D-11-HYDBBA106'
-        if self.stats.location == 'LJ01A':  # LJ01AOregon Slope Base Seafloor
-            ref = 'RS01SLBS-LJ01A-09-HYDBBA102'
-        if self.stats.location == 'PC01A':  # Oregan Slope Base Shallow
-            ref = 'RS01SBPS-PC01A-08-HYDBBA103'
-        if self.stats.location == 'PC03A':  # Axial Base Shallow Profiler
-            ref = 'RS03AXPS-PC03A-08-HYDBBA303'
-        if self.stats.location == 'LJ01C':  # Oregon Offshore Base Seafloor
-            ref = 'CE04OSBP-LJ01C-11-HYDBBA105'
-        if self.stats.location == 'LJ03A':  # Axial Base Seafloor
-            ref = 'RS03AXBS-LJ03A-09-HYDBBA302'
-
-        hyd_df['referenceDesignator']
-
-        df_ref = hyd_df.loc[hyd_df['referenceDesignator'] == ref]
-
-        df_start = df_ref.loc[(df_ref['startTime'] < self.stats.starttime) &
-                              (df_ref['endTime'] > self.stats.starttime)]
-        df_end = df_ref.loc[(df_ref['startTime'] < self.stats.endtime) &
-                            (df_ref['endTime'] > self.stats.endtime)]
-
-        if df_start.index.to_numpy() == df_end.index.to_numpy():
-            idx = df_start.index.to_numpy()
-            asset_ID = df_start['assetID'][int(idx)]
+        '''
+        get_asset_ID returns the hydrophone asset ID for a given data sample.
+        This data can be foun `here <https://raw.githubusercontent.com/
+        OOI-CabledArray/deployments/main/HYDBBA_deployments.csv'>`_ for
+        broadband hydrophones. Since Low frequency hydrophones remain
+        constant with location and time, if the hydrophone is low frequency,
+        the node ID is returned
+        '''
+        # Low frequency hydrophone
+        if round(self.stats.sampling_rate) == 200:
+            asset_ID = self.stats.location
+
+        elif round(self.stats.sampling_rate) == 64000:
+            url = 'https://raw.githubusercontent.com/OOI-CabledArray/' \
+                'deployments/main/HYDBBA_deployments.csv'
+            hyd_df = pd.read_csv(url)
+
+            # LJ01D'Oregon Shelf Base Seafloor
+            if self.stats.location == 'LJ01D':
+                ref = 'CE02SHBP-LJ01D-11-HYDBBA106'
+            # LJ01AOregon Slope Base Seafloor
+            if self.stats.location == 'LJ01A':
+                ref = 'RS01SLBS-LJ01A-09-HYDBBA102'
+            # Oregan Slope Base Shallow
+            if self.stats.location == 'PC01A':
+                ref = 'RS01SBPS-PC01A-08-HYDBBA103'
+            # Axial Base Shallow Profiler
+            if self.stats.location == 'PC03A':
+                ref = 'RS03AXPS-PC03A-08-HYDBBA303'
+            # Oregon Offshore Base Seafloor
+            if self.stats.location == 'LJ01C':
+                ref = 'CE04OSBP-LJ01C-11-HYDBBA105'
+            # Axial Base Seafloor
+            if self.stats.location == 'LJ03A':
+                ref = 'RS03AXBS-LJ03A-09-HYDBBA302'
+
+            hyd_df['referenceDesignator']
+
+            df_ref = hyd_df.loc[hyd_df['referenceDesignator'] == ref]
+
+            df_start = df_ref.loc[(df_ref['startTime'] < self.stats.starttime)
+                                  & (df_ref['endTime']
+                                  > self.stats.starttime)]
+            df_end = df_ref.loc[(df_ref['startTime'] < self.stats.endtime) &
+                                (df_ref['endTime'] > self.stats.endtime)]
+
+            if df_start.index.to_numpy() == df_end.index.to_numpy():
+                idx = df_start.index.to_numpy()
+                asset_ID = df_start['assetID'][int(idx)]
+            else:
+                raise Exception('Hydrophone Data involves multiple'
+                                'deployments. Feature to be added later')
         else:
-            raise Exception('Hydrophone Data involves multiple deployments.'
-                            'Feature to be added later')
+            raise Exception('Invalid hydrophone sampling rate')
+
         return asset_ID
 
 
diff --git a/ooipy/hydrophone/calibration_by_assetID.csv b/ooipy/hydrophone/calibration_by_assetID.csv
diff --git a/ooipy/hydrophone/calibration_by_assetID.pkl b/ooipy/hydrophone/calibration_by_assetID.pkl
diff --git a/ooipy/request/hydrophone_request.py b/ooipy/request/hydrophone_request.py
diff --git a/requirements.txt b/requirements.txt
