replace real/imag stats with power/phase stats (#150)

* replace stats real/imag with power/phase
* add pytest-order to force test CSLC HDF5 to be generated 1st
* update specifile.txt with pytest-order
Co-authored-by: Seongsu Jeong <[email protected]>

Author: LiangJYu

docker/specfile.txt

@@ -3,7 +3,7 @@
 # platform: linux-64
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.5.7-hbcca054_0.conda
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
@@ -61,7 +61,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.b
 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.0-h0b41bf4_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.0-hd590300_3.conda
 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-h9fff704_0.conda
@@ -122,7 +122,7 @@ https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_8.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.5.2-h0b41bf4_0.conda
 https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/certifi-2023.5.7-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/click-8.1.3-unix_pyhd8ed1ab_2.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/cloudpickle-2.2.1-pyhd8ed1ab_0.conda
@@ -219,6 +219,7 @@ https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.1.1-pyhd8ed1ab_0.co
 https://conda.anaconda.org/conda-forge/linux-64/poppler-23.03.0-hf052cbe_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h945b3ca_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py311hbbb8f27_3.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pytest-order-1.0.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda
 https://conda.anaconda.org/conda-forge/noarch/geopandas-base-0.12.2-pyha770c72_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.3-h93ed92d_1.conda
@@ -242,4 +243,4 @@ https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.2.2-py311h67c5ca5
 https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/pysolid-0.2.3-py311ha29c927_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/geopandas-0.12.2-pyhd8ed1ab_0.conda
-https://conda.anaconda.org/conda-forge/noarch/dem_stitcher-2.4.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/dem_stitcher-2.4.0-pyhd8ed1ab_0.conda

requirements.txt

@@ -8,6 +8,7 @@ pandas
 pyproj
 pysolid
 pytest
+pytest-order
 ruamel.yaml
 scipy
 yamale

src/compass/s1_cslc_qa.py

@@ -11,6 +11,19 @@
                                       QA_PATH, add_dataset_and_attrs, Meta)
 
 
+def _compute_slc_array_stats(arr: np.ndarray, pwr_phase: str):
+    # internal to function to compute min, max, mean, and std dev of power or
+    # phase of SLC array. Default to phase stat computation.
+    if pwr_phase == 'power':
+        post_op_arr = np.abs(arr)**2
+    else:
+        post_op_arr = np.angle(arr)
+
+    return [np_op(post_op_arr)
+            for np_op in [np.nanmean, np.nanmin, np.nanmax,
+                                      np.nanstd]]
+
+
 def value_description_dict(val, desc):
     '''
     Convenience function that returns dict with description and value
@@ -64,40 +77,32 @@ def compute_CSLC_raster_stats(self, cslc_h5py_root, bursts):
 
             # get dataset and compute stats according to dtype
             pol_path = f'{DATA_PATH}/{pol}'
-            pol_ds = cslc_h5py_root[pol_path]
-
-            # compute stats for real and complex
-            stat_obj = isce3.math.StatsRealImagFloat32(pol_ds[()])
+            pol_arr = cslc_h5py_root[pol_path][()]
 
             # create dict for current polarization
             self.stats_dict[pol] = {}
             pol_dict = self.stats_dict[pol]
 
-            # write stats to HDF5
-            for real_imag, cstat_member in zip(['real', 'imaginary'],
-                                               [stat_obj.real, stat_obj.imag]):
+            # compute power or phase then write stats to HDF5 for CSLC
+            for pwr_phase in ['power', 'phase']:
                 # create dict to store real/imaginary stat items
-                pol_dict[real_imag] = {}
+                pol_dict[pwr_phase] = {}
 
-                # create HDF5 group for real/imaginary stats of current
+                # create HDF5 group for power or phase stats of current
                 # polarization
-                h5_stats_path = f'{QA_PATH}/statistics/data/{pol}/{real_imag}'
+                h5_stats_path = f'{QA_PATH}/statistics/data/{pol}/{pwr_phase}'
                 stats_group = cslc_h5py_root.require_group(h5_stats_path)
 
-                # add description for stat items
-                suffix_qa_item_desc = f'{real_imag} part of geoocoded SLC'
-
-                # build list of QA stat items for real_imag
+                # build list of QA stat items for pwr_phase
                 qa_items = []
-                vals = [cstat_member.mean, cstat_member.min,
-                        cstat_member.max, cstat_member.sample_stddev]
+                vals = _compute_slc_array_stats(pol_arr, pwr_phase)
                 for val_name, val in zip(self.stat_names, vals):
-                    desc = f'{val_name} of {suffix_qa_item_desc}'
+                    desc = f'{val_name} of {pwr_phase} of {pol} geocoded SLC'
                     qa_items.append(Meta(val_name, val, desc))
 
                 # save stats to dict and write to HDF5
-                _qa_items_to_h5_and_dict(stats_group, pol_dict[real_imag],
-                                      qa_items)
+                _qa_items_to_h5_and_dict(stats_group, pol_dict[pwr_phase],
+                                         qa_items)
 
 
     def compute_static_layer_stats(self, cslc_h5py_root, rdr2geo_params):

src/compass/utils/browse_image.py

@@ -94,7 +94,7 @@ def _normalize_apply_gamma(image, vmin, vmax, gamma=1.0):
         Normalized and gamma corrected image
     '''
     if vmax <= vmin:
-        raise ValueError('maximum value not > minimum value')
+        raise ValueError(f'maximum value {vmax} not > minimum value {vmin}')
 
     # scale to 0-1 for gray scale and then apply gamma correction
     image = (image - vmin) / (vmax - vmin)

tests/test_qa.py

@@ -0,0 +1,22 @@
+import h5py
+import numpy as np
+
+from compass.utils.h5_helpers import QA_PATH
+
+def test_qa_power_stats(geocode_slc_params):
+    def _power_test(arr):
+        return arr > 0.0
+
+    def _phase_test(arr):
+        return abs(arr) <= np.pi
+
+    # basic sanity checks of mean, min, and max
+    with h5py.File(geocode_slc_params.output_hdf5_path, 'r') as h5_obj:
+        for pwr_phase, test in {'power':_power_test,
+                                'phase':_phase_test}.items():
+            h5_stats_path = f'{QA_PATH}/statistics/data/VV/{pwr_phase}'
+            stat_names = ['mean', 'min', 'max']
+            for stat_name in stat_names:
+                print(f'Testing: {stat_name} of {pwr_phase}')
+                stat_val = h5_obj[f'{h5_stats_path}/{stat_name}'][()]
+                assert test(stat_val)

tests/test_s1_geocode_slc.py

@@ -3,11 +3,13 @@
 import h5py
 import isce3
 import numpy as np
+import pytest
 
 from compass import s1_geocode_slc
 from compass.utils.geo_runconfig import GeoRunConfig
 
 
+@pytest.mark.order(1)
 def test_geocode_slc_run(geocode_slc_params):
     '''
     Run s1_geocode_slc to ensure it does not crash