Add testing with PyTest and GitHub Actions workflow

.github/actions/setup-env/action.yml

@@ -0,0 +1,28 @@
+name: Set up environment
+description: Set up environment and install package
+
+inputs:
+  python-version:
+    default: "3.10"
+    required: true
+  package-root-dir:
+    default: "./"
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache-dependency-path: |
+          ${{ inputs.package-root-dir }}/setup.py
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install ${{ inputs.package-root-dir }}[test]
+

.github/workflows/test.yml

@@ -0,0 +1,38 @@
+name: Tests
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+  workflow_dispatch:
+
+jobs:
+  tests:
+    if: github.actor != 'copybara-service[bot]'
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10']
+        package-root-dir: ['./', './tensorboard_plugin']
+
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v4
+
+    - name: Set up environment
+      uses: ./.github/actions/setup-env
+      with:
+        python-version: ${{ matrix.python-version }}
+        package-root-dir: ${{ matrix.package-root-dir }}
+
+    - name: Run tests
+      shell: bash
+      run: |
+        cd ${{ matrix.package-root-dir }}
+        pytest

fairness_indicators/example_model_test.py

@@ -163,7 +163,3 @@ def test_example_model(self):
         metric_values['fairness_indicators_metrics/[email protected]'],
         {'doubleValue': 1.0},
     )
-
-
-if __name__ == '__main__':
-  tf.test.main()

fairness_indicators/remediation/weight_utils_test.py

@@ -215,7 +215,3 @@ def test_create_difference_dictionary_bounded_metrics(self):
     self.assertIn('gender', res)
     self.assertAlmostEqual(res['gender']['female'], 0.125)
     self.assertAlmostEqual(res[''][''], 0)
-
-
-if __name__ == '__main__':
-  tf.test.main()

fairness_indicators/tutorial_utils/util_test.py

@@ -323,6 +323,3 @@ def test_get_eval_results_called_correclty(self, mock_run_model_analysis,
         eval_config=expected_eval_config,
         output_path=eval_results_path,
         extractors=None)
-
-if __name__ == '__main__':
-  tf.test.main()

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = "--import-mode=importlib"
+testpaths = "fairness_indicators"
+python_files = "*_test.py"

setup.py

@@ -43,6 +43,10 @@ def select_constraint(default, nightly=None, git_master=None):
     'witwidget>=1.4.4,<2',
     'protobuf>=3.20.3,<5',
 ]
+
+TEST_PACKAGES = [
+    'pytest>=8.3.0,<9',
+]
 # Get version from version module.
 with open('fairness_indicators/version.py') as fp:
   globals_dict = {}
@@ -66,6 +70,9 @@ def select_constraint(default, nightly=None, git_master=None):
     python_requires='>=3.9,<4',
     install_requires=REQUIRED_PACKAGES,
     tests_require=REQUIRED_PACKAGES,
+    extras_require={
+        'test': TEST_PACKAGES,
+    },
     # PyPI package information.
     classifiers=[
         'Development Status :: 4 - Beta',

tensorboard_plugin/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = "--import-mode=importlib"
+testpaths = "tensorboard_plugin_fairness_indicators"
+python_files = "*_test.py"

tensorboard_plugin/setup.py

@@ -50,6 +50,10 @@ def select_constraint(default, nightly=None, git_master=None):
     'werkzeug<2',
 ]
 
+TEST_PACKAGES = [
+    'pytest>=8.3.0,<9',
+]
+
 with open('README.md', 'r', encoding='utf-8') as fh:
   long_description = fh.read()
 
@@ -80,6 +84,9 @@ def select_constraint(default, nightly=None, git_master=None):
     python_requires='>=3.9,<4',
     install_requires=REQUIRED_PACKAGES,
     tests_require=REQUIRED_PACKAGES,
+    extras_require={
+        'test': TEST_PACKAGES,
+    },
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

tensorboard_plugin/tensorboard_plugin_fairness_indicators/metadata_test.py

@@ -34,7 +34,3 @@ def testCreateSummaryMetadata_withoutDescription(self):
     summary_metadata = metadata.CreateSummaryMetadata()
     self.assertEqual(metadata.PLUGIN_NAME,
                      summary_metadata.plugin_data.plugin_name)
-
-
-if __name__ == '__main__':
-  tf.test.main()

tensorboard_plugin/tensorboard_plugin_fairness_indicators/plugin_test.py

@@ -19,6 +19,7 @@
 
 from collections import abc
 import os
+import pytest
 import shutil
 from unittest import mock
 
@@ -150,6 +151,12 @@ def testIndexJsRoute(self):
     response = self._server.get("/data/plugin/fairness_indicators/index.js")
     self.assertEqual(200, response.status_code)
 
+  @pytest.mark.xfail(
+    reason=(
+      "Failing on `master` as of `942b672457e07ac2ac27de0bcc45a4c80276785c`. "
+      "Please remove once fixed."
+      )
+  )
   def testVulcanizedTemplateRoute(self):
     """Tests that the /tags route offers the correct run to tag mapping."""
     response = self._server.get(
@@ -222,7 +229,3 @@ def testGetOutputFileFormat(self):
     self.assertEqual(
         "tfrecord", self._plugin._get_output_file_format("abc_path.tfrecord")
     )
-
-
-if __name__ == "__main__":
-  tf.test.main()

tensorboard_plugin/tensorboard_plugin_fairness_indicators/summary_v2_test.py

@@ -67,7 +67,3 @@ def testSummary(self):
         six.ensure_text(summary_value.tensor.string_val[0], 'utf-8'))
     self.assertEqual(metadata.PLUGIN_NAME,
                      summary_value.metadata.plugin_data.plugin_name)
-
-
-if __name__ == '__main__':
-  tf.test.main()