diff --git a/model_card_toolkit/dependencies.py b/model_card_toolkit/dependencies.py index 2bb2977..90faa68 100644 --- a/model_card_toolkit/dependencies.py +++ b/model_card_toolkit/dependencies.py @@ -58,13 +58,13 @@ _TEST_EXTRA_DEPS = ['absl', 'isort', 'pre-commit', 'pylint', 'pytest', 'yapf'] -TENSORFLOW_EXTRA_IMPORT_ERROR_MSG = """ +TENSORFLOW_EXTRA_IMPORT_ERROR_MSG = ''' This functionaliy requires `tensorflow` extra dependencies but they were not found in your environment. You can install them with: ``` pip install model-card-toolkit[tensorflow] ``` -""" +''' def _make_deps_list(package_names: List[str]) -> List[str]: diff --git a/model_card_toolkit/proto/model_card.proto b/model_card_toolkit/proto/model_card.proto index 8128588..7d43c30 100644 --- a/model_card_toolkit/proto/model_card.proto +++ b/model_card_toolkit/proto/model_card.proto @@ -12,7 +12,7 @@ message Owner { optional string contact = 2; } -// The information about verions of a model. +// The information about version of a model. // If there are multiple versions of the model, or there may be in the future, // it’s useful for your audience to know which version of the model is discussed // in the Model Card. If there are previous versions of this model, briefly @@ -177,7 +177,7 @@ message ConfidenceInterval { // The details of the performance metric. message PerformanceMetric { - // TODO(b/179415408): revist the design the of message + // TODO(b/179415408): revisit the design the of message // The following fields are EXPERIMENTAL and introduced for migration purpose. // For proto users, please do not rely on the fields. diff --git a/model_card_toolkit/tfx/component.py b/model_card_toolkit/tfx/component.py index c6f3486..f462394 100644 --- a/model_card_toolkit/tfx/component.py +++ b/model_card_toolkit/tfx/component.py @@ -23,7 +23,7 @@ except ImportError: ModelCardGenerator = None logging.info( - """ + ''' The ModelCardGenerator component and its dependencies have been moved to the TFX Addons project (https://github.com/tensorflow/tfx-addons). @@ -33,5 +33,5 @@ ``` $ pip install tfx-addons[model_card_generator] ``` - """ + ''' ) diff --git a/model_card_toolkit/utils/json_utils_test.py b/model_card_toolkit/utils/json_utils_test.py index af9b70e..55c3f21 100644 --- a/model_card_toolkit/utils/json_utils_test.py +++ b/model_card_toolkit/utils/json_utils_test.py @@ -23,219 +23,219 @@ from model_card_toolkit.utils import json_utils _MODEL_DETAILS_V1_DICT = { - "name": - "my model", - "owners": [ + 'name': + 'my model', + 'owners': [ { - "name": "foo", - "contact": "foo@xyz.com" + 'name': 'foo', + 'contact': 'foo@xyz.com' }, { - "name": "bar", - "contact": "bar@xyz.com" + 'name': 'bar', + 'contact': 'bar@xyz.com' } ], - "version": { - "name": "0.01", - "date": "2020-01-01" + 'version': { + 'name': '0.01', + 'date': '2020-01-01' }, - "license": - "Apache 2.0", - "references": ["https://my_model.xyz.com"], - "citation": - "https://doi.org/foo/bar" + 'license': + 'Apache 2.0', + 'references': ['https://my_model.xyz.com'], + 'citation': + 'https://doi.org/foo/bar' } _MODEL_PARAMETERS_V1_DICT = { - "model_architecture": "knn", - "data": { - "train": { - "name": "train_split", - "link": "path/to/train", - "sensitive": True, - "graphics": { - "collection": [{ - "name": "image1", - "image": "rawbytes" + 'model_architecture': 'knn', + 'data': { + 'train': { + 'name': 'train_split', + 'link': 'path/to/train', + 'sensitive': True, + 'graphics': { + 'collection': [{ + 'name': 'image1', + 'image': 'rawbytes' }] } }, - "eval": { - "name": "eval_split", - "link": "path/to/eval" + 'eval': { + 'name': 'eval_split', + 'link': 'path/to/eval' } } } _QUANTITATIVE_ANALYSIS_V1_DICT = { - "graphics": { - "collection": [{ - "name": "image1", - "image": "rawbytes" + 'graphics': { + 'collection': [{ + 'name': 'image1', + 'image': 'rawbytes' }] }, - "performance_metrics": [{ - "type": "log_loss", - "value": 0.2 + 'performance_metrics': [{ + 'type': 'log_loss', + 'value': 0.2 }] } _CONSIDERATIONS_V1_DICT = { - "users": ["foo", "bar"], - "use_cases": ["use case 1"], - "limitations": ["a limitation"], - "tradeoffs": ["tradeoff 1"], - "ethical_considerations": + 'users': ['foo', 'bar'], + 'use_cases': ['use case 1'], + 'limitations': ['a limitation'], + 'tradeoffs': ['tradeoff 1'], + 'ethical_considerations': [{ - "name": "risk1", - "mitigation_strategy": "a solution" + 'name': 'risk1', + 'mitigation_strategy': 'a solution' }] } _MODEL_CARD_V1_DICT = { - "model_details": _MODEL_DETAILS_V1_DICT, - "model_parameters": _MODEL_PARAMETERS_V1_DICT, - "quantitative_analysis": _QUANTITATIVE_ANALYSIS_V1_DICT, - "considerations": _CONSIDERATIONS_V1_DICT + 'model_details': _MODEL_DETAILS_V1_DICT, + 'model_parameters': _MODEL_PARAMETERS_V1_DICT, + 'quantitative_analysis': _QUANTITATIVE_ANALYSIS_V1_DICT, + 'considerations': _CONSIDERATIONS_V1_DICT } _MODEL_DETAILS_V2_DICT = { - "name": - "my model", - "owners": [ + 'name': + 'my model', + 'owners': [ { - "name": "foo", - "contact": "foo@xyz.com" + 'name': 'foo', + 'contact': 'foo@xyz.com' }, { - "name": "bar", - "contact": "bar@xyz.com" + 'name': 'bar', + 'contact': 'bar@xyz.com' } ], - "version": { - "name": "0.01", - "date": "2020-01-01" + 'version': { + 'name': '0.01', + 'date': '2020-01-01' }, - "license": [{ - "identifier": "Apache 2.0" + 'license': [{ + 'identifier': 'Apache 2.0' }], - "references": [{ - "reference": "https://my_model.xyz.com" + 'references': [{ + 'reference': 'https://my_model.xyz.com' }], - "citation": { - "citation": "https://doi.org/foo/bar" + 'citation': { + 'citation': 'https://doi.org/foo/bar' } } _MODEL_PARAMETERS_V2_DICT = { - "model_architecture": - "knn", - "data": [ + 'model_architecture': + 'knn', + 'data': [ { - "name": "train_split", - "link": "path/to/train", - "sensitive": { - "sensitive_data": [ - "this dataset contains PII", - "this dataset contains geo data" + 'name': 'train_split', + 'link': 'path/to/train', + 'sensitive': { + 'sensitive_data': [ + 'this dataset contains PII', + 'this dataset contains geo data' ] }, - "graphics": { - "collection": [{ - "name": "image1", - "image": "rawbytes" + 'graphics': { + 'collection': [{ + 'name': 'image1', + 'image': 'rawbytes' }] } }, { - "name": "eval_split", - "link": "path/to/eval" + 'name': 'eval_split', + 'link': 'path/to/eval' } ] } _QUANTITATIVE_ANALYSIS_V2_DICT = { - "graphics": { - "collection": [{ - "name": "image1", - "image": "rawbytes" + 'graphics': { + 'collection': [{ + 'name': 'image1', + 'image': 'rawbytes' }] }, - "performance_metrics": [{ - "type": "log_loss", - "value": "0.2" + 'performance_metrics': [{ + 'type': 'log_loss', + 'value': '0.2' }] } _CONSIDERATIONS_V2_DICT = { - "users": [{ - "description": "foo" + 'users': [{ + 'description': 'foo' }, { - "description": "bar" + 'description': 'bar' }], - "use_cases": [{ - "description": "use case 1" + 'use_cases': [{ + 'description': 'use case 1' }], - "limitations": [{ - "description": "a limitation" + 'limitations': [{ + 'description': 'a limitation' }], - "tradeoffs": [{ - "description": "tradeoff 1" + 'tradeoffs': [{ + 'description': 'tradeoff 1' }], - "ethical_considerations": + 'ethical_considerations': [{ - "name": "risk1", - "mitigation_strategy": "a solution" + 'name': 'risk1', + 'mitigation_strategy': 'a solution' }] } _MODEL_CARD_V2_DICT = { - "model_details": _MODEL_DETAILS_V2_DICT, - "model_parameters": _MODEL_PARAMETERS_V2_DICT, - "quantitative_analysis": _QUANTITATIVE_ANALYSIS_V2_DICT, - "considerations": _CONSIDERATIONS_V2_DICT + 'model_details': _MODEL_DETAILS_V2_DICT, + 'model_parameters': _MODEL_PARAMETERS_V2_DICT, + 'quantitative_analysis': _QUANTITATIVE_ANALYSIS_V2_DICT, + 'considerations': _CONSIDERATIONS_V2_DICT } _CATS_VS_DOGS_V1_PATH = os.path.join( - "utils", "testdata", "cats_vs_dogs_v0_0_1.json" + 'utils', 'testdata', 'cats_vs_dogs_v0_0_1.json' ) _CATS_VS_DOGS_V1_TEXT = pkgutil.get_data( - "model_card_toolkit", _CATS_VS_DOGS_V1_PATH + 'model_card_toolkit', _CATS_VS_DOGS_V1_PATH ) _CATS_VS_DOGS_V1_DICT = json.loads(_CATS_VS_DOGS_V1_TEXT) _CATS_VS_DOGS_V2_PATH = os.path.join( - "utils", "testdata", "cats_vs_dogs_v0_0_2.json" + 'utils', 'testdata', 'cats_vs_dogs_v0_0_2.json' ) _CATS_VS_DOGS_V2_TEXT = pkgutil.get_data( - "model_card_toolkit", _CATS_VS_DOGS_V2_PATH + 'model_card_toolkit', _CATS_VS_DOGS_V2_PATH ) _CATS_VS_DOGS_V2_DICT = json.loads(_CATS_VS_DOGS_V2_TEXT) -_MODEL_CARD_SECTIONS = ("model_details", "model_parameters", "considerations") -_QUANTITATIVE_ANALYSIS = "quantitative_analysis" -_CONFIDENCE_INTERVAL = "confidence_interval" +_MODEL_CARD_SECTIONS = ('model_details', 'model_parameters', 'considerations') +_QUANTITATIVE_ANALYSIS = 'quantitative_analysis' +_CONFIDENCE_INTERVAL = 'confidence_interval' class JsonUtilsTest(parameterized.TestCase): def test_validate_json_schema(self): json_utils.validate_json_schema( - _MODEL_CARD_V1_DICT, schema_version="0.0.1" + _MODEL_CARD_V1_DICT, schema_version='0.0.1' ) json_utils.validate_json_schema( - _MODEL_CARD_V2_DICT, schema_version="0.0.2" + _MODEL_CARD_V2_DICT, schema_version='0.0.2' ) def test_validate_json_schema_invalid_dict(self): - invalid_json_dict = {"model_name": "the_greatest_model"} + invalid_json_dict = {'model_name': 'the_greatest_model'} with self.assertRaises(jsonschema.ValidationError): json_utils.validate_json_schema(invalid_json_dict) def test_validate_json_schema_invalid_version(self): - invalid_schema_version = "0.0.3" + invalid_schema_version = '0.0.3' with self.assertRaises(ValueError): json_utils.validate_json_schema( _MODEL_CARD_V1_DICT, schema_version=invalid_schema_version ) @parameterized.named_parameters( - ("train_data", "train_data.json"), - ("considerations", "considerations.json"), - ("cats_vs_dogs", "cats_vs_dogs_v0_0_2.json"), ("full", "full.json") + ('train_data', 'train_data.json'), + ('considerations', 'considerations.json'), + ('cats_vs_dogs', 'cats_vs_dogs_v0_0_2.json'), ('full', 'full.json') ) def test_template_test_files(self, file_name): - template_path = os.path.join("utils", "testdata", file_name) + template_path = os.path.join('utils', 'testdata', file_name) json_data = json.loads( - pkgutil.get_data("model_card_toolkit", template_path) + pkgutil.get_data('model_card_toolkit', template_path) ) json_utils.validate_json_schema(json_data) @@ -257,17 +257,17 @@ def test_json_update_succeeds(self): # Check Graphics fields are equal v1_graphics_updated = updated_cats_vs_dogs_dict.get( _QUANTITATIVE_ANALYSIS - ).get("graphics") + ).get('graphics') v2_graphics = _CATS_VS_DOGS_V2_DICT.get(_QUANTITATIVE_ANALYSIS - ).get("graphics") + ).get('graphics') self.assertDictEqual(v1_graphics_updated, v2_graphics) # Check PerformanceMetrics fields are equal length v1_metrics_updated = updated_cats_vs_dogs_dict.get( _QUANTITATIVE_ANALYSIS - ).get("performance_metrics") + ).get('performance_metrics') v2_metrics = _CATS_VS_DOGS_V2_DICT.get(_QUANTITATIVE_ANALYSIS - ).get("performance_metrics") + ).get('performance_metrics') self.assertLen(v1_metrics_updated, len(v2_metrics)) # Check PerformanceMetrics fields are equal value @@ -276,12 +276,12 @@ def test_json_update_succeeds(self): for field in v1m.keys(): if field == _CONFIDENCE_INTERVAL: self.assertEqual( - str(v1m[_CONFIDENCE_INTERVAL]["lower_bound"]), - str(v2m[_CONFIDENCE_INTERVAL]["lower_bound"]) + str(v1m[_CONFIDENCE_INTERVAL]['lower_bound']), + str(v2m[_CONFIDENCE_INTERVAL]['lower_bound']) ) self.assertEqual( - str(v1m[_CONFIDENCE_INTERVAL]["upper_bound"]), - str(v2m[_CONFIDENCE_INTERVAL]["upper_bound"]) + str(v1m[_CONFIDENCE_INTERVAL]['upper_bound']), + str(v2m[_CONFIDENCE_INTERVAL]['upper_bound']) ) else: self.assertEqual(str(v1m[field]), str(v2m[field])) @@ -299,8 +299,8 @@ def test_json_update_latest_version_should_be_identity_function(self): def test_json_update_validation_error(self): with self.assertRaises(jsonschema.ValidationError): - json_utils.update(json_dict={"model_name": "the_greatest_model"}) + json_utils.update(json_dict={'model_name': 'the_greatest_model'}) -if __name__ == "__main__": +if __name__ == '__main__': absltest.main() diff --git a/model_card_toolkit/utils/tf_graphics_test.py b/model_card_toolkit/utils/tf_graphics_test.py index a6cbc41..f1d6f9d 100644 --- a/model_card_toolkit/utils/tf_graphics_test.py +++ b/model_card_toolkit/utils/tf_graphics_test.py @@ -40,13 +40,13 @@ def assertGraphEqual(self, g: graphics.Graph, h: graphics.Graph): def test_extract_graph_data_from_dataset_feature_statistics(self): empty_numeric_feature_stats = text_format.Parse( - """ + ''' path { - step: "numeric_feature" + step: 'numeric_feature' } type: INT num_stats { - }""", statistics_pb2.FeatureNameStatistics() + }''', statistics_pb2.FeatureNameStatistics() ) self.assertIsNone( tf_graphics._extract_graph_data_from_dataset_feature_statistics( @@ -55,9 +55,9 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): ) numeric_feature_stats = text_format.Parse( - """ + ''' path { - step: "numeric_feature" + step: 'numeric_feature' } type: INT num_stats { @@ -73,7 +73,7 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): sample_count: 4.0 } } - }""", statistics_pb2.FeatureNameStatistics() + }''', statistics_pb2.FeatureNameStatistics() ) self.assertGraphEqual( tf_graphics._extract_graph_data_from_dataset_feature_statistics( @@ -87,9 +87,9 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): ) string_feature_stats = text_format.Parse( - """ + ''' path { - step: "string_feature" + step: 'string_feature' } type: STRING string_stats { @@ -107,7 +107,7 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): sample_count: 2395.0 } } - }""", statistics_pb2.FeatureNameStatistics() + }''', statistics_pb2.FeatureNameStatistics() ) self.assertGraphEqual( tf_graphics._extract_graph_data_from_dataset_feature_statistics( @@ -121,12 +121,12 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): ) bytes_feature_stats = text_format.Parse( - """ + ''' path { - step: "bytes_feature" + step: 'bytes_feature' } type: BYTES - bytes_stats {}""", statistics_pb2.FeatureNameStatistics() + bytes_stats {}''', statistics_pb2.FeatureNameStatistics() ) self.assertIsNone( tf_graphics._extract_graph_data_from_dataset_feature_statistics( @@ -135,12 +135,12 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): ) struct_feature_stats = text_format.Parse( - """ + ''' path { - step: "struct_feature" + step: 'struct_feature' } type: STRUCT - struct_stats {}""", statistics_pb2.FeatureNameStatistics() + struct_stats {}''', statistics_pb2.FeatureNameStatistics() ) self.assertIsNone( tf_graphics._extract_graph_data_from_dataset_feature_statistics( @@ -150,11 +150,11 @@ def test_extract_graph_data_from_dataset_feature_statistics(self): def test_annotate_dataset_feature_statistics_plots(self): train_stats = text_format.Parse( - """ + ''' datasets { features { path { - step: "LDA_00" + step: 'LDA_00' } type: FLOAT num_stats { @@ -182,7 +182,7 @@ def test_annotate_dataset_feature_statistics_plots(self): } features { path { - step: "LDA_01" + step: 'LDA_01' } type: FLOAT num_stats { @@ -210,7 +210,7 @@ def test_annotate_dataset_feature_statistics_plots(self): } features { path { - step: "LDA_02" + step: 'LDA_02' } type: FLOAT num_stats { @@ -238,7 +238,7 @@ def test_annotate_dataset_feature_statistics_plots(self): } features { path { - step: "LDA_03" + step: 'LDA_03' } type: STRING bytes_stats { @@ -246,14 +246,14 @@ def test_annotate_dataset_feature_statistics_plots(self): } } } - """, statistics_pb2.DatasetFeatureStatisticsList() + ''', statistics_pb2.DatasetFeatureStatisticsList() ) eval_stats = text_format.Parse( - """ + ''' datasets { features { path { - step: "data_channel" + step: 'data_channel' } type: STRING string_stats { @@ -310,7 +310,7 @@ def test_annotate_dataset_feature_statistics_plots(self): } } } - """, statistics_pb2.DatasetFeatureStatisticsList() + ''', statistics_pb2.DatasetFeatureStatisticsList() ) model_card = model_card_module.ModelCard() diff --git a/model_card_toolkit/version.py b/model_card_toolkit/version.py index 71e3851..507c288 100644 --- a/model_card_toolkit/version.py +++ b/model_card_toolkit/version.py @@ -14,18 +14,18 @@ """Defines Model Card Toolkit version information.""" # We follow Semantic Versioning (https://semver.org/) -_MAJOR_VERSION = "3" -_MINOR_VERSION = "0" -_PATCH_VERSION = "0" +_MAJOR_VERSION = '3' +_MINOR_VERSION = '0' +_PATCH_VERSION = '0' # When building releases, we can update this value on the release branch to # reflect the current release candidate ('rc0', 'rc1') or, finally, the official # stable release (indicated by `_VERSION_SUFFIX = ''`). Outside the context of a # release branch, the current version is by default assumed to be a # 'development' version, labeled 'dev'. -_VERSION_SUFFIX = "dev" +_VERSION_SUFFIX = 'dev' # This produces version numbers such as, '0.1.0-dev', for example. -__version__ = ".".join([_MAJOR_VERSION, _MINOR_VERSION, _PATCH_VERSION]) +__version__ = '.'.join([_MAJOR_VERSION, _MINOR_VERSION, _PATCH_VERSION]) if _VERSION_SUFFIX: - __version__ = f"{__version__}-{_VERSION_SUFFIX}" + __version__ = f'{__version__}-{_VERSION_SUFFIX}' diff --git a/tools/build_api_docs.py b/tools/build_api_docs.py index aa0496e..b05cf33 100644 --- a/tools/build_api_docs.py +++ b/tools/build_api_docs.py @@ -10,24 +10,24 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - "output_dir", default="/tmp/model_card_toolkit", - help="Where to output the docs" + 'output_dir', default='/tmp/model_card_toolkit', + help='Where to output the docs' ) flags.DEFINE_string( - "code_url_prefix", default= - "https://github.com/tensorflow/model-card-toolkit/tree/main/model-card-toolkit", - help="The URL prefix for links to code." + 'code_url_prefix', default= + 'https://github.com/tensorflow/model-card-toolkit/tree/main/model-card-toolkit', + help='The URL prefix for links to code.' ) flags.DEFINE_bool( - "search_hints", default=True, - help="Include metadata search hints in the generated files" + 'search_hints', default=True, + help='Include metadata search hints in the generated files' ) flags.DEFINE_string( - "site_path", default="responsible-ai/model_card_toolkit/api_docs/python", - help="Path prefix in the _toc.yaml" + 'site_path', default='responsible-ai/model_card_toolkit/api_docs/python', + help='Path prefix in the _toc.yaml' ) @@ -39,8 +39,8 @@ def execute( # TODO(b/175031010): add missing class vars, or remove all class vars doc_generator = generate_lib.DocGenerator( - root_title="Model Card Toolkit", py_modules=[ - ("model_card_toolkit", model_card_toolkit) + root_title='Model Card Toolkit', py_modules=[ + ('model_card_toolkit', model_card_toolkit) ], base_dir=os.path.dirname(model_card_toolkit.__file__), search_hints=search_hints, code_url_prefix=code_url_prefix, site_path=site_path, callbacks=[ @@ -59,5 +59,5 @@ def main(unused_argv): ) -if __name__ == "__main__": +if __name__ == '__main__': app.run(main)