From d8dda47d1ca536551397d0c2b010ccbaa1d759ba Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 16:58:55 -0400 Subject: [PATCH 1/9] refactor operations api to reuse models from public api --- .gitignore | 5 +- api/src/shared/db_models/gtfs_dataset_impl.py | 2 +- api/src/shared/db_models/gtfs_rt_feed_impl.py | 6 +- .../shared/db_models/latest_dataset_impl.py | 2 +- api/src/utils/model_utils.py | 26 - api/tests/utils/test_compare_java_versions.py | 2 +- docs/OperationsAPI.yaml | 1033 ++++++++++++++--- functions-python/.flake8 | 2 +- .../tests/test_feed_sync_process.py | 6 +- functions-python/helpers/query_helper.py | 17 +- functions-python/operations_api/.coveragerc | 2 +- functions-python/operations_api/.gitignore | 2 +- .../operations_api/.openapi-generator/FILES | 57 +- .../operations_api/function_config.json | 2 +- .../operations_api/requirements.txt | 1 + .../impl/feeds_operations_impl.py | 78 +- .../impl/models/basic_feed_impl.py | 59 - .../impl/models/get_feeds_response.py | 93 -- .../impl/models/gtfs_feed_impl.py | 37 - .../impl/models/gtfs_rt_feed_impl.py | 56 - .../impl/models/location_impl.py | 23 - ..._impl.py => operation_entity_type_impl.py} | 4 +- ..._impl.py => operation_external_id_impl.py} | 10 +- .../impl/models/operation_feed_impl.py | 33 + ...ect_impl.py => operation_redirect_impl.py} | 10 +- .../models/update_request_gtfs_feed_impl.py | 19 +- .../update_request_gtfs_rt_feed_impl.py | 28 +- functions-python/operations_api/src/main.py | 2 +- .../impl/models/test_entity_type_impl.py | 27 - .../impl/models/test_external_id_impl.py | 26 - .../impl/models/test_feed_responses.py | 26 +- .../impl/models/test_redirect_impl.py | 53 - .../test_update_request_gtfs_feed_impl.py | 24 +- .../test_update_request_gtfs_rt_feed_impl.py | 17 +- .../impl/test_feeds_operations_impl_gtfs.py | 13 +- .../test_feeds_operations_impl_gtfs_rt.py | 13 +- .../feeds_operations/impl/test_get_feeds.py | 44 - scripts/gen-operations-config.yaml | 2 +- 38 files changed, 1083 insertions(+), 779 deletions(-) delete mode 100644 api/src/utils/model_utils.py delete mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/basic_feed_impl.py delete mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/get_feeds_response.py delete mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/gtfs_feed_impl.py delete mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/gtfs_rt_feed_impl.py delete mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/location_impl.py rename functions-python/operations_api/src/feeds_operations/impl/models/{entity_type_impl.py => operation_entity_type_impl.py} (92%) rename functions-python/operations_api/src/feeds_operations/impl/models/{external_id_impl.py => operation_external_id_impl.py} (88%) create mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py rename functions-python/operations_api/src/feeds_operations/impl/models/{redirect_impl.py => operation_redirect_impl.py} (90%) delete mode 100644 functions-python/operations_api/tests/feeds_operations/impl/models/test_entity_type_impl.py delete mode 100644 functions-python/operations_api/tests/feeds_operations/impl/models/test_external_id_impl.py delete mode 100644 functions-python/operations_api/tests/feeds_operations/impl/models/test_redirect_impl.py diff --git a/.gitignore b/.gitignore index 2c0a3926a..17896ff5a 100644 --- a/.gitignore +++ b/.gitignore @@ -80,4 +80,7 @@ tf.plan functions-python/**/*.csv # Local emulators -.cloudstorage \ No newline at end of file +.cloudstorage + +# Project files +*.code-workspace \ No newline at end of file diff --git a/api/src/shared/db_models/gtfs_dataset_impl.py b/api/src/shared/db_models/gtfs_dataset_impl.py index b7161ca44..3076b5d64 100644 --- a/api/src/shared/db_models/gtfs_dataset_impl.py +++ b/api/src/shared/db_models/gtfs_dataset_impl.py @@ -5,7 +5,7 @@ from shared.db_models.bounding_box_impl import BoundingBoxImpl from shared.db_models.validation_report_impl import ValidationReportImpl from feeds_gen.models.gtfs_dataset import GtfsDataset -from utils.model_utils import compare_java_versions +from shared.db_models.model_utils import compare_java_versions class GtfsDatasetImpl(GtfsDataset): diff --git a/api/src/shared/db_models/gtfs_rt_feed_impl.py b/api/src/shared/db_models/gtfs_rt_feed_impl.py index 789cfe5d6..429855ded 100644 --- a/api/src/shared/db_models/gtfs_rt_feed_impl.py +++ b/api/src/shared/db_models/gtfs_rt_feed_impl.py @@ -18,7 +18,7 @@ def from_orm(cls, feed: GtfsRTFeedOrm | None) -> GtfsRTFeed | None: gtfs_rt_feed: GtfsRTFeed = super().from_orm(feed) if not gtfs_rt_feed: return None - gtfs_rt_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations] - gtfs_rt_feed.entity_types = [item.name for item in feed.entitytypes] - gtfs_rt_feed.feed_references = [item.stable_id for item in feed.gtfs_feeds] + gtfs_rt_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations] if feed.locations else [] + gtfs_rt_feed.entity_types = [item.name for item in feed.entitytypes] if feed.entitytypes else [] + gtfs_rt_feed.feed_references = [item.stable_id for item in feed.gtfs_feeds] if feed.gtfs_feeds else [] return gtfs_rt_feed diff --git a/api/src/shared/db_models/latest_dataset_impl.py b/api/src/shared/db_models/latest_dataset_impl.py index f2d18f6fa..b650f1ab1 100644 --- a/api/src/shared/db_models/latest_dataset_impl.py +++ b/api/src/shared/db_models/latest_dataset_impl.py @@ -4,7 +4,7 @@ from shared.db_models.bounding_box_impl import BoundingBoxImpl from feeds_gen.models.latest_dataset import LatestDataset from feeds_gen.models.latest_dataset_validation_report import LatestDatasetValidationReport -from utils.model_utils import compare_java_versions +from shared.db_models.model_utils import compare_java_versions class LatestDatasetImpl(LatestDataset): diff --git a/api/src/utils/model_utils.py b/api/src/utils/model_utils.py deleted file mode 100644 index 90d7398a0..000000000 --- a/api/src/utils/model_utils.py +++ /dev/null @@ -1,26 +0,0 @@ -from packaging.version import Version - - -def compare_java_versions(v1: str | None, v2: str | None): - """ - Compare two version strings v1 and v2. - Returns 1 if v1 > v2, -1 if v1 < v2, - otherwise 0. - The version strings are expected to be in the format of - major.minor.patch[-SNAPSHOT] - """ - if v1 is None and v2 is None: - return 0 - if v1 is None: - return -1 - if v2 is None: - return 1 - # clean version strings replacing the SNAPSHOT suffix with .dev0 - v1 = v1.replace("-SNAPSHOT", ".dev0") - v2 = v2.replace("-SNAPSHOT", ".dev0") - if Version(v1) > Version(v2): - return 1 - elif Version(v1) < Version(v2): - return -1 - else: - return 0 diff --git a/api/tests/utils/test_compare_java_versions.py b/api/tests/utils/test_compare_java_versions.py index 91fd37bf9..b44c6d9a5 100644 --- a/api/tests/utils/test_compare_java_versions.py +++ b/api/tests/utils/test_compare_java_versions.py @@ -1,5 +1,5 @@ import unittest -from utils.model_utils import compare_java_versions +from shared.db_models.model_utils import compare_java_versions class TestCompareJavaVersions(unittest.TestCase): diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 44c43dcad..4d0b6d48e 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -14,11 +14,9 @@ info: license: name: MobilityData License url: https://www.apache.org/licenses/LICENSE-2.0 - tags: - name: "operations" description: "Mobility Database Operations" - paths: /v1/operations/feeds: get: @@ -33,14 +31,14 @@ paths: required: false schema: type: string - enum: [ wip, published, unpublished ] + enum: [wip, published, unpublished] - name: data_type in: query description: Filter feeds by data type. required: false schema: type: string - enum: [ gtfs, gtfs_rt ] + enum: [gtfs, gtfs_rt] - name: offset in: query description: Number of items to skip for pagination. @@ -73,12 +71,11 @@ paths: feeds: type: array items: - $ref: "#/components/schemas/BaseFeed" + $ref: "#/components/schemas/OperationFeed" 401: description: Unauthorized. 500: description: Internal Server Error. - /v1/operations/feeds/gtfs: put: description: Update the specified GTFS feed in the Mobility Database. @@ -86,7 +83,7 @@ paths: - "operations" operationId: updateGtfsFeed security: - - ApiKeyAuth: [ ] + - ApiKeyAuth: [] requestBody: description: Payload to update the specified GTFS feed. required: true @@ -98,19 +95,61 @@ paths: 200: description: > The feed was successfully updated. No content is returned. + 204: description: > The feed update request was successfully received, but the update process was skipped as the request matches with the source feed. + 400: description: > The request was invalid. + 401: description: > The request was not authenticated or has invalid authentication credentials. + 500: description: > An internal server error occurred. + /v1/operations/gtfs_feeds/{id}: + parameters: + - $ref: "#/components/parameters/feed_id_path_param" + get: + description: Get the specified GTFS feed from the Mobility Database. + tags: + - "operations" + operationId: getGtfsFeed + security: + - Authentication: [] + responses: + 200: + description: > + Successful pull of the GTFS feeds common info for the provided ID. + + content: + application/json: + schema: + $ref: "#/components/schemas/OperationGtfsFeed" + /v1/operations/gtfs_rt_feeds/{id}: + parameters: + - $ref: "#/components/parameters/feed_id_path_param" + get: + description: Get the specified GTFS-RT feed from the Mobility Database. + tags: + - "operations" + operationId: getGtfsRtFeed + security: + - Authentication: [] + responses: + 200: + description: > + Successful pull of the GTFS-RT feeds common info for the provided ID. + + content: + application/json: + schema: + $ref: "#/components/schemas/OperationGtfsRtFeed" /v1/operations/feeds/gtfs_rt: put: description: Update the specified GTFS-RT feed in the Mobility Database. @@ -118,7 +157,7 @@ paths: - "operations" operationId: updateGtfsRtFeed security: - - ApiKeyAuth: [ ] + - ApiKeyAuth: [] requestBody: description: Payload to update the specified GTFS-RT feed. required: true @@ -130,92 +169,306 @@ paths: 200: description: > The feed was successfully updated. No content is returned. + 204: description: > The feed update request was successfully received, but the update process was skipped as the request matches with the source feed. + 400: description: > The request was invalid. + 401: description: > The request was not authenticated or has invalid authentication credentials. + 500: description: > An internal server error occurred. components: schemas: - BaseFeed: + Redirect: type: object - discriminator: - propertyName: data_type - mapping: - gtfs: '#/components/schemas/GtfsFeedResponse' - gtfs_rt: '#/components/schemas/GtfsRtFeedResponse' properties: - id: + target_id: + description: The feed ID that should be used in replacement of the current one. type: string - description: Unique identifier for the feed. - stable_id: + example: mdb-10 + comment: + description: A comment explaining the redirect. type: string - description: Stable identifier for the feed. - status: - $ref: "#/components/schemas/FeedStatus" + example: Redirected because of a change of URL. + BasicFeed: + type: object + properties: + id: + description: Unique identifier used as a key for the feeds table. + type: string + example: mdb-1210 data_type: - $ref: "#/components/schemas/DataType" - provider: type: string - description: Name of the transit provider. - feed_name: + enum: + - gtfs + - gtfs_rt + - gbfs + example: gtfs + # Have to put the enum inline because of a bug in openapi-generator + # $ref: "#/components/schemas/DataType" + created_at: + description: The date and time the feed was added to the database, in ISO 8601 date-time format. type: string - description: Name of the feed. - note: + example: 2023-07-10T22:06:00Z + format: date-time + external_ids: + $ref: "#/components/schemas/ExternalIds" + provider: + description: A commonly used name for the transit provider included in the feed. type: string - description: Additional notes about the feed. + example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express) feed_contact_email: + description: Use to contact the feed producer. type: string - description: Contact email for the feed. + example: someEmail@ladotbus.com source_info: $ref: "#/components/schemas/SourceInfo" - operational_status: + redirects: + type: array + items: + $ref: "#/components/schemas/Redirect" + Feed: + allOf: + - $ref: "#/components/schemas/BasicFeed" + - type: object + discriminator: + propertyName: data_type + mapping: + gtfs: '#/components/schemas/GtfsFeed' + gtfs_rt: '#/components/schemas/GtfsRTFeed' + properties: + status: + description: > + Describes status of the Feed. Should be one of + + * `active` Feed should be used in public trip planners. + * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. + * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. + * `development` Feed is being used for development purposes and should not be used in public trip planners. + * `future` Feed is not yet active but will be in the future. + type: string + enum: + - active + - deprecated + - inactive + - development + - future + example: deprecated + official: + description: > + A boolean value indicating if the feed is official or not. Official feeds are provided by the transit agency or a trusted source. + + type: boolean + example: true + official_updated_at: + description: > + The date and time the official status was last updated, in ISO 8601 date-time format. + + type: string + example: 2023-07-10T22:06:00Z + format: date-time + feed_name: + description: > + An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. + + type: string + example: Bus + note: + description: A note to clarify complex use cases for consumers. + type: string + GtfsFeed: + allOf: + - $ref: "#/components/schemas/Feed" + - type: object + properties: + # We reproduce this property here so we can have a specific example. + data_type: + type: string + enum: + - gtfs + - gtfs_rt + - gbfs + example: gtfs + locations: + $ref: "#/components/schemas/Locations" + latest_dataset: + $ref: "#/components/schemas/LatestDataset" + bounding_box: + $ref: "#/components/schemas/BoundingBox" + visualization_dataset_id: + description: > + The dataset ID of the dataset used to compute the visualization files. + + type: string + example: mdb-1210-202402121801 + GbfsFeed: + allOf: + - $ref: "#/components/schemas/BasicFeed" + - type: object + properties: + # We reproduce this property here so we can have a specific example. + data_type: + type: string + enum: + - gtfs + - gtfs_rt + - gbfs + example: gbfs + locations: + $ref: "#/components/schemas/Locations" + system_id: + description: > + The system ID of the feed. This is a unique identifier for the system that the feed belongs to. + + type: string + example: system-1234 + provider_url: + description: > + The URL of the provider's website. This is the website of the organization that operates the system that the feed belongs to. + + type: string + format: url + example: https://www.citybikenyc.com/ + versions: + description: > + A list of GBFS versions that the feed supports. Each version is represented by its version number and a list of endpoints. + + type: array + items: + $ref: "#/components/schemas/GbfsVersion" + bounding_box: + $ref: "#/components/schemas/BoundingBox" + bounding_box_generated_at: + description: The date and time the bounding box was generated, in ISO 8601 date-time format. + type: string + example: 2023-07-10T22:06:00Z + format: date-time + GbfsVersion: + type: object + properties: + version: + description: > + The version of the GBFS specification that the feed is using. This is a string that follows the semantic versioning format. + type: string - enum: [ wip, published, unpublished ] - description: Current operational status of the feed. + example: 2.3 created_at: + description: > + The date when the GBFS version was saved to the database. + type: string format: date-time - description: When the feed was created. - official: - type: boolean - description: Whether this is an official feed. - official_updated_at: + example: 2023-07-10T22:06:00Z + last_updated_at: + description: > + The date when the GBFS version was last updated in the database. + type: string format: date-time - description: When the official status was last updated. - locations: - type: array - items: - $ref: "#/components/schemas/Location" - external_ids: - type: array - items: - $ref: "#/components/schemas/ExternalId" - redirects: + example: 2023-07-10T22:06:00Z + source: + description: > + Indicates the origin of the version information. Possible values are: + + * `autodiscovery`: Retrieved directly from the main GBFS autodiscovery URL. + * `gbfs_versions`: Retrieved from the `gbfs_versions` endpoint. + type: string + enum: + - autodiscovery + - gbfs_versions + endpoints: + description: > + A list of endpoints that are available in the version. + type: array items: - $ref: "#/components/schemas/Redirect" + $ref: "#/components/schemas/GbfsEndpoint" + latest_validation_report: + $ref: "#/components/schemas/GbfsValidationReport" + GbfsValidationReport: + type: object + description: > + A validation report of the GBFS feed. - GtfsFeedResponse: - allOf: - - $ref: "#/components/schemas/BaseFeed" - - type: object - description: GTFS feed response model. + properties: + validated_at: + description: > + The date and time the GBFS feed was validated, in ISO 8601 date-time format. + + type: string + example: 2023-07-10T22:06:00Z + format: date-time + total_error: + type: integer + example: 10 + minimum: 0 + report_summary_url: + description: > + The URL of the JSON report of the validation summary. + + type: string + format: url + example: https://storage.googleapis.com/mobilitydata-datasets-prod/validation-reports/gbfs-1234-202402121801.json + validator_version: + description: > + The version of the validator used to validate the GBFS feed. + + type: string + example: 1.0.13 + GbfsEndpoint: + type: object + properties: + name: + description: > + The name of the endpoint. This is a human-readable name for the endpoint. + + type: string + example: system_information + url: + description: > + The URL of the endpoint. This is the URL where the endpoint can be accessed. - GtfsRtFeedResponse: + type: string + format: url + example: https://gbfs.citibikenyc.com/gbfs/system_information.json + language: + description: > + The language of the endpoint. This is the language that the endpoint is available in for versions 2.3 and prior. + + type: string + example: en + is_feature: + description: > + A boolean value indicating if the endpoint is a feature. A feature is defined as an optionnal endpoint. + + type: boolean + example: false + GbfsFeeds: + type: array + items: + $ref: "#/components/schemas/GbfsFeed" + GtfsRTFeed: allOf: - - $ref: "#/components/schemas/BaseFeed" + - $ref: "#/components/schemas/Feed" - type: object properties: + # We reproduce this property here so we can have a specific example. + data_type: + type: string + enum: + - gtfs + - gtfs_rt + - gbfs + example: gtfs_rt entity_types: type: array items: @@ -227,48 +480,539 @@ components: example: vp description: > The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts + # Have to put the enum inline because of a bug in openapi-generator + # $ref: "#/components/schemas/EntityTypes" feed_references: - description: - A list of the GTFS feeds that the real time source is associated with, represented by their MDB source IDs. + description: A list of the GTFS feeds that the real time source is associated with, represented by their MDB source IDs. type: array items: type: string example: "mdb-20" - - GetFeeds200Response: + locations: + $ref: "#/components/schemas/Locations" + SearchFeedItemResult: + # The following schema is used to represent the search results for feeds. + # The schema is a union of all the possible types(Feed, GtfsFeed, GtfsRTFeed and GbfsFeed) of feeds that can be returned. + # This union is not based on its original types due to the limitations of openapi-generator. + # For the same reason it's not defined as anyOf, but as a single object with all the possible properties. type: object + required: + - id + - data_type + - status properties: - total: - type: integer - description: Total number of feeds matching the criteria. - offset: + id: + description: Unique identifier used as a key for the feeds table. type: string - description: Current offset for pagination. - limit: + example: mdb-1210 + data_type: type: string - description: Maximum number of items per page. - feeds: + enum: + - gtfs + - gtfs_rt + - gbfs + example: gtfs + # Have to put the enum inline because of a bug in openapi-generator + # $ref: "#/components/schemas/DataType" + status: + description: > + Describes status of the Feed. Should be one of + + * `active` Feed should be used in public trip planners. + * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. + * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. + * `development` Feed is being used for development purposes and should not be used in public trip planners. + * `future` Feed is not yet active but will be in the future. + type: string + enum: + - active + - deprecated + - inactive + - development + - future + example: deprecated + # Have to put the enum inline because of a bug in openapi-generator + # $ref: "#/components/schemas/FeedStatus" + created_at: + description: The date and time the feed was added to the database, in ISO 8601 date-time format. + type: string + example: 2023-07-10T22:06:00Z + format: date-time + official: + description: > + A boolean value indicating if the feed is official or not. Official feeds are provided by the transit agency or a trusted source. + + type: boolean + example: true + external_ids: + $ref: "#/components/schemas/ExternalIds" + provider: + description: A commonly used name for the transit provider included in the feed. + type: string + example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express) + feed_name: + description: > + An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. + + type: string + example: Bus + note: + description: A note to clarify complex use cases for consumers. + type: string + feed_contact_email: + description: Use to contact the feed producer. + type: string + example: someEmail@ladotbus.com + source_info: + $ref: "#/components/schemas/SourceInfo" + redirects: type: array items: - $ref: "#/components/schemas/BaseFeed" - description: List of feeds. + $ref: "#/components/schemas/Redirect" + locations: + $ref: "#/components/schemas/Locations" + latest_dataset: + $ref: "#/components/schemas/LatestDataset" + entity_types: + type: array + items: + type: string + enum: + - vp + - tu + - sa + example: vp + description: > + The type of realtime entry: - Redirect: + * vp - vehicle positions + * tu - trip updates + * sa - service alerts + # Have to put the enum inline because of a bug in openapi-generator + # $ref: "#/components/schemas/EntityTypes" + versions: + type: array + items: + type: string + example: 2.3 + description: The supported versions of the GBFS feed. + feed_references: + description: A list of the GTFS feeds that the real time source is associated with, represented by their MDB source IDs. + type: array + items: + type: string + example: "mdb-20" + Feeds: + type: array + items: + $ref: "#/components/schemas/Feed" + GtfsFeeds: + type: array + items: + $ref: "#/components/schemas/GtfsFeed" + GtfsRTFeeds: + type: array + items: + $ref: "#/components/schemas/GtfsRTFeed" + LatestDataset: type: object properties: - target_id: - description: The feed ID that should be used in replacement of the current one. + id: + description: Identifier of the latest dataset for this feed. type: string - example: mdb-10 - comment: - description: A comment explaining the redirect. + example: mdb-1210-202402121801 + hosted_url: + description: > + As a convenience, the URL of the latest uploaded dataset hosted by MobilityData. It should be the same URL as the one found in the latest dataset id dataset. An alternative way to find this is to use the latest dataset id to obtain the dataset and then use its hosted_url. + type: string - example: Redirected because of a change of URL. + format: url + example: https://storage.googleapis.com/mobilitydata-datasets-prod/mdb-1210/mdb-1210-202402121801/mdb-1210-202402121801.zip + bounding_box: + $ref: "#/components/schemas/BoundingBox" + downloaded_at: + description: The date and time the dataset was downloaded from the producer, in ISO 8601 date-time format. + type: string + example: 2023-07-10T22:06:00Z + format: date-time + hash: + description: A hash of the dataset. + type: string + example: ad3805c4941cd37881ff40c342e831b5f5224f3d8a9a2ec3ac197d3652c78e42 + service_date_range_start: + description: The start date of the service date range for the dataset in UTC. Timing starts at 00:00:00 of the day. + type: string + example: 2023-07-10T06:00:00Z + format: date-time + service_date_range_end: + description: The start date of the service date range for the dataset in UTC. Timing ends at 23:59:59 of the day. + type: string + example: 2023-07-10T05:59:59+00Z + format: date-time + agency_timezone: + description: The timezone of the agency. + type: string + example: America/Los_Angeles + zipped_folder_size_mb: + description: The size of the zipped folder in MB. + type: number + example: 100.2 + unzipped_folder_size_mb: + description: The size of the unzipped folder in MB. + type: number + example: 200.5 + validation_report: + type: object + properties: + features: + description: List of GTFS features associated to the dataset. More information, https://gtfs.org/getting-started/features/overview + type: array + items: + type: string + example: ["Shapes", "Headsigns", "Wheelchair Accessibility"] + total_error: + type: integer + example: 10 + minimum: 0 + total_warning: + type: integer + example: 20 + minimum: 0 + total_info: + type: integer + example: 30 + minimum: 0 + unique_error_count: + type: integer + example: 1 + minimum: 0 + unique_warning_count: + type: integer + example: 2 + minimum: 0 + unique_info_count: + type: integer + example: 3 + minimum: 0 + # Have to put the enum inline because of a bug in openapi-generator + # EntityTypes: + # type: array + # items: + # $ref: "#/components/schemas/EntityType" + # EntityType: + # type: string + # enum: + # - vp + # - tu + # - sa + # example: vp + # description: > + # The type of realtime entry: + # * vp - vehicle positions + # * tu - trip updates + # * sa - service alerts + ExternalIds: + type: array + items: + $ref: "#/components/schemas/ExternalId" + ExternalId: + type: object + properties: + external_id: + description: The ID that can be use to find the feed data in an external or legacy database. + type: string + example: 1210 + source: + description: The source of the external ID, e.g. the name of the database where the external ID can be used. + type: string + example: mdb + SourceInfo: + type: object + properties: + producer_url: + description: > + URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. + + type: string + format: url + example: https://ladotbus.com/gtfs + authentication_type: + description: > + Defines the type of authentication required to access the `producer_url`. Valid values for this field are: + + * 0 or (empty) - No authentication required. + * 1 - The authentication requires an API key, which should be passed as value of the parameter api_key_parameter_name in the URL. Please visit URL in authentication_info_url for more information. + * 2 - The authentication requires an HTTP header, which should be passed as the value of the header api_key_parameter_name in the HTTP request. + When not provided, the authentication type is assumed to be 0. + + type: integer + enum: + - 0 + - 1 + - 2 + example: 2 + authentication_info_url: + description: > + Contains a URL to a human-readable page describing how the authentication should be performed and how credentials can be created. This field is required for `authentication_type=1` and `authentication_type=2`. + + type: string + format: url + example: https://apidevelopers.ladottransit.com + api_key_parameter_name: + type: string + description: > + Defines the name of the parameter to pass in the URL to provide the API key. This field is required for `authentication_type=1` and `authentication_type=2`. + + example: Ocp-Apim-Subscription-Key + license_url: + description: A URL where to find the license for the feed. + type: string + format: url + example: https://www.ladottransit.com/dla.html + Locations: + type: array + items: + $ref: "#/components/schemas/Location" + Location: + type: object + properties: + country_code: + description: > + ISO 3166-1 alpha-2 code designating the country where the system is located. For a list of valid codes [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). + + type: string + example: US + country: + description: The english name of the country where the system is located. + type: string + example: United States + subdivision_name: + description: > + ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). + + type: string + example: California + municipality: + description: Primary municipality in english in which the transit system is located. + type: string + example: Los Angeles + # Have to put the enum inline because of a bug in openapi-generator + # FeedStatus: + # description: > + # Describes status of the Feed. Should be one of + # * `active` Feed should be used in public trip planners. + # * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. + # * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. + # * `development` Feed is being used for development purposes and should not be used in public trip planners. + # * `future` Feed is not yet active but will be in the future + # type: string + # enum: + # - active + # - deprecated + # - inactive + # - development + # - future + # example: active + BasicDataset: + type: object + properties: + id: + description: Unique identifier used as a key for the datasets table. + type: string + example: mdb-10-202402080058 + feed_id: + description: ID of the feed related to this dataset. + type: string + example: mdb-10 + GtfsDataset: + allOf: + - $ref: "#/components/schemas/BasicDataset" + - type: object + properties: + hosted_url: + description: The URL of the dataset data as hosted by MobilityData. No authentication required. + type: string + example: https://storage.googleapis.com/storage/v1/b/mdb-latest/o/us-maine-casco-bay-lines-gtfs-1.zip?alt=media + note: + description: A note to clarify complex use cases for consumers. + type: string + downloaded_at: + description: The date and time the dataset was downloaded from the producer, in ISO 8601 date-time format. + type: string + example: 2023-07-10T22:06:00Z + format: date-time + hash: + description: A hash of the dataset. + type: string + example: 6497e85e34390b8b377130881f2f10ec29c18a80dd6005d504a2038cdd00aa71 + bounding_box: + $ref: "#/components/schemas/BoundingBox" + validation_report: + $ref: "#/components/schemas/ValidationReport" + service_date_range_start: + description: The start date of the service date range for the dataset in UTC. Timing starts at 00:00:00 of the day. + type: string + example: 2023-07-10T06:00:00Z + format: date-time + service_date_range_end: + description: The start date of the service date range for the dataset in UTC. Timing ends at 23:59:59 of the day. + type: string + example: 2023-07-10T05:59:59+00Z + format: date-time + agency_timezone: + description: The timezone of the agency. + type: string + example: America/Los_Angeles + zipped_folder_size_mb: + description: The size of the zipped folder in MB. + type: number + example: 100.2 + unzipped_folder_size_mb: + description: The size of the unzipped folder in MB. + type: number + example: 200.5 + BoundingBox: + description: Bounding box of the dataset when it was first added to the catalog. + type: object + properties: + minimum_latitude: + description: The minimum latitude for the dataset bounding box. + type: number + example: 33.721601 + maximum_latitude: + description: The maximum latitude for the dataset bounding box. + type: number + example: 34.323077 + minimum_longitude: + description: The minimum longitude for the dataset bounding box. + type: number + example: -118.882829 + maximum_longitude: + description: The maximum longitude for the dataset bounding box. + type: number + example: -118.131748 + GtfsDatasets: + type: array + items: + $ref: "#/components/schemas/GtfsDataset" + Metadata: + type: object + properties: + version: + type: string + example: 1.0.0 + commit_hash: + type: string + example: 8635fdac4fbff025b4eaca6972fcc9504bc1552d + ValidationReport: + description: Validation report + type: object + properties: + validated_at: + description: The date and time the report was generated, in ISO 8601 date-time format. + type: string + example: 2023-07-10T22:06:00Z + format: date-time + features: + description: List of GTFS features associated to the dataset. More information, https://gtfs.org/getting-started/features/overview + type: array + items: + type: string + example: ["Shapes", "Headsigns", "Wheelchair Accessibility"] + validator_version: + type: string + example: 4.2.0 + total_error: + type: integer + example: 10 + minimum: 0 + total_warning: + type: integer + example: 20 + minimum: 0 + total_info: + type: integer + example: 30 + minimum: 0 + unique_error_count: + type: integer + example: 1 + minimum: 0 + unique_warning_count: + type: integer + example: 2 + minimum: 0 + unique_info_count: + type: integer + example: 3 + minimum: 0 + url_json: + type: string + format: url + description: JSON validation report URL + example: https://storage.googleapis.com/mobilitydata-datasets-dev/mdb-10/mdb-10-202312181718/mdb-10-202312181718-report-4_2_0.json + url_html: + type: string + format: url + description: HTML validation report URL + example: https://storage.googleapis.com/mobilitydata-datasets-dev/mdb-10/mdb-10-202312181718/mdb-10-202312181718-report-4_2_0.html + OperationFeed: + x-operation: true + allOf: + - $ref: "#/components/schemas/BasicFeed" + - type: object + description: Feed response model. + type: object + properties: + stable_id: + description: Unique stable identifier used as a key for the feeds table. + type: string + example: mdb-1210 + operational_status: + type: string + enum: [wip, published, unpublished] + description: Current operational status of the feed. + status: + $ref: "#/components/schemas/FeedStatus" + OperationGtfsFeed: + x-operation: true + allOf: + - $ref: "#/components/schemas/GtfsFeed" + - type: object + description: Feed response model. + type: object + properties: + stable_id: + description: Unique stable identifier used as a key for the feeds table. + type: string + example: mdb-1210 + operational_status: + type: string + enum: [wip, published, unpublished] + description: Current operational status of the feed. + OperationGtfsRtFeed: + x-operation: true + allOf: + - $ref: "#/components/schemas/GtfsRTFeed" + - type: object + description: Feed response model. + type: object + properties: + stable_id: + description: Unique stable identifier used as a key for the feeds table. + type: string + example: mdb-1210 + operational_status: + type: string + enum: [wip, published, unpublished] + description: Current operational status of the feed. UpdateRequestGtfsRtFeed: + x-operation: true type: object properties: id: @@ -285,8 +1029,8 @@ components: example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express) feed_name: description: > - An optional description of the data feed, e.g to specify if the data feed is an aggregate of - multiple providers, or which network is represented by the feed. + An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. + type: string example: Bus note: @@ -305,10 +1049,20 @@ components: entity_types: type: array items: - $ref: "#/components/schemas/EntityType" + type: string + enum: + - vp + - tu + - sa + example: vp + description: > + The type of realtime entry: + + * vp - vehicle positions + * tu - trip updates + * sa - service alerts feed_references: - description: - A list of the GTFS feeds that the real time source is associated with, represented by their MDB source IDs. + description: A list of the GTFS feeds that the real time source is associated with, represented by their MDB source IDs. type: array items: type: string @@ -327,8 +1081,8 @@ components: - id - status - entity_types - UpdateRequestGtfsFeed: + x-operation: true type: object properties: id: @@ -345,8 +1099,8 @@ components: example: Los Angeles Department of Transportation (LADOT, DASH, Commuter Express) feed_name: description: > - An optional description of the data feed, e.g to specify if the data feed is an aggregate of - multiple providers, or which network is represented by the feed. + An optional description of the data feed, e.g to specify if the data feed is an aggregate of multiple providers, or which network is represented by the feed. + type: string example: Bus note: @@ -375,71 +1129,12 @@ components: required: - id - status - - ExternalIds: - type: array - items: - $ref: "#/components/schemas/ExternalId" - - ExternalId: - type: object - properties: - external_id: - description: The ID that can be use to find the feed data in an external or legacy database. - type: string - example: 1210 - source: - description: The source of the external ID, e.g. the name of the database where the external ID can be used. - type: string - example: mdb - - SourceInfo: - type: object - properties: - producer_url: - description: > - URL where the producer is providing the dataset. - Refer to the authentication information to know how to access this URL. - type: string - format: url - example: https://ladotbus.com/gtfs - authentication_type: - $ref: "#/components/schemas/Authentication_type" - authentication_info_url: - description: > - Contains a URL to a human-readable page describing how the authentication should be performed and how credentials can be created. - This field is required for `authentication_type=1` and `authentication_type=2`. - type: string - format: url - example: https://apidevelopers.ladottransit.com - api_key_parameter_name: - type: string - description: > - Defines the name of the parameter to pass in the URL to provide the API key. - This field is required for `authentication_type=1` and `authentication_type=2`. - example: Ocp-Apim-Subscription-Key - license_url: - description: A URL where to find the license for the feed. - type: string - format: url - example: https://www.ladottransit.com/dla.html - - EntityType: - type: string - enum: - - vp - - tu - - sa - example: vp - description: > - The type of realtime entry: - * vp - vehicle positions - * tu - trip updates - * sa - service alerts - FeedStatus: + x-operation: true description: > Describes status of the Feed. Should be one of + + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -453,10 +1148,12 @@ components: - development - future example: active - DataType: + x-operation: true description: > Describes data type of a feed. Should be one of + + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. @@ -466,42 +1163,20 @@ components: - gtfs_rt - gbfs example: gtfs - - Authentication_type: - description: > - Defines the type of authentication required to access the `producer_url`. Valid values for this field are: - * 0 or (empty) - No authentication required. - * 1 - The authentication requires an API key, which should be passed as value of the parameter api_key_parameter_name in the URL. Please visit URL in authentication_info_url for more information. - * 2 - The authentication requires an HTTP header, which should be passed as the value of the header api_key_parameter_name in the HTTP request. - When not provided, the authentication type is assumed to be 0. - type: integer - enum: - - 0 - - 1 - - 2 - example: 2 - - Location: - type: object - properties: - country_code: - type: string - description: ISO 3166-1 alpha-2 country code. - country: - type: string - description: English name of the country. - subdivision_name: - type: string - description: Name of the subdivision (state/province/region). - municipality: - type: string - description: Name of the municipality. - + parameters: + feed_id_path_param: + x-operation: true + name: id + in: path + description: The feed ID of the requested feed. + required: True + schema: + type: string + example: mdb-1210 securitySchemes: ApiKeyAuth: type: apiKey name: x-api-key in: header - security: - - ApiKeyAuth: [ ] + - ApiKeyAuth: [] diff --git a/functions-python/.flake8 b/functions-python/.flake8 index 36d003839..178b8601f 100644 --- a/functions-python/.flake8 +++ b/functions-python/.flake8 @@ -1,5 +1,5 @@ [flake8] max-line-length = 120 -exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache,venv,build,.*,database_gen,feeds_operations_gen,shared +exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache,venv,build,.*,database_gen,feeds_gen,shared # Ignored because conflict with black extend-ignore = E203 \ No newline at end of file diff --git a/functions-python/feed_sync_process_transitland/tests/test_feed_sync_process.py b/functions-python/feed_sync_process_transitland/tests/test_feed_sync_process.py index 7071da4b9..87b5f06fd 100644 --- a/functions-python/feed_sync_process_transitland/tests/test_feed_sync_process.py +++ b/functions-python/feed_sync_process_transitland/tests/test_feed_sync_process.py @@ -150,9 +150,13 @@ def test_check_feed_url_exists_comprehensive(self, processor): result = processor._check_feed_url_exists(test_url) assert result is True - def test_database_error_handling(self, processor, feed_payload): + @patch("main.check_url_status") + def test_database_error_handling( + self, mock_check_url_status, processor, feed_payload + ): """Test database error handling in different scenarios.""" + mock_check_url_status.return_value = True # Test case 1: General database error during feed processing processor.session.query.side_effect = SQLAlchemyError("Database error") processor._rollback_transaction = MagicMock(return_value=None) diff --git a/functions-python/helpers/query_helper.py b/functions-python/helpers/query_helper.py index 5b95726ec..afe367567 100644 --- a/functions-python/helpers/query_helper.py +++ b/functions-python/helpers/query_helper.py @@ -78,6 +78,7 @@ def get_feeds_query( data_type: str | None = None, limit: int | None = None, offset: int | None = None, + model: Type[Feed | Gtfsfeed | Gtfsrealtimefeed] | None = Feed, ) -> Query: """ Build a consolidated query for feeds with filtering options. @@ -94,25 +95,19 @@ def get_feeds_query( """ try: logging.info( - "Building query with params: data_type=%s, operation_status=%s", + "Building query with params: data_type=%s, operation_status=%s and model=%s", data_type, operation_status, + model.__name__, ) - - if data_type == "gtfs": - model = Gtfsfeed - elif data_type == "gtfs_rt": - model = Gtfsrealtimefeed # Force concrete model - else: - model = Feed - - logging.info(f"Using concrete model: {model.__name__}") - conditions = [] if data_type is None: conditions.append(model.data_type.in_(["gtfs", "gtfs_rt"])) logging.info("Added filter to exclude gbfs feeds") + else: + conditions.append(model.data_type == data_type) + logging.info("Added data_type filter: %s", data_type) if operation_status: conditions.append(model.operational_status == operation_status) diff --git a/functions-python/operations_api/.coveragerc b/functions-python/operations_api/.coveragerc index 778fe7512..65d63464b 100644 --- a/functions-python/operations_api/.coveragerc +++ b/functions-python/operations_api/.coveragerc @@ -4,7 +4,7 @@ omit = */helpers/* */database_gen/* */dataset_service/* - */feeds_operations_gen/* + */feeds_gen/* */shared/* [report] diff --git a/functions-python/operations_api/.gitignore b/functions-python/operations_api/.gitignore index e139d59ad..cf71b055d 100644 --- a/functions-python/operations_api/.gitignore +++ b/functions-python/operations_api/.gitignore @@ -1,2 +1,2 @@ # Generated files -src/feeds_operations_gen \ No newline at end of file +src/feeds_gen \ No newline at end of file diff --git a/functions-python/operations_api/.openapi-generator/FILES b/functions-python/operations_api/.openapi-generator/FILES index 1de8effcb..79307ea11 100644 --- a/functions-python/operations_api/.openapi-generator/FILES +++ b/functions-python/operations_api/.openapi-generator/FILES @@ -1,22 +1,37 @@ +src/feeds_gen/apis/__init__.py +src/feeds_gen/apis/operations_api.py +src/feeds_gen/apis/operations_api_base.py +src/feeds_gen/main.py +src/feeds_gen/models/__init__.py +src/feeds_gen/models/basic_dataset.py +src/feeds_gen/models/basic_feed.py +src/feeds_gen/models/bounding_box.py +src/feeds_gen/models/data_type.py +src/feeds_gen/models/entity_type.py +src/feeds_gen/models/external_id.py +src/feeds_gen/models/extra_models.py +src/feeds_gen/models/feed.py +src/feeds_gen/models/feed_status.py +src/feeds_gen/models/gbfs_endpoint.py +src/feeds_gen/models/gbfs_feed.py +src/feeds_gen/models/gbfs_validation_report.py +src/feeds_gen/models/gbfs_version.py +src/feeds_gen/models/get_feeds200_response.py +src/feeds_gen/models/gtfs_dataset.py +src/feeds_gen/models/gtfs_feed.py +src/feeds_gen/models/gtfs_rt_feed.py +src/feeds_gen/models/latest_dataset.py +src/feeds_gen/models/latest_dataset_validation_report.py +src/feeds_gen/models/location.py +src/feeds_gen/models/metadata.py +src/feeds_gen/models/operation_feed.py +src/feeds_gen/models/operation_gtfs_feed.py +src/feeds_gen/models/operation_gtfs_rt_feed.py +src/feeds_gen/models/redirect.py +src/feeds_gen/models/search_feed_item_result.py +src/feeds_gen/models/source_info.py +src/feeds_gen/models/update_request_gtfs_feed.py +src/feeds_gen/models/update_request_gtfs_rt_feed.py +src/feeds_gen/models/validation_report.py +src/feeds_gen/security_api.py src/feeds_operations/impl/__init__.py -src/feeds_operations_gen/apis/__init__.py -src/feeds_operations_gen/apis/operations_api.py -src/feeds_operations_gen/apis/operations_api_base.py -src/feeds_operations_gen/main.py -src/feeds_operations_gen/models/__init__.py -src/feeds_operations_gen/models/authentication_type.py -src/feeds_operations_gen/models/base_feed.py -src/feeds_operations_gen/models/data_type.py -src/feeds_operations_gen/models/entity_type.py -src/feeds_operations_gen/models/external_id.py -src/feeds_operations_gen/models/extra_models.py -src/feeds_operations_gen/models/feed_status.py -src/feeds_operations_gen/models/get_feeds200_response.py -src/feeds_operations_gen/models/gtfs_feed_response.py -src/feeds_operations_gen/models/gtfs_rt_feed_response.py -src/feeds_operations_gen/models/location.py -src/feeds_operations_gen/models/redirect.py -src/feeds_operations_gen/models/source_info.py -src/feeds_operations_gen/models/update_request_gtfs_feed.py -src/feeds_operations_gen/models/update_request_gtfs_rt_feed.py -src/feeds_operations_gen/security_api.py diff --git a/functions-python/operations_api/function_config.json b/functions-python/operations_api/function_config.json index af52c991b..e44ebfb10 100644 --- a/functions-python/operations_api/function_config.json +++ b/functions-python/operations_api/function_config.json @@ -6,7 +6,7 @@ "memory": "1Gi", "trigger_http": true, "include_folders": ["helpers"], - "include_api_folders": ["database_gen", "database", "common"], + "include_api_folders": ["database_gen", "database", "common", "db_models"], "environment_variables": [ { "key": "GOOGLE_CLIENT_ID" diff --git a/functions-python/operations_api/requirements.txt b/functions-python/operations_api/requirements.txt index f395f23fa..edbba8cfc 100644 --- a/functions-python/operations_api/requirements.txt +++ b/functions-python/operations_api/requirements.txt @@ -31,3 +31,4 @@ psycopg2-binary==2.9.6 cachetools deepdiff fastapi_filter +pycountry \ No newline at end of file diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index d2a26ace7..12cdd7fd5 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -19,55 +19,41 @@ from deepdiff import DeepDiff from fastapi import HTTPException -from pydantic import Field +from pydantic import Field, StrictStr from sqlalchemy.orm import Session from starlette.responses import Response -from feeds_operations.impl.models.get_feeds_response import GetFeeds200Response -from feeds_operations.impl.models.gtfs_feed_impl import GtfsFeedImpl -from feeds_operations.impl.models.gtfs_rt_feed_impl import GtfsRtFeedImpl +from feeds_gen.models.data_type import DataType +from feeds_gen.models.get_feeds200_response import GetFeeds200Response +from feeds_gen.models.operation_gtfs_feed import OperationGtfsFeed +from feeds_gen.models.operation_gtfs_rt_feed import OperationGtfsRtFeed from feeds_operations.impl.models.update_request_gtfs_feed_impl import ( UpdateRequestGtfsFeedImpl, ) from feeds_operations.impl.models.update_request_gtfs_rt_feed_impl import ( UpdateRequestGtfsRtFeedImpl, ) -from feeds_operations_gen.apis.operations_api_base import BaseOperationsApi -from feeds_operations_gen.models.data_type import DataType -from feeds_operations_gen.models.gtfs_feed_response import GtfsFeedResponse -from feeds_operations_gen.models.gtfs_rt_feed_response import GtfsRtFeedResponse -from feeds_operations_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed -from feeds_operations_gen.models.update_request_gtfs_rt_feed import ( +from feeds_gen.apis.operations_api_base import BaseOperationsApi +from feeds_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_gen.models.update_request_gtfs_rt_feed import ( UpdateRequestGtfsRtFeed, ) from shared.database.database import with_db_session, refresh_materialized_view -from shared.database_gen.sqlacodegen_models import Gtfsfeed, t_feedsearch +from shared.database_gen.sqlacodegen_models import Gtfsfeed, t_feedsearch, Feed from shared.helpers.query_helper import ( query_feed_by_stable_id, get_feeds_query, ) +from shared.helpers.src.shared.database_gen.sqlacodegen_models import Gtfsrealtimefeed +from .models.operation_feed_impl import OperationFeedImpl +from .models.operation_gtfs_feed_impl import OperationGtfsFeedImpl +from .models.operation_gtfs_rt_feed_impl import OperationGtfsRtFeedImpl from .request_validator import validate_request class OperationsApiImpl(BaseOperationsApi): """Implementation of the operations API.""" - def process_feed(self, feed) -> GtfsFeedResponse | GtfsRtFeedResponse: - """Process a feed into the appropriate response type using fromOrm methods.""" - logging.debug("Processing feed %s with type %s", feed.stable_id, feed.data_type) - - if feed.data_type == "gtfs": - result = GtfsFeedImpl.from_orm(feed) - logging.debug("Successfully processed GTFS feed %s", feed.stable_id) - return result - elif feed.data_type == "gtfs_rt": - result = GtfsRtFeedImpl.from_orm(feed) - logging.debug("Successfully processed GTFS-RT feed %s", feed.stable_id) - return result - - logging.error("Unsupported feed type: %s", feed.data_type) - raise ValueError(f"Unsupported feed type: {feed.data_type}") - @with_db_session async def get_feeds( self, @@ -88,6 +74,7 @@ async def get_feeds( data_type=data_type, limit=limit_int, offset=offset_int, + model=Feed, ) logging.info("Executing query with data_type: %s", data_type) @@ -98,8 +85,7 @@ async def get_feeds( feed_list = [] for feed in feeds: - processed_feed = self.process_feed(feed) - feed_list.append(processed_feed) + feed_list.append(OperationFeedImpl.from_orm(feed)) response = GetFeeds200Response( total=total, offset=offset_int, limit=limit_int, feeds=feed_list @@ -113,6 +99,40 @@ async def get_feeds( status_code=500, detail=f"Internal server error: {str(e)}" ) + @with_db_session + async def get_gtfs_feed( + self, + id: Annotated[ + StrictStr, Field(description="The feed ID of the requested feed.") + ], + db_session: Session = None, + ) -> OperationGtfsFeed: + """Get the specified GTFS feed from the Mobility Database.""" + gtfs_feed = ( + db_session.query(Gtfsfeed).filter(Gtfsfeed.stable_id == id).one_or_none() + ) + if gtfs_feed is None: + raise HTTPException(status_code=404, detail="GTFS feed not found") + return OperationGtfsFeedImpl.from_orm(gtfs_feed) + + @with_db_session + async def get_gtfs_rt_feed( + self, + id: Annotated[ + StrictStr, Field(description="The feed ID of the requested feed.") + ], + db_session: Session = None, + ) -> OperationGtfsRtFeed: + """Get the specified GTFS-RT feed from the Mobility Database.""" + gtfs_rt_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == id) + .one_or_none() + ) + if gtfs_rt_feed is None: + raise HTTPException(status_code=404, detail="GTFS-RT feed not found") + return OperationGtfsRtFeedImpl.from_orm(gtfs_rt_feed) + @staticmethod def detect_changes( feed: Gtfsfeed, diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/basic_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/basic_feed_impl.py deleted file mode 100644 index a3dbfe6a7..000000000 --- a/functions-python/operations_api/src/feeds_operations/impl/models/basic_feed_impl.py +++ /dev/null @@ -1,59 +0,0 @@ -from feeds_operations.impl.models.external_id_impl import ExternalIdImpl -from feeds_operations.impl.models.location_impl import LocationImpl -from feeds_operations.impl.models.redirect_impl import RedirectImpl -from feeds_operations_gen.models.base_feed import BaseFeed -from feeds_operations_gen.models.source_info import SourceInfo -from shared.database_gen.sqlacodegen_models import Feed - - -class BaseFeedImpl(BaseFeed): - """Base implementation of the feeds models.""" - - class Config: - """Pydantic configuration. - Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. - """ - - from_attributes = True - - @classmethod - def from_orm(cls, feed: Feed | None) -> BaseFeed | None: - """Convert a SQLAlchemy row object to a Pydantic model.""" - if not feed: - return None - - return cls( - id=feed.id, - stable_id=feed.stable_id, - status=feed.status, - data_type=feed.data_type, - provider=feed.provider, - feed_name=feed.feed_name, - note=feed.note, - feed_contact_email=feed.feed_contact_email, - source_info=SourceInfo( - producer_url=feed.producer_url, - authentication_type=None - if feed.authentication_type is None - else int(feed.authentication_type), - authentication_info_url=feed.authentication_info_url, - api_key_parameter_name=feed.api_key_parameter_name, - license_url=feed.license_url, - ), - operational_status=feed.operational_status, - created_at=feed.created_at, - official=feed.official, - official_updated_at=feed.official_updated_at, - locations=sorted( - [LocationImpl.from_orm(item) for item in feed.locations], - key=lambda x: (x.country_code or "", x.municipality or ""), - ), - external_ids=sorted( - [ExternalIdImpl.from_orm(item) for item in feed.externalids], - key=lambda x: x.external_id, - ), - redirects=sorted( - [RedirectImpl.from_orm(item) for item in feed.redirectingids], - key=lambda x: x.target_id, - ), - ) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/get_feeds_response.py b/functions-python/operations_api/src/feeds_operations/impl/models/get_feeds_response.py deleted file mode 100644 index aa7f37017..000000000 --- a/functions-python/operations_api/src/feeds_operations/impl/models/get_feeds_response.py +++ /dev/null @@ -1,93 +0,0 @@ -# coding: utf-8 - -""" - Mobility Database Catalog Operations - - API for the Mobility Database Catalog Operations. See [https://mobilitydatabase.org/] - (https://mobilitydatabase.org/). This API was designed for internal use and is not intended to be used by - the general public. The Mobility Database Operation API uses Auth2.0 authentication. - - The version of the OpenAPI document: 1.0.0 - Contact: api@mobilitydata.org - Generated by OpenAPI Generator (https://openapi-generator.tech) - - Do not edit the class manually. -""" - -from __future__ import annotations - -import json -import pprint -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, Field, StrictInt - -try: - from typing import Self -except ImportError: - from typing_extensions import Self - -from feeds_operations_gen.models.base_feed import BaseFeed - - -class GetFeeds200Response(BaseModel): - """Response model for get_feeds endpoint""" - - total: Optional[StrictInt] = Field( - default=None, description="Total number of feeds matching the criteria." - ) - offset: Optional[StrictInt] = Field( - default=None, description="Current offset for pagination." - ) - limit: Optional[StrictInt] = Field( - default=None, description="Maximum number of items per page." - ) - feeds: Optional[List[BaseFeed]] = Field( - default=None, description="List of feeds using polymorphic serialization" - ) - - model_config = { - "populate_by_name": True, - "validate_assignment": True, - "protected_namespaces": (), - } - - def to_str(self) -> str: - """Returns the string representation of the model using alias""" - return pprint.pformat(self.model_dump(by_alias=True)) - - def to_json(self) -> str: - """Returns the JSON representation of the model using alias""" - return json.dumps(self.to_dict()) - - def to_dict(self) -> Dict[str, Any]: - """Return the dictionary representation of the model using alias""" - _dict = self.model_dump( - by_alias=True, - exclude={}, - exclude_none=True, - ) - if self.feeds: - _dict["feeds"] = [feed.to_dict() for feed in self.feeds] - return _dict - - @classmethod - def from_dict(cls, obj: Dict) -> Self: - """Create an instance of GetFeeds200Response from a dict""" - if obj is None: - return None - - if not isinstance(obj, dict): - return cls.model_validate(obj) - - _obj = cls.model_validate( - { - "total": obj.get("total"), - "offset": obj.get("offset"), - "limit": obj.get("limit"), - "feeds": [BaseFeed.from_dict(feed) for feed in obj.get("feeds", [])] - if obj.get("feeds") - else None, - } - ) - return _obj diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_feed_impl.py deleted file mode 100644 index 9903064ad..000000000 --- a/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_feed_impl.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import Optional, List - -from pydantic import Field - -from feeds_operations_gen.models.gtfs_feed_response import GtfsFeedResponse as GtfsFeed -from shared.database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm -from .basic_feed_impl import BaseFeedImpl - - -class GtfsFeedImpl(BaseFeedImpl, GtfsFeed): - """Implementation of the GTFS feed model.""" - - entity_types: Optional[List[str]] = Field( - default=None, exclude=True, description="Not used in GTFS feeds" - ) - feed_references: Optional[List[str]] = Field( - default=None, exclude=True, description="Not used in GTFS feeds" - ) - - class Config: - """Pydantic configuration. - Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. - """ - - from_attributes = True - - @classmethod - def from_orm(cls, feed: GtfsfeedOrm | None) -> GtfsFeed | None: - """Convert ORM object to Pydantic model.""" - if not feed: - return None - - gtfs_feed = super().from_orm(feed) - if not gtfs_feed: - return None - - return cls.model_construct(**gtfs_feed.__dict__) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_rt_feed_impl.py deleted file mode 100644 index 5ff0212bf..000000000 --- a/functions-python/operations_api/src/feeds_operations/impl/models/gtfs_rt_feed_impl.py +++ /dev/null @@ -1,56 +0,0 @@ -from typing import List - -from pydantic import Field - -from feeds_operations_gen.models.gtfs_rt_feed_response import ( - GtfsRtFeedResponse as GtfsRtFeed, -) -from shared.database_gen.sqlacodegen_models import Gtfsrealtimefeed as GtfsRTFeedOrm -from .basic_feed_impl import BaseFeedImpl - - -class GtfsRtFeedImpl(BaseFeedImpl, GtfsRtFeed): - """Implementation of the GTFS-RT feed model.""" - - entity_types: List[str] = Field( - default_factory=list, description="Types of GTFS-RT entities" - ) - feed_references: List[str] = Field( - default_factory=list, description="References to related GTFS feeds" - ) - - class Config: - """Pydantic configuration. - Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. - """ - - from_attributes = True - - @classmethod - def from_orm(cls, feed: GtfsRTFeedOrm | None) -> GtfsRtFeed | None: - """Convert ORM object to Pydantic model without validation.""" - if not feed: - return None - - gtfs_rt_feed = super().from_orm(feed) - if not gtfs_rt_feed: - return None - - gtfs_rt_feed_dict = gtfs_rt_feed.model_dump() - - gtfs_rt_feed_dict["entity_types"] = [] - gtfs_rt_feed_dict["feed_references"] = [] - - if hasattr(feed, "entitytypes"): - entity_types = [item.name for item in (feed.entitytypes or [])] - gtfs_rt_feed_dict["entity_types"] = ( - sorted(entity_types) if entity_types else [] - ) - - if hasattr(feed, "gtfs_feeds"): - feed_references = [item.stable_id for item in (feed.gtfs_feeds or [])] - gtfs_rt_feed_dict["feed_references"] = ( - sorted(feed_references) if feed_references else [] - ) - - return cls.model_construct(**gtfs_rt_feed_dict) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/location_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/location_impl.py deleted file mode 100644 index f4f65af1a..000000000 --- a/functions-python/operations_api/src/feeds_operations/impl/models/location_impl.py +++ /dev/null @@ -1,23 +0,0 @@ -from feeds_operations_gen.models.location import Location -from shared.database_gen.sqlacodegen_models import Location as LocationOrm - - -class LocationImpl(Location): - class Config: - """Pydantic configuration. - Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. - """ - - from_attributes = True - - @classmethod - def from_orm(cls, location: LocationOrm | None) -> Location | None: - """Create a model instance from a SQLAlchemy a Location row object.""" - if not location: - return None - return cls( - country_code=location.country_code, - country=location.country, - subdivision_name=location.subdivision_name, - municipality=location.municipality, - ) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/entity_type_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py similarity index 92% rename from functions-python/operations_api/src/feeds_operations/impl/models/entity_type_impl.py rename to functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py index 02db9705b..1b186217e 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/entity_type_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py @@ -1,10 +1,10 @@ from pydantic import BaseModel -from feeds_operations_gen.models.entity_type import EntityType +from feeds_gen.models.entity_type import EntityType from shared.database_gen.sqlacodegen_models import Entitytype as EntityTypeOrm -class EntityTypeImpl(BaseModel): +class OperationEntityTypeImpl(BaseModel): """Implementation of the EntityType model. This class converts a SQLAlchemy row DB object with the gtfs feed fields to a Pydantic model. """ diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/external_id_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_external_id_impl.py similarity index 88% rename from functions-python/operations_api/src/feeds_operations/impl/models/external_id_impl.py rename to functions-python/operations_api/src/feeds_operations/impl/models/operation_external_id_impl.py index 9f66a83d5..f5146d3ea 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/external_id_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_external_id_impl.py @@ -20,10 +20,11 @@ Gtfsrealtimefeed, Gbfsfeed, ) -from feeds_operations_gen.models.external_id import ExternalId +from feeds_gen.models.external_id import ExternalId +from shared.db_models.external_id_impl import ExternalIdImpl -class ExternalIdImpl(ExternalId): +class OperationExternalIdImpl(ExternalIdImpl, ExternalId): """Implementation of the `ExternalId` model. This class converts a SQLAlchemy row DB object to a Pydantic model. """ @@ -42,10 +43,7 @@ def from_orm(cls, external_id: Externalid | None) -> ExternalId | None: """ if not external_id: return None - return cls( - external_id=external_id.associated_id, - source=external_id.source, - ) + return super().from_orm(external_id) @classmethod def to_orm( diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py new file mode 100644 index 000000000..0f0d888cc --- /dev/null +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py @@ -0,0 +1,33 @@ +from feeds_gen.models.operation_feed import OperationFeed +from shared.database_gen.sqlacodegen_models import Feed +from shared.db_models.basic_feed_impl import BasicFeedImpl + + +class OperationFeedImpl(BasicFeedImpl, OperationFeed): + """Base implementation of the feeds models.""" + + class Config: + """Pydantic configuration. + Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. + """ + + from_attributes = True + + @classmethod + def from_orm(cls, feed: Feed | None) -> OperationFeed | None: + """Convert a SQLAlchemy row object to a Pydantic model.""" + if not feed: + return None + operation_feed = super().from_orm(feed) + if not operation_feed: + return None + + data = dict(operation_feed.__dict__) + # Override id and add stable_id + data["id"] = feed.id + data["stable_id"] = feed.stable_id + # Add missing fields from public API model + data["status"] = feed.status + data["operational_status"] = feed.operational_status + + return cls.model_construct(**data) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/redirect_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_redirect_impl.py similarity index 90% rename from functions-python/operations_api/src/feeds_operations/impl/models/redirect_impl.py rename to functions-python/operations_api/src/feeds_operations/impl/models/operation_redirect_impl.py index 74cf869c6..f422e90a9 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/redirect_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_redirect_impl.py @@ -20,11 +20,12 @@ Gbfsfeed, Gtfsrealtimefeed, ) -from feeds_operations_gen.models.redirect import Redirect +from feeds_gen.models.redirect import Redirect +from shared.db_models.redirect_impl import RedirectImpl from shared.helpers.query_helper import query_feed_by_stable_id -class RedirectImpl(Redirect): +class OperationRedirectImpl(RedirectImpl, Redirect): """Implementation of the `Redirect` model. This class converts a SQLAlchemy row DB object to a Pydantic model. """ @@ -43,10 +44,7 @@ def from_orm(cls, redirect: Redirectingid | None) -> Redirect | None: """ if not redirect: return None - return cls( - target_id=redirect.target.stable_id, - comment=redirect.redirect_comment, - ) + return super().from_orm(redirect) @classmethod def to_orm( diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py index 0a20199e9..5ee106237 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_feed_impl.py @@ -14,11 +14,14 @@ # limitations under the License. # -from feeds_operations.impl.models.external_id_impl import ExternalIdImpl -from feeds_operations.impl.models.redirect_impl import RedirectImpl -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_operations.impl.models.operation_external_id_impl import ( + OperationExternalIdImpl, +) +from feeds_operations.impl.models.operation_redirect_impl import OperationRedirectImpl from shared.database_gen.sqlacodegen_models import Gtfsfeed +from shared.db_models.redirect_impl import RedirectImpl class UpdateRequestGtfsFeedImpl(UpdateRequestGtfsFeed): @@ -61,7 +64,7 @@ def from_orm(cls, obj: Gtfsfeed | None) -> UpdateRequestGtfsFeed | None: key=lambda x: x.target_id, ), external_ids=sorted( - [ExternalIdImpl.from_orm(item) for item in obj.externalids], + [OperationExternalIdImpl.from_orm(item) for item in obj.externalids], key=lambda x: x.external_id, ), official=obj.official, @@ -94,7 +97,7 @@ def to_orm( update_request.source_info is None or update_request.source_info.authentication_type is None ) - else str(update_request.source_info.authentication_type.value) + else str(update_request.source_info.authentication_type) ) entity.authentication_info_url = ( None @@ -125,7 +128,7 @@ def to_orm( [] if update_request.redirects is None else [ - RedirectImpl.to_orm(item, entity, session) + OperationRedirectImpl.to_orm(item, entity, session) for item in update_request.redirects ] ) @@ -136,7 +139,7 @@ def to_orm( [] if update_request.external_ids is None else [ - ExternalIdImpl.to_orm(item, entity) + OperationExternalIdImpl.to_orm(item, entity) for item in update_request.external_ids ] ) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py index d21963f5b..373991e07 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/update_request_gtfs_rt_feed_impl.py @@ -14,14 +14,20 @@ # limitations under the License. # -from feeds_operations.impl.models.entity_type_impl import EntityTypeImpl -from feeds_operations.impl.models.external_id_impl import ExternalIdImpl -from feeds_operations.impl.models.redirect_impl import RedirectImpl -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_rt_feed import ( +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_rt_feed import ( UpdateRequestGtfsRtFeed, ) +from feeds_operations.impl.models.operation_entity_type_impl import ( + OperationEntityTypeImpl, +) +from feeds_operations.impl.models.operation_external_id_impl import ( + OperationExternalIdImpl, +) +from feeds_operations.impl.models.operation_redirect_impl import OperationRedirectImpl from shared.database_gen.sqlacodegen_models import Gtfsfeed, Gtfsrealtimefeed +from shared.db_models.external_id_impl import ExternalIdImpl +from shared.db_models.redirect_impl import RedirectImpl class UpdateRequestGtfsRtFeedImpl(UpdateRequestGtfsRtFeed): @@ -68,7 +74,9 @@ def from_orm(cls, obj: Gtfsrealtimefeed | None) -> UpdateRequestGtfsRtFeed | Non key=lambda x: x.external_id, ), entity_types=sorted( - [EntityTypeImpl.from_orm(item) for item in obj.entitytypes] + [OperationEntityTypeImpl.from_orm(item) for item in obj.entitytypes] + if obj.entitytypes + else [] ), feed_references=sorted([item.stable_id for item in obj.gtfs_feeds]), official=obj.official, @@ -101,7 +109,7 @@ def to_orm( update_request.source_info is None or update_request.source_info.authentication_type is None ) - else str(update_request.source_info.authentication_type.value) + else str(update_request.source_info.authentication_type) ) entity.authentication_info_url = ( None @@ -132,7 +140,7 @@ def to_orm( [] if update_request.redirects is None else [ - RedirectImpl.to_orm(item, entity, session) + OperationRedirectImpl.to_orm(item, entity, session) for item in update_request.redirects ] ) @@ -143,7 +151,7 @@ def to_orm( [] if update_request.external_ids is None else [ - ExternalIdImpl.to_orm(item, entity) + OperationExternalIdImpl.to_orm(item, entity) for item in update_request.external_ids ] ) @@ -151,7 +159,7 @@ def to_orm( [] if update_request.entity_types is None else [ - EntityTypeImpl.to_orm(item, session) + OperationEntityTypeImpl.to_orm(item, session) for item in update_request.entity_types ] ) diff --git a/functions-python/operations_api/src/main.py b/functions-python/operations_api/src/main.py index 14ce372d0..7a24f2eec 100644 --- a/functions-python/operations_api/src/main.py +++ b/functions-python/operations_api/src/main.py @@ -16,7 +16,7 @@ from flask import Request, Response from fastapi import FastAPI -from feeds_operations_gen.apis.operations_api import router as FeedsApiRouter +from feeds_gen.apis.operations_api import router as FeedsApiRouter import functions_framework import asyncio diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_entity_type_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_entity_type_impl.py deleted file mode 100644 index 915ddae06..000000000 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_entity_type_impl.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import Mock - -from shared.database_gen.sqlacodegen_models import Entitytype -from feeds_operations.impl.models.entity_type_impl import EntityTypeImpl -from feeds_operations_gen.models.entity_type import EntityType - - -def test_from_orm(): - entity_type = Entitytype(name="VP") - result = EntityTypeImpl.from_orm(entity_type) - assert result.name == "VP" - - -def test_from_orm_none(): - result = EntityTypeImpl.from_orm(None) - assert result is None - - -def test_to_orm(): - entity_type = EntityType("vp") - session = Mock() - mock_query = Mock() - resulting_entity = Mock() - mock_query.filter.return_value.first.return_value = resulting_entity - session.query.return_value = mock_query - result = EntityTypeImpl.to_orm(entity_type, session) - assert result == resulting_entity diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_external_id_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_external_id_impl.py deleted file mode 100644 index 59255eeaa..000000000 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_external_id_impl.py +++ /dev/null @@ -1,26 +0,0 @@ -from shared.database_gen.sqlacodegen_models import Externalid, Gtfsfeed -from feeds_operations_gen.models.external_id import ExternalId -from feeds_operations.impl.models.external_id_impl import ( - ExternalIdImpl, -) - - -def test_from_orm(): - external_id = Externalid(associated_id="12345", source="test_source") - result = ExternalIdImpl.from_orm(external_id) - assert result.external_id == "12345" - assert result.source == "test_source" - - -def test_from_orm_none(): - result = ExternalIdImpl.from_orm(None) - assert result is None - - -def test_to_orm(): - external_id = ExternalId(external_id="12345", source="test_source") - feed = Gtfsfeed(id=1) - result = ExternalIdImpl.to_orm(external_id, feed) - assert result.feed_id == 1 - assert result.associated_id == "12345" - assert result.source == "test_source" diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_feed_responses.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_feed_responses.py index 95dabb5b8..c304e9c94 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_feed_responses.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_feed_responses.py @@ -17,8 +17,10 @@ from datetime import datetime -from feeds_operations.impl.models.gtfs_feed_impl import GtfsFeedImpl -from feeds_operations.impl.models.gtfs_rt_feed_impl import GtfsRtFeedImpl +from feeds_operations.impl.models.operation_gtfs_feed_impl import OperationGtfsFeedImpl +from feeds_operations.impl.models.operation_gtfs_rt_feed_impl import ( + OperationGtfsRtFeedImpl, +) def test_gtfs_feed_response_creation(): @@ -34,7 +36,7 @@ def test_gtfs_feed_response_creation(): "created_at": "2024-02-14T12:00:00+00:00", } - feed = GtfsFeedImpl(**feed_data) + feed = OperationGtfsFeedImpl(**feed_data) assert feed.id == "mdb-123" assert feed.data_type == "gtfs" assert feed.operational_status == "wip" @@ -46,10 +48,10 @@ def test_gtfs_feed_response_creation(): def test_gtfs_feed_response_optional_fields(): """Test GtfsFeedResponse with minimal required fields.""" - feed = GtfsFeedImpl() + feed = OperationGtfsFeedImpl() assert feed.id is None assert feed.stable_id is None - assert feed.locations is None + assert feed.locations == [] def test_gtfs_feed_response_locations(): @@ -66,7 +68,7 @@ def test_gtfs_feed_response_locations(): ], } - feed = GtfsFeedImpl(**feed_data) + feed = OperationGtfsFeedImpl(**feed_data) assert len(feed.locations) == 1 assert feed.locations[0].country_code == "US" assert feed.locations[0].municipality == "San Francisco" @@ -74,7 +76,7 @@ def test_gtfs_feed_response_locations(): def test_gtfs_rt_feed_response_optional_fields(): """Test GtfsRtFeedResponse with minimal required fields.""" - feed = GtfsRtFeedImpl() + feed = OperationGtfsRtFeedImpl() assert feed.id is None assert feed.entity_types == [] assert feed.feed_references == [] @@ -84,7 +86,7 @@ def test_gtfs_rt_feed_response_entity_types(): """Test GtfsRtFeedResponse entity_types validation.""" feed_data = {"data_type": "gtfs_rt", "entity_types": ["vp", "tu", "sa"]} - feed = GtfsRtFeedImpl(**feed_data) + feed = OperationGtfsRtFeedImpl(**feed_data) assert len(feed.entity_types) == 3 assert all(et in ["vp", "tu", "sa"] for et in feed.entity_types) @@ -100,8 +102,8 @@ def test_feed_response_serialization(): "entity_types": ["vp"], } - gtfs_feed = GtfsFeedImpl(**gtfs_data) - gtfs_rt_feed = GtfsRtFeedImpl(**gtfs_rt_data) + gtfs_feed = OperationGtfsFeedImpl(**gtfs_data) + gtfs_rt_feed = OperationGtfsRtFeedImpl(**gtfs_rt_data) gtfs_dict = gtfs_feed.to_dict() assert gtfs_dict["id"] == "mdb-123" @@ -159,13 +161,13 @@ def test_feed_response_from_orm(): "feed_references": ["mdb-123"], } - gtfs_feed = GtfsFeedImpl.model_validate(gtfs_feed_data) + gtfs_feed = OperationGtfsFeedImpl.model_validate(gtfs_feed_data) assert gtfs_feed.id == "mdb-123" assert gtfs_feed.data_type == "gtfs" assert isinstance(gtfs_feed.created_at, datetime) assert gtfs_feed.created_at.isoformat() == current_time_str - gtfs_rt_feed = GtfsRtFeedImpl.model_validate(gtfs_rt_feed_data) + gtfs_rt_feed = OperationGtfsRtFeedImpl.model_validate(gtfs_rt_feed_data) assert gtfs_rt_feed.id == "mdb-456" assert gtfs_rt_feed.data_type == "gtfs_rt" assert isinstance(gtfs_rt_feed.created_at, datetime) diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_redirect_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_redirect_impl.py deleted file mode 100644 index 36505c076..000000000 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_redirect_impl.py +++ /dev/null @@ -1,53 +0,0 @@ -import pytest -from unittest.mock import MagicMock -from shared.database_gen.sqlacodegen_models import Redirectingid, Gtfsfeed -from feeds_operations_gen.models.redirect import Redirect -from feeds_operations.impl.models.redirect_impl import RedirectImpl - - -def test_from_orm(): - redirecting_id = Redirectingid( - target=MagicMock(stable_id="target_stable_id"), redirect_comment="Test comment" - ) - result = RedirectImpl.from_orm(redirecting_id) - assert result.target_id == "target_stable_id" - assert result.comment == "Test comment" - - -def test_from_orm_none(): - result = RedirectImpl.from_orm(None) - assert result is None - - -def test_to_orm(): - redirect = Redirect(target_id="target_stable_id", comment="Test comment") - source_feed = Gtfsfeed(id=1, data_type="gtfs") - target_feed = Gtfsfeed(id=2, stable_id="target_stable_id") - session = MagicMock() - session.query.return_value.filter.return_value.first.return_value = target_feed - result = RedirectImpl.to_orm(redirect, source_feed, session) - assert result.source_id == 1 - assert result.target_id == 2 - assert result.redirect_comment == "Test comment" - - -def test_to_orm_invalid_source(): - redirect = Redirect(target_id="target_stable_id", comment="Test comment") - session = MagicMock() - - with pytest.raises( - ValueError, match="Invalid source object or source.id is not set" - ): - RedirectImpl.to_orm(redirect, None, session) - - -def test_to_orm_invalid_target(): - redirect = Redirect(target_id="target_stable_id", comment="Test comment") - source_feed = Gtfsfeed(id=1, data_type="gtfs") - session = MagicMock() - session.query.return_value.filter.return_value.first.return_value = None - - with pytest.raises( - ValueError, match="Invalid target_feed object or target_feed.id is not set" - ): - RedirectImpl.to_orm(redirect, source_feed, session) diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py index 9f505c62a..965729c1d 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_feed_impl.py @@ -1,16 +1,14 @@ from unittest.mock import Mock, MagicMock + +from feeds_operations.impl.models.operation_redirect_impl import OperationRedirectImpl from shared.database_gen.sqlacodegen_models import Gtfsfeed, Redirectingid, Externalid -from feeds_operations_gen.models.authentication_type import AuthenticationType -from feeds_operations_gen.models.feed_status import FeedStatus -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_gen.models.feed_status import FeedStatus +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed from feeds_operations.impl.models.update_request_gtfs_feed_impl import ( UpdateRequestGtfsFeedImpl, ) -from feeds_operations.impl.models.redirect_impl import RedirectImpl -from feeds_operations.impl.models.external_id_impl import ( - ExternalIdImpl, -) +from shared.db_models.external_id_impl import ExternalIdImpl def test_from_orm(): @@ -65,12 +63,14 @@ def test_to_orm(): feed_contact_email="email@example.com", source_info=SourceInfo( producer_url="http://producer.url", - authentication_type=AuthenticationType.NUMBER_1, + authentication_type=1, authentication_info_url="http://auth.info.url", api_key_parameter_name="api_key", license_url="http://license.url", ), - redirects=[RedirectImpl(target_id="target_stable_id", comment="Test comment")], + redirects=[ + OperationRedirectImpl(target_id="target_stable_id", comment="Test comment") + ], external_ids=[ExternalIdImpl(external_id="external_id")], ) entity = Gtfsfeed(id="1", stable_id="stable_id", data_type="gtfs") @@ -104,7 +104,9 @@ def test_to_orm_invalid_source_info(): note="note", feed_contact_email="email@example.com", source_info=None, - redirects=[RedirectImpl(target_id="target_stable_id", comment="Test comment")], + redirects=[ + OperationRedirectImpl(target_id="target_stable_id", comment="Test comment") + ], external_ids=[ExternalIdImpl(external_id="external_id")], ) entity = Gtfsfeed(id="id") diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py index 3cfe6e884..d156fbf5b 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py @@ -9,17 +9,14 @@ from feeds_operations.impl.models.update_request_gtfs_rt_feed_impl import ( UpdateRequestGtfsRtFeedImpl, ) -from feeds_operations_gen.models.authentication_type import AuthenticationType -from feeds_operations_gen.models.entity_type import EntityType -from feeds_operations_gen.models.feed_status import FeedStatus -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_rt_feed import ( +from feeds_gen.models.entity_type import EntityType +from feeds_gen.models.feed_status import FeedStatus +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_rt_feed import ( UpdateRequestGtfsRtFeed, ) -from feeds_operations.impl.models.redirect_impl import RedirectImpl -from feeds_operations.impl.models.external_id_impl import ( - ExternalIdImpl, -) +from shared.db_models.external_id_impl import ExternalIdImpl +from shared.db_models.redirect_impl import RedirectImpl def test_from_orm(): @@ -74,7 +71,7 @@ def test_to_orm(): feed_contact_email="email@example.com", source_info=SourceInfo( producer_url="http://producer.url", - authentication_type=AuthenticationType.NUMBER_1, + authentication_type=1, authentication_info_url="http://auth.info.url", api_key_parameter_name="api_key", license_url="http://license.url", diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py index 227b20767..33fa202aa 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py @@ -4,11 +4,10 @@ from conftest import feed_mdb_40 from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl -from feeds_operations_gen.models.authentication_type import AuthenticationType -from feeds_operations_gen.models.external_id import ExternalId -from feeds_operations_gen.models.feed_status import FeedStatus -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_gen.models.external_id import ExternalId +from feeds_gen.models.feed_status import FeedStatus +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed from shared.database.database import Database from shared.database_gen.sqlacodegen_models import Gtfsfeed from test_shared.test_utils.database_utils import default_db_url @@ -26,9 +25,7 @@ def update_request_gtfs_feed(): feed_contact_email=feed_mdb_40.feed_contact_email, source_info=SourceInfo( producer_url=feed_mdb_40.producer_url, - authentication_type=AuthenticationType( - int(feed_mdb_40.authentication_type) - ), + authentication_type=int(feed_mdb_40.authentication_type), authentication_info_url=feed_mdb_40.authentication_info_url, api_key_parameter_name=feed_mdb_40.api_key_parameter_name, license_url=feed_mdb_40.license_url, diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py index 0bc238001..597f1e919 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py @@ -3,11 +3,10 @@ from conftest import feed_mdb_41 from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl -from feeds_operations_gen.models.authentication_type import AuthenticationType -from feeds_operations_gen.models.entity_type import EntityType -from feeds_operations_gen.models.feed_status import FeedStatus -from feeds_operations_gen.models.source_info import SourceInfo -from feeds_operations_gen.models.update_request_gtfs_rt_feed import ( +from feeds_gen.models.entity_type import EntityType +from feeds_gen.models.feed_status import FeedStatus +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_rt_feed import ( UpdateRequestGtfsRtFeed, ) from shared.database.database import Database @@ -27,9 +26,7 @@ def update_request_gtfs_rt_feed(): feed_contact_email=feed_mdb_41.feed_contact_email, source_info=SourceInfo( producer_url=feed_mdb_41.producer_url, - authentication_type=AuthenticationType( - int(feed_mdb_41.authentication_type) - ), + authentication_type=int(feed_mdb_41.authentication_type), authentication_info_url=feed_mdb_41.authentication_info_url, api_key_parameter_name=feed_mdb_41.api_key_parameter_name, license_url=feed_mdb_41.license_url, diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_get_feeds.py b/functions-python/operations_api/tests/feeds_operations/impl/test_get_feeds.py index 7439cff8a..e1c4c087b 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_get_feeds.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_get_feeds.py @@ -18,8 +18,6 @@ import pytest from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl -from feeds_operations_gen.models.gtfs_feed_response import GtfsFeedResponse -from feeds_operations_gen.models.gtfs_rt_feed_response import GtfsRtFeedResponse @pytest.mark.asyncio @@ -42,12 +40,6 @@ async def test_get_feeds_no_filters(): assert feed_types.count("gtfs") == 2 assert feed_types.count("gtfs_rt") == 1 - for feed in response.feeds: - if feed.data_type == "gtfs": - assert isinstance(feed, GtfsFeedResponse) - else: - assert isinstance(feed, GtfsRtFeedResponse) - @pytest.mark.asyncio async def test_get_feeds_gtfs_rt_filter(): @@ -64,10 +56,8 @@ async def test_get_feeds_gtfs_rt_filter(): assert len(response.feeds) == 1 rt_feed = response.feeds[0] - assert isinstance(rt_feed, GtfsRtFeedResponse) assert rt_feed.data_type == "gtfs_rt" assert rt_feed.stable_id == "mdb-41" - assert sorted(rt_feed.entity_types) == ["vp"] @pytest.mark.asyncio @@ -85,7 +75,6 @@ async def test_get_feeds_gtfs_filter(): assert len(response.feeds) == 2 for feed in response.feeds: - assert isinstance(feed, GtfsFeedResponse) assert feed.data_type == "gtfs" @@ -189,37 +178,6 @@ async def test_get_feeds_combined_filters(): assert response.feeds[0].data_type == "gtfs" -@pytest.mark.asyncio -async def test_get_feeds_with_locations(): - """Test get_feeds with location data.""" - api = OperationsApiImpl() - response = await api.get_feeds() - - for feed in response.feeds: - if feed.locations: - for location in feed.locations: - assert "country_code" in location - assert "subdivision_name" in location - assert "municipality" in location - - -@pytest.mark.asyncio -async def test_get_feeds_gtfs_rt_entity_types(): - """Test get_feeds for GTFS-RT with entity types.""" - api = OperationsApiImpl() - - response = await api.get_feeds(data_type="gtfs_rt") - assert response is not None - - for feed in response.feeds: - assert isinstance(feed, GtfsRtFeedResponse) - assert feed.data_type == "gtfs_rt" - assert feed.entity_types is not None - assert isinstance(feed.entity_types, list) - for entity_type in feed.entity_types: - assert entity_type in ["vp", "tu", "sa"] - - @pytest.mark.asyncio async def test_get_feeds_unpublished_status(): """ @@ -265,7 +223,6 @@ async def test_get_feeds_unpublished_with_data_type(): for feed in gtfs_response.feeds: assert feed.operational_status == "unpublished" assert feed.data_type == "gtfs" - assert isinstance(feed, GtfsFeedResponse) rt_response = await api.get_feeds( operation_status="unpublished", data_type="gtfs_rt" @@ -274,4 +231,3 @@ async def test_get_feeds_unpublished_with_data_type(): for feed in rt_response.feeds: assert feed.operational_status == "unpublished" assert feed.data_type == "gtfs_rt" - assert isinstance(feed, GtfsRtFeedResponse) diff --git a/scripts/gen-operations-config.yaml b/scripts/gen-operations-config.yaml index 5d4a76bff..bb151793f 100644 --- a/scripts/gen-operations-config.yaml +++ b/scripts/gen-operations-config.yaml @@ -1,6 +1,6 @@ # Documentation, https://openapi-generator.tech/docs/generators/python-fastapi/ additionalProperties: - packageName: feeds_operations_gen + packageName: feeds_gen # modelNameSuffix: Api removeOperationIdPrefix: true fastapiImplementationPackage: feeds_operations.impl From e5797ed2de839e460c402a6bc308ee726a5a25bc Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 16:59:14 -0400 Subject: [PATCH 2/9] add missing files --- .github/copilot-instructions.md | 110 ++++++++++++++++++ api/src/shared/db_models/model_utils.py | 26 +++++ .../impl/models/operation_gtfs_feed_impl.py | 37 ++++++ .../models/operation_gtfs_rt_feed_impl.py | 39 +++++++ scripts/api-operations-update-schema.sh | 68 +++++++++++ 5 files changed, 280 insertions(+) create mode 100644 .github/copilot-instructions.md create mode 100644 api/src/shared/db_models/model_utils.py create mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py create mode 100644 functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py create mode 100755 scripts/api-operations-update-schema.sh diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..37b99bf20 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,110 @@ +# Mobility Feed API - AI Coding Assistant Instructions + +## Project Architecture + +This is a **mobility data API service** built with FastAPI, serving open mobility data from across the world. The architecture follows a **code-generation pattern** with clear separation between generated and implementation code. + +### Core Components + +- **`api/`**: Main FastAPI application with spec-first development using OpenAPI Generator +- **`functions-python/`**: Google Cloud Functions for data processing (batch jobs, validation, analytics) +- **`web-app/`**: Frontend application +- **PostgreSQL + PostGIS**: Database with geospatial support for mobility data + +### Key Generated vs Implementation Split + +- **Generated code** (never edit): `api/src/feeds_gen/` and `api/src/shared/database_gen/` +- **Implementation code**: `api/src/feeds/impl/` contains actual business logic +- **Schema source**: `docs/DatabaseCatalogAPI.yaml` drives code generation + +## Critical Development Workflows + +### Initial Setup +```bash +# One-time OpenAPI setup +scripts/setup-openapi-generator.sh + +# Install dependencies +cd api && pip3 install -r requirements.txt -r requirements_dev.txt + +# Start local database +docker-compose --env-file ./config/.env.local up -d --force-recreate + +# Generate API stubs (run after schema changes) +scripts/api-gen.sh +scripts/db-gen.sh +``` + +### Common Development Commands +```bash +# Start API server (includes Swagger UI at http://localhost:8080/docs/) +scripts/api-start.sh + +# Run tests with coverage +scripts/api-tests.sh +# Run specific test file +scripts/api-tests.sh my_test_file.py + +# Lint checks (Flake8 + Black) +scripts/lint-tests.sh + +# Reset and populate local database +./scripts/docker-localdb-rebuild-data.sh --populate-db +# Include test datasets +./scripts/docker-localdb-rebuild-data.sh --populate-db --populate-test-data +``` + +## Project-Specific Patterns + +### Error Handling Convention +- Use `shared.common.error_handling.InternalHTTPException` for internal errors +- Convert to FastAPI HTTPException using `feeds.impl.error_handling.convert_exception()` +- Store error messages as Finals in `api/src/feeds/impl/error_handling.py` +- Error responses follow: `{"details": "The error message"}` + +### Database Patterns +- **Polymorphic inheritance**: `Feed` base class with `GtfsFeed`, `GbfsFeed`, `GtfsRTFeed` subclasses +- **SQLAlchemy ORM**: Models in `shared/database_gen/sqlacodegen_models.py` (generated) +- **Session management**: Use `@with_db_session` decorator for database operations +- **Unique IDs**: Generate with `generate_unique_id()` (36-char UUID4) + +### API Implementation Structure +- Endpoints in `feeds/impl/*_api_impl.py` extend generated base classes from `feeds_gen/` +- Filter classes in `shared/feed_filters/` for query parameter handling +- Model implementations in `feeds/impl/models/` extend generated models + +### Code Generation Workflow +1. Modify `docs/DatabaseCatalogAPI.yaml` for API changes +2. Run `scripts/api-gen.sh` to regenerate FastAPI stubs +3. Run `scripts/db-gen.sh` for database schema changes +4. Implement business logic in `feeds/impl/` classes + +### Testing Patterns +- Tests use empty local test DB (reset with `--use-test-db` flag) +- Coverage reports in `scripts/coverage_reports/` +- Python path configured to `src/` in `pyproject.toml` + +### Functions Architecture +- **Google Cloud Functions** in `functions-python/` for background processing +- Shared database models via `database_gen/` symlink +- Each function has its own deployment configuration +- Tasks include: validation reports, batch datasets, GBFS validation, BigQuery ingestion + +### Authentication +- **OAuth2 Bearer tokens** for API access +- Refresh tokens from mobilitydatabase.org account +- Access tokens valid for 1 hour +- Test endpoint: `/v1/metadata` with Bearer token + +## Integration Points + +- **BigQuery**: Analytics data pipeline via `big_query_ingestion/` function +- **PostGIS**: Geospatial queries for location-based feed filtering +- **Liquibase**: Database schema migrations in `liquibase/` directory +- **Docker**: Multi-service setup with PostgreSQL, test DB, and schema documentation + +## File Exclusions for AI Context +- Skip `src/feeds_gen/*` and `src/shared/database_gen/*` (generated code) +- Skip `data/` and `data-test/` (database volumes) +- Skip `htmlcov/` (coverage reports) +- Black formatter excludes these paths automatically \ No newline at end of file diff --git a/api/src/shared/db_models/model_utils.py b/api/src/shared/db_models/model_utils.py new file mode 100644 index 000000000..90d7398a0 --- /dev/null +++ b/api/src/shared/db_models/model_utils.py @@ -0,0 +1,26 @@ +from packaging.version import Version + + +def compare_java_versions(v1: str | None, v2: str | None): + """ + Compare two version strings v1 and v2. + Returns 1 if v1 > v2, -1 if v1 < v2, + otherwise 0. + The version strings are expected to be in the format of + major.minor.patch[-SNAPSHOT] + """ + if v1 is None and v2 is None: + return 0 + if v1 is None: + return -1 + if v2 is None: + return 1 + # clean version strings replacing the SNAPSHOT suffix with .dev0 + v1 = v1.replace("-SNAPSHOT", ".dev0") + v2 = v2.replace("-SNAPSHOT", ".dev0") + if Version(v1) > Version(v2): + return 1 + elif Version(v1) < Version(v2): + return -1 + else: + return 0 diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py new file mode 100644 index 000000000..3745cf794 --- /dev/null +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_feed_impl.py @@ -0,0 +1,37 @@ +from feeds_gen.models.operation_gtfs_feed import OperationGtfsFeed +from shared.database_gen.sqlacodegen_models import Gtfsfeed +from shared.db_models.gtfs_feed_impl import GtfsFeedImpl + + +class OperationGtfsFeedImpl(GtfsFeedImpl, OperationGtfsFeed): + """Base implementation of the feeds models.""" + + class Config: + """Pydantic configuration. + Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. + """ + + from_attributes = True + + def __init__(self, **data): + super().__init__(**data) + self.locations = self.locations or [] + self.redirects = self.redirects or [] + + @classmethod + def from_orm(cls, feed: Gtfsfeed | None) -> OperationGtfsFeed | None: + """Convert a SQLAlchemy row object to a Pydantic model.""" + if not feed: + return None + operation_gtfs_feed = super().from_orm(feed) + if not operation_gtfs_feed: + return None + + data = dict(operation_gtfs_feed.__dict__) + # Override id and add stable_id + data["id"] = feed.id + data["stable_id"] = feed.stable_id + # Add missing fields from public API model + data["operational_status"] = feed.operational_status + + return cls.model_construct(**data) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py new file mode 100644 index 000000000..290117d37 --- /dev/null +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_gtfs_rt_feed_impl.py @@ -0,0 +1,39 @@ +from feeds_gen.models.operation_gtfs_rt_feed import OperationGtfsRtFeed +from shared.database_gen.sqlacodegen_models import Gtfsrealtimefeed +from shared.db_models.gtfs_rt_feed_impl import GtfsRTFeedImpl + + +class OperationGtfsRtFeedImpl(GtfsRTFeedImpl, OperationGtfsRtFeed): + """Base implementation of the feeds models.""" + + class Config: + """Pydantic configuration. + Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object. + """ + + from_attributes = True + + def __init__(self, **data): + super().__init__(**data) + self.entity_types = self.entity_types or [] + self.locations = self.locations or [] + self.redirects = self.redirects or [] + self.feed_references = self.feed_references or [] + + @classmethod + def from_orm(cls, feed: Gtfsrealtimefeed | None) -> OperationGtfsRtFeed | None: + """Convert a SQLAlchemy row object to a Pydantic model.""" + if not feed: + return None + operation_gtfs_feed = super().from_orm(feed) + if not operation_gtfs_feed: + return None + + data = dict(operation_gtfs_feed.__dict__) + # Override id and add stable_id + data["id"] = feed.id + data["stable_id"] = feed.stable_id + # Add missing fields from public API model + data["operational_status"] = feed.operational_status + + return cls.model_construct(**data) diff --git a/scripts/api-operations-update-schema.sh b/scripts/api-operations-update-schema.sh new file mode 100755 index 000000000..b1aafa0bf --- /dev/null +++ b/scripts/api-operations-update-schema.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ./scripts/api-operations-update-schema.sh \ +# [--source ./docs/DatabaseCatalogAPI.yaml] \ +# [--dest ./docs/OperationsAPI.yaml] +# +# Behavior: +# - Replaces components.schemas in Operations with those from Catalog. +# - Preserves only schemas in Operations that have x-operation: true at the schema root (these override source). +# - Removes any non-operation schemas that exist only in Operations. + +SCRIPT_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +SOURCE="" +DEST="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --source|-s) SOURCE="${2:-}"; shift 2 ;; + --dest|-d) DEST="${2:-}"; shift 2 ;; + -h|--help) echo "Usage: $0 [--source ] [--dest ]"; exit 0 ;; + *) echo "Unknown arg: $1" >&2; exit 1 ;; + esac +done + +: "${SOURCE:=${REPO_ROOT}/docs/DatabaseCatalogAPI.yaml}" +: "${DEST:=${REPO_ROOT}/docs/OperationsAPI.yaml}" + +if ! command -v yq >/dev/null 2>&1; then + echo "yq not found. Install with: brew install yq" >&2 + exit 1 +fi +YQ_MAJOR="$(yq --version 2>/dev/null | sed -n 's/.*version v\([0-9][0-9]*\).*/\1/p')" +if [[ -z "${YQ_MAJOR:-}" || "${YQ_MAJOR}" -lt 4 ]]; then + echo "yq v4+ required. Current: $(yq --version 2>/dev/null)" >&2 + exit 1 +fi + +[[ -f "${SOURCE}" ]] || { echo "Source not found: ${SOURCE}" >&2; exit 1; } +[[ -f "${DEST}" ]] || { echo "Dest not found: ${DEST}" >&2; exit 1; } + +cp -f "${DEST}" "${DEST}.bak" + +# Merge strategy: +# - Start from source schemas (Catalog): ensures Operations aligns with source by default +# - Overlay ONLY the destination schemas that are explicitly marked with x-operation: true +# (these are preserved and override the source) +# - Any non-operation schemas that exist only in Operations are DROPPED +SRC_ABS="$(cd "$(dirname "${SOURCE}")" && pwd)/$(basename "${SOURCE}")" +export SRC="${SRC_ABS}" + +yq -i ' + (.components.schemas // {}) as $dst + | (load(strenv(SRC)).components.schemas // {}) as $src + | .components.schemas = ( + $src + * ( + $dst + | with_entries( select(.value."x-operation" == true) ) + ) + ) +' "${DEST}" + +echo "Synced schemas from ${SOURCE} -> ${DEST} (${DEST}.bak created)." +echo "Note: Schemas in Operations with x-operation: true were preserved." \ No newline at end of file From ae327348ee3d5136655ee65bdc882d2fa41efebe Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 17:44:51 -0400 Subject: [PATCH 3/9] fix missing entity type --- .../operations_api/.openapi-generator/FILES | 1 - .../impl/models/operation_entity_type_impl.py | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/functions-python/operations_api/.openapi-generator/FILES b/functions-python/operations_api/.openapi-generator/FILES index 79307ea11..8903be1b1 100644 --- a/functions-python/operations_api/.openapi-generator/FILES +++ b/functions-python/operations_api/.openapi-generator/FILES @@ -7,7 +7,6 @@ src/feeds_gen/models/basic_dataset.py src/feeds_gen/models/basic_feed.py src/feeds_gen/models/bounding_box.py src/feeds_gen/models/data_type.py -src/feeds_gen/models/entity_type.py src/feeds_gen/models/external_id.py src/feeds_gen/models/extra_models.py src/feeds_gen/models/feed.py diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py index 1b186217e..9db323ccc 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py @@ -1,6 +1,5 @@ from pydantic import BaseModel -from feeds_gen.models.entity_type import EntityType from shared.database_gen.sqlacodegen_models import Entitytype as EntityTypeOrm @@ -17,26 +16,22 @@ class Config: from_attributes = True @classmethod - def from_orm(cls, obj: EntityTypeOrm | None) -> EntityType | None: + def from_orm(cls, obj: EntityTypeOrm | None) -> str | None: """ Convert a SQLAlchemy row object to a Pydantic model. """ if obj is None: return None - return EntityType(obj.name.lower()) + return obj.name @classmethod - def to_orm(cls, entity_type: EntityType, session) -> EntityTypeOrm: + def to_orm(cls, entity_type: str, session) -> EntityTypeOrm: """ Convert a Pydantic model to a SQLAlchemy row object. """ result = ( session.query(EntityTypeOrm) - .filter(EntityTypeOrm.name.ilike(entity_type.name)) + .filter(EntityTypeOrm.name.ilike(entity_type)) .first() ) - return ( - result - if result is not None - else EntityTypeOrm(name=entity_type.name.lower()) - ) + return result if result is not None else EntityTypeOrm(name=entity_type.lower()) From 26c2ec0da4c2b98964a83a7dee47b5853caddeff Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 17:49:55 -0400 Subject: [PATCH 4/9] Update operations api ci --- .github/workflows/api-deployer.yml | 2 +- .github/workflows/build-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api-deployer.yml b/.github/workflows/api-deployer.yml index c17e9b4a5..1f383a4a1 100644 --- a/.github/workflows/api-deployer.yml +++ b/.github/workflows/api-deployer.yml @@ -260,7 +260,7 @@ jobs: - uses: actions/download-artifact@v4 with: name: feeds_operations_gen - path: functions-python/operations_api/src/feeds_operations_gen/ + path: functions-python/operations_api/src/feeds_gen/ - name: Build python functions run: | diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6241f99bb..ec8544edf 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -140,5 +140,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: feeds_operations_gen - path: functions-python/operations_api/src/feeds_operations_gen/ + path: functions-python/operations_api/src/feeds_gen/ overwrite: true \ No newline at end of file From 00b3a7f0d74d03915af0374c75bb2cc9af5956a4 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 17:58:36 -0400 Subject: [PATCH 5/9] Update entity type model and tests --- .../impl/models/operation_entity_type_impl.py | 2 +- .../impl/models/test_update_request_gtfs_rt_feed_impl.py | 5 ++--- .../impl/test_feeds_operations_impl_gtfs_rt.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py index 9db323ccc..fcf8efd15 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_entity_type_impl.py @@ -31,7 +31,7 @@ def to_orm(cls, entity_type: str, session) -> EntityTypeOrm: """ result = ( session.query(EntityTypeOrm) - .filter(EntityTypeOrm.name.ilike(entity_type)) + .filter(EntityTypeOrm.name.ilike(entity_type.lower())) .first() ) return result if result is not None else EntityTypeOrm(name=entity_type.lower()) diff --git a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py index d156fbf5b..f275912a7 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/models/test_update_request_gtfs_rt_feed_impl.py @@ -9,7 +9,6 @@ from feeds_operations.impl.models.update_request_gtfs_rt_feed_impl import ( UpdateRequestGtfsRtFeedImpl, ) -from feeds_gen.models.entity_type import EntityType from feeds_gen.models.feed_status import FeedStatus from feeds_gen.models.source_info import SourceInfo from feeds_gen.models.update_request_gtfs_rt_feed import ( @@ -78,7 +77,7 @@ def test_to_orm(): ), redirects=[RedirectImpl(target_id="target_stable_id", comment="Test comment")], external_ids=[ExternalIdImpl(external_id="external_id")], - entity_types=[EntityType.VP], + entity_types=["vp"], feed_references=["feed_reference"], ) entity = Gtfsrealtimefeed(id="1", stable_id="stable_id", data_type="gtfs") @@ -120,7 +119,7 @@ def test_to_orm_invalid_source_info(): source_info=None, redirects=[RedirectImpl(target_id="target_stable_id", comment="Test comment")], external_ids=[ExternalIdImpl(external_id="external_id")], - entity_types=[EntityType.VP], + entity_types=["vp"], feed_references=["feed_reference"], ) entity = Gtfsrealtimefeed(id="1", stable_id="stable_id", data_type="gtfs") diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py index 597f1e919..1a4518ec4 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py @@ -3,7 +3,6 @@ from conftest import feed_mdb_41 from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl -from feeds_gen.models.entity_type import EntityType from feeds_gen.models.feed_status import FeedStatus from feeds_gen.models.source_info import SourceInfo from feeds_gen.models.update_request_gtfs_rt_feed import ( @@ -33,7 +32,7 @@ def update_request_gtfs_rt_feed(): ), redirects=[], operational_status_action="no_change", - entity_types=[EntityType.VP], + entity_types=["vp"], official=True, ) From aeb36346d1d033bf59c0a38c006273316b09da38 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 23 Oct 2025 18:34:46 -0400 Subject: [PATCH 6/9] Update documentation --- functions-python/operations_api/README.md | 66 ++++++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/functions-python/operations_api/README.md b/functions-python/operations_api/README.md index 382a865ef..ea1f4cec8 100644 --- a/functions-python/operations_api/README.md +++ b/functions-python/operations_api/README.md @@ -1,6 +1,8 @@ # Operations API -The Operations API is a function that exposes the operations API. -The operations API schema is located at ../../docs/OperationsAPI.yml. +The Operations API is a Google Cloud Function exposing internal operations endpoints for the Mobility Database. +The Operations API OpenAPI schema lives at `../../docs/OperationsAPI.yaml`. + +> Note: generated server stubs are created from the schema. Do not edit generated code under `src/feeds_gen/`; put implementation under `src/feeds_operations/impl/`. # Function configuration The function is configured using the following environment variables: @@ -18,7 +20,67 @@ The function is configured using the following environment variables: ``` - Start local and test database ``` +# from the repository root docker compose --env-file ./config/.env.local up -d liquibase-test +``` +- Update OperationsAPI OpenAPI components with Mobility Database Catalog API components +``` +./scripts/api-operations-update-schema.sh +``` + + +## Development process + +Follow these steps when working on the Operations API: + +1) Prerequisites +- Install OpenAPI Generator (one-time): + ``` + ./scripts/setup-openapi-generator.sh + ``` +- Ensure yq v4+ is available (required by the schema sync script). On macOS: + ``` + brew install yq + ``` + +2) Start databases locally (from repo root) +``` +docker compose --env-file ./config/.env.local up -d liquibase-test +``` + +3) Sync Operations schema components from the Catalog API +- When the Catalog API schemas (`docs/DatabaseCatalogAPI.yaml`) change, update Operations to keep shared data models in sync while preserving operation-only ones: + ``` + ./scripts/api-operations-update-schema.sh + ``` + This replaces `components.schemas` in `docs/OperationsAPI.yaml` with those from the Catalog, but preserves only schemas marked `x-operation: true` in Operations (non-operation dest-only schemas are removed). + +4) Regenerate Operations API server stubs +``` +./scripts/api-operations-gen.sh +``` +Generated code goes to `functions-python/operations_api/src/feeds_gen/`. + +5) Implement or update handlers +- Extend the generated base classes under `src/feeds_gen/` in your implementation files under: + - `src/feeds_operations/impl/feeds_operations_impl.py` + - `src/feeds_operations/impl/models/*` + +6) Run locally +``` +./scripts/function-python-run.sh --function_name operations_api +``` + +7) Run tests +- Using the repo test runner (recommended): + ```bash + ./scripts/api-tests.sh --folder functions-python/operations_api + ``` + +8) Build an artifact (zip) for deployment +``` +./scripts/function-python-build.sh --function_name operations_api +``` # Local development From 4baefcdbc9fbcdaf7074b183c5a1e3aa277ac16e Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 24 Oct 2025 09:26:39 -0400 Subject: [PATCH 7/9] Fix model import --- .../src/feeds_operations/impl/feeds_operations_impl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index 12cdd7fd5..f3915f620 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -39,12 +39,16 @@ UpdateRequestGtfsRtFeed, ) from shared.database.database import with_db_session, refresh_materialized_view -from shared.database_gen.sqlacodegen_models import Gtfsfeed, t_feedsearch, Feed +from shared.database_gen.sqlacodegen_models import ( + Gtfsfeed, + t_feedsearch, + Feed, + Gtfsrealtimefeed, +) from shared.helpers.query_helper import ( query_feed_by_stable_id, get_feeds_query, ) -from shared.helpers.src.shared.database_gen.sqlacodegen_models import Gtfsrealtimefeed from .models.operation_feed_impl import OperationFeedImpl from .models.operation_gtfs_feed_impl import OperationGtfsFeedImpl from .models.operation_gtfs_rt_feed_impl import OperationGtfsRtFeedImpl From 046755ca3e863a061db958e64789ee652cfe96eb Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 24 Oct 2025 10:58:20 -0400 Subject: [PATCH 8/9] add feed level fields --- docs/OperationsAPI.yaml | 2 +- .../impl/models/operation_feed_impl.py | 4 +- scripts/api-operations-token.sh | 149 ++++++++++++++++++ 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100755 scripts/api-operations-token.sh diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 4d0b6d48e..e9b698ee3 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -964,7 +964,7 @@ components: OperationFeed: x-operation: true allOf: - - $ref: "#/components/schemas/BasicFeed" + - $ref: "#/components/schemas/Feed" - type: object description: Feed response model. type: object diff --git a/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py b/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py index 0f0d888cc..fbdfa8753 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/models/operation_feed_impl.py @@ -1,9 +1,9 @@ from feeds_gen.models.operation_feed import OperationFeed from shared.database_gen.sqlacodegen_models import Feed -from shared.db_models.basic_feed_impl import BasicFeedImpl +from shared.db_models.basic_feed_impl import BaseFeedImpl -class OperationFeedImpl(BasicFeedImpl, OperationFeed): +class OperationFeedImpl(BaseFeedImpl, OperationFeed): """Base implementation of the feeds models.""" class Config: diff --git a/scripts/api-operations-token.sh b/scripts/api-operations-token.sh new file mode 100755 index 000000000..b89e09aa1 --- /dev/null +++ b/scripts/api-operations-token.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# api-operations-token.sh /path/to/credentials.json [scopes] [port] +# +# Examples: +# api-operations-token.sh ./client_secret.json \ +# "https://www.googleapis.com/auth/cloud-platform openid email profile" 8080 +# api-operations-token.sh ./client_secret.json \ +# "openid email profile" 8888 +# +# Requires: jq, python3, openssl, curl +# Note: Ensure the chosen port's redirect URI (http://localhost:) is listed under +# your OAuth client’s "Authorized redirect URIs" in GCP. + +CREDS_JSON="${1:-}" +SCOPES="${2:-https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/userinfo.profile openid}" +PORT="${3:-8080}" + +if [[ -z "${CREDS_JSON}" || ! -f "${CREDS_JSON}" ]]; then + echo "Error: provide path to your Google OAuth 'web' credentials JSON." >&2 + exit 1 +fi + +need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" >&2; exit 1; }; } +need jq +need python3 +need openssl +need curl + +CLIENT_ID="$(jq -r '.web.client_id' "${CREDS_JSON}")" +CLIENT_SECRET="$(jq -r '.web.client_secret // empty' "${CREDS_JSON}")" +AUTH_URI="$(jq -r '.web.auth_uri' "${CREDS_JSON}")" +TOKEN_URI="$(jq -r '.web.token_uri' "${CREDS_JSON}")" + +if [[ -z "${CLIENT_ID}" || -z "${AUTH_URI}" || -z "${TOKEN_URI}" ]]; then + echo "Error: client_id/auth_uri/token_uri not found under .web in ${CREDS_JSON}" >&2 + exit 1 +fi + +REDIRECT_URI="http://localhost:${PORT}" + +# Helpers +b64url() { openssl base64 -A | tr '+/' '-_' | tr -d '='; } + +# PKCE + state/nonce +CODE_VERIFIER="$(openssl rand -base64 64 | tr -d '\n' | tr '+/' '-_' | tr -d '=')" +CODE_CHALLENGE="$(printf '%s' "${CODE_VERIFIER}" | openssl dgst -binary -sha256 | b64url)" +STATE="$(openssl rand -hex 16)" +NONCE="$(openssl rand -hex 16)" + +# Tiny one-shot HTTP server to capture ?code= +TMP_DIR="$(mktemp -d)" +CODE_FILE="${TMP_DIR}/auth_code.txt" + +cat > "${TMP_DIR}/server.py" <<'PY' +import http.server, socketserver, urllib.parse, sys +PORT = int(sys.argv[1]); OUT = sys.argv[2] +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + p = urllib.parse.urlparse(self.path); q = urllib.parse.parse_qs(p.query) + code = q.get("code", [""])[0]; state = q.get("state", [""])[0] + with open(OUT, "w") as f: f.write(code + "\n" + state + "\n") + self.send_response(200); self.send_header("Content-Type","text/html"); self.end_headers() + self.wfile.write(b"

You can close this window.

") +with socketserver.TCPServer(("127.0.0.1", PORT), H) as httpd: httpd.handle_request() +PY + +python3 "${TMP_DIR}/server.py" "${PORT}" "${CODE_FILE}" & +SERVER_PID=$! + +cleanup() { + echo + echo "Cleaning up local server (PID ${SERVER_PID})..." + kill "${SERVER_PID}" >/dev/null 2>&1 || true + rm -rf "${TMP_DIR}" >/dev/null 2>&1 || true +} +# Clean up on normal exit, Ctrl+C (INT), termination (TERM), or error (ERR) +trap cleanup EXIT INT TERM ERR + +# Build consent URL +AUTH_URL="$AUTH_URI?response_type=code&client_id=$(printf %s "${CLIENT_ID}" | jq -sRr @uri)\ +&redirect_uri=$(printf %s "${REDIRECT_URI}" | jq -sRr @uri)\ +&scope=$(printf %s "${SCOPES}" | jq -sRr @uri)\ +&state=${STATE}&code_challenge=${CODE_CHALLENGE}&code_challenge_method=S256\ +&access_type=offline&prompt=consent&nonce=${NONCE}" + +# Open browser +if command -v open >/dev/null 2>&1; then + open "${AUTH_URL}" +elif command -v xdg-open >/dev/null 2>&1; then + xdg-open "${AUTH_URL}" +else + echo "Open this URL in your browser:" + echo "${AUTH_URL}" +fi + +echo "Waiting for authorization on ${REDIRECT_URI} ..." +for _ in {1..180}; do [[ -s "${CODE_FILE}" ]] && break; sleep 1; done +if [[ ! -s "${CODE_FILE}" ]]; then + echo "Error: no authorization code received." >&2 + exit 1 +fi + +AUTH_CODE="$(head -n1 "${CODE_FILE}")" +READ_STATE="$(sed -n '2p' "${CODE_FILE}")" + +if [[ "${READ_STATE}" != "${STATE}" ]]; then + echo "Error: state mismatch. Aborting." >&2 + exit 1 +fi + +# Exchange code for tokens +POST_DATA=( + -d "grant_type=authorization_code" + -d "code=${AUTH_CODE}" + -d "redirect_uri=${REDIRECT_URI}" + -d "client_id=${CLIENT_ID}" + -d "code_verifier=${CODE_VERIFIER}" +) +[[ -n "${CLIENT_SECRET}" ]] && POST_DATA+=(-d "client_secret=${CLIENT_SECRET}") + +TOKENS_JSON="$(curl -sS -X POST "${TOKEN_URI}" \ + -H "Content-Type: application/x-www-form-urlencoded" "${POST_DATA[@]}")" + +# Parse +ACCESS_TOKEN="$(echo "${TOKENS_JSON}" | jq -r '.access_token // empty')" +EXPIRES_IN="$(echo "${TOKENS_JSON}" | jq -r '.expires_in // empty')" +REFRESH_TOKEN="$(echo "${TOKENS_JSON}" | jq -r '.refresh_token // empty')" +TOKEN_TYPE="$(echo "${TOKENS_JSON}" | jq -r '.token_type // empty')" + + +echo +echo "=== Token Response (trimmed) ===" +echo "${TOKENS_JSON}" | jq '{access_token, expires_in, token_type, scope, refresh_token: (has("refresh_token"))}' + +echo +echo "Saved:" +echo " ${BASE}.json # full token response (keep secure)" +echo " ${BASE}_access.token # access token" +[[ -n "${REFRESH_TOKEN}" ]] && echo " ${BASE}_refresh.token # refresh token (store securely)" + + +# Exit non-zero if we failed to produce an access token +if [[ -z "${ACCESS_TOKEN}" ]]; then + echo "ERROR: access_token is empty. Inspect ${BASE}.json for details." >&2 + exit 2 +fi From 9a6d5e112ceec93db09829e767aed20459040a6e Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 27 Oct 2025 12:40:30 -0400 Subject: [PATCH 9/9] add missing shapely dependency --- functions-python/operations_api/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions-python/operations_api/requirements.txt b/functions-python/operations_api/requirements.txt index edbba8cfc..536e57997 100644 --- a/functions-python/operations_api/requirements.txt +++ b/functions-python/operations_api/requirements.txt @@ -31,4 +31,5 @@ psycopg2-binary==2.9.6 cachetools deepdiff fastapi_filter -pycountry \ No newline at end of file +pycountry +shapely \ No newline at end of file