From b8ef2d3c4bf120b658861695b4c4ed981dd29855 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 3 Apr 2025 21:48:42 +0200 Subject: [PATCH 1/6] doc something --- .../models/confirmations.py | 13 ++-- .../models/groups.py | 11 ++-- .../models/jinja2_templates.py | 8 ++- .../models/products.py | 6 ++ .../models/products_to_templates.py | 10 +++ .../models/projects.py | 10 ++- .../models/projects_comments.py | 10 +++ .../models/projects_networks.py | 10 +++ .../models/projects_nodes.py | 12 +++- .../models/projects_tags.py | 10 +++ .../models/projects_to_folders.py | 10 +++ .../models/projects_to_products.py | 10 +++ .../models/scicrunch_resources.py | 13 +++- .../models/service_compatibility.py | 39 ++++++++++++ .../models/services_consume_filetypes.py | 24 +++---- .../simcore_postgres_database/models/tags.py | 10 +++ .../models/tags_access_rights.py | 10 +++ .../models/user_to_projects.py | 38 ------------ .../simcore_postgres_database/models/users.py | 11 ++++ .../models/users_pre_registration_details.py | 62 +++++++++++++++++++ 20 files changed, 263 insertions(+), 64 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/confirmations.py b/packages/postgres-database/src/simcore_postgres_database/models/confirmations.py index 6fd56e8c8e0..a31418cb16a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/confirmations.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/confirmations.py @@ -1,10 +1,15 @@ -""" User's confirmations table +"""User's confirmations table - - Keeps a list of tokens to identify an action (registration, invitation, reset, etc) authorized - by link to a a user in the framework - - These tokens have an expiration date defined by configuration +- Keeps a list of tokens to identify an action (registration, invitation, reset, etc) authorized +by link to a a user in the framework +- These tokens have an expiration date defined by configuration +Migration strategy: +- The primary key is `code`, which is unique and sufficient for migration. +- Ensure `user_id` foreign key references are valid in the target database. +- No additional changes are required; this table can be migrated as is. """ + import enum import sqlalchemy as sa diff --git a/packages/postgres-database/src/simcore_postgres_database/models/groups.py b/packages/postgres-database/src/simcore_postgres_database/models/groups.py index 940e1a78769..ab7f7e168ba 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/groups.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/groups.py @@ -1,9 +1,12 @@ -""" Groups table +"""Groups table - - List of groups in the framework - - Groups have a ID, name and a list of users that belong to the group -""" +- Represents user groups in the system. +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" import sqlalchemy as sa from common_library.groups_enums import GroupType diff --git a/packages/postgres-database/src/simcore_postgres_database/models/jinja2_templates.py b/packages/postgres-database/src/simcore_postgres_database/models/jinja2_templates.py index 2a1042d7365..663b09af5ad 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/jinja2_templates.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/jinja2_templates.py @@ -1,5 +1,11 @@ -""" Collection of jinja2 templates for customizable docs (e.g. emails, sites) +"""Jinja2 templates table +- Collection of jinja2 templates for customizable docs (e.g. emails, sites) + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- No foreign key dependencies exist. +- No additional changes are required; this table can be migrated as is. """ import sqlalchemy as sa diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products.py b/packages/postgres-database/src/simcore_postgres_database/models/products.py index a646bf7ef8f..a2c5452bdae 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products.py @@ -1,8 +1,13 @@ """Products table +- Represents products in the system. - List of products served by the simcore platform - Products have a name and an associated host (defined by a regex) - Every product has a front-end with exactly the same name + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. """ from typing import Literal @@ -123,6 +128,7 @@ class ProductLoginSettingsDict(TypedDict, total=False): # NOTE: a default entry is created in the table Product # see packages/postgres-database/src/simcore_postgres_database/migration/versions/350103a7efbd_modified_products_table.py + products = sa.Table( "products", metadata, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products_to_templates.py b/packages/postgres-database/src/simcore_postgres_database/models/products_to_templates.py index 44115660735..6afa1f676ec 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products_to_templates.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products_to_templates.py @@ -1,3 +1,13 @@ +"""Products to templates table + +- Links products to Jinja2 templates. + +Migration strategy: +- Composite primary key (`product_id`, `template_id`) is unique and sufficient for migration. +- Ensure foreign key references to `products` and `jinja2_templates` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import ( diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects.py b/packages/postgres-database/src/simcore_postgres_database/models/projects.py index 1db9db0fd7b..53c3f18ace2 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects.py @@ -1,4 +1,12 @@ -"""Projects table""" +"""Projects table + +- Represents user projects in the system. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" import enum diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py index 919b143bff3..16dc3c708af 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py @@ -1,3 +1,13 @@ +"""Projects comments table + +- Stores comments associated with projects. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references to `projects` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import RefActions, column_created_datetime, column_modified_datetime diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_networks.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_networks.py index 905c1aa4cf0..739ff8a0a5b 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_networks.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_networks.py @@ -1,3 +1,13 @@ +"""Projects networks table + +- Represents networks associated with projects. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references to `projects` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from sqlalchemy.dialects.postgresql import JSONB diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index a5991e7f9db..1489b7650d7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -1,7 +1,13 @@ -""" Groups table +"""Projects nodes table - - List of groups in the framework - - Groups have a ID, name and a list of users that belong to the group +- Represents nodes within projects. +- List of groups in the framework +- Groups have a ID, name and a list of users that belong to the group + +Migration strategy: +- The primary key is `node_id`, which is unique and sufficient for migration. +- Ensure foreign key references to `projects` are valid in the target database. +- No additional changes are required; this table can be migrated as is. """ import sqlalchemy as sa diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py index 3507fc8d239..636d91f3e8f 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py @@ -1,3 +1,13 @@ +"""Projects tags table + +- Links tags to projects. + +Migration strategy: +- Composite primary key (`project_id`, `tag_id`) is unique and sufficient for migration. +- Ensure foreign key references to `projects` and `tags` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import RefActions diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py index 34c24221936..2a382cce510 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_folders.py @@ -1,3 +1,13 @@ +"""Projects to folders table + +- Links projects to folders. + +Migration strategy: +- Composite primary key (`project_id`, `folder_id`) is unique and sufficient for migration. +- Ensure foreign key references to `projects` and `folders` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import RefActions, column_created_datetime, column_modified_datetime diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py index 0d8429ab3ee..7621718bead 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_products.py @@ -1,3 +1,13 @@ +"""Projects to products table + +- Links projects to products. + +Migration strategy: +- Composite primary key (`project_id`, `product_id`) is unique and sufficient for migration. +- Ensure foreign key references to `projects` and `products` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import RefActions, column_created_datetime, column_modified_datetime diff --git a/packages/postgres-database/src/simcore_postgres_database/models/scicrunch_resources.py b/packages/postgres-database/src/simcore_postgres_database/models/scicrunch_resources.py index 794fba1005d..e449fb86ca2 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/scicrunch_resources.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/scicrunch_resources.py @@ -1,4 +1,15 @@ -""" Stores SOME of the information associated to Research Resource Identifiers (RRIDs) as defined in https://scicrunch.org/resources +"""Scicrunch resources table + +- Stores resources fetched from Scicrunch. +- Stores SOME of the information associated to Research Resource Identifiers (RRIDs) as defined in https://scicrunch.org/resources + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- No foreign key dependencies exist. +- No additional changes are required; this table can be migrated as is. +""" + +""" """ import sqlalchemy as sa diff --git a/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py b/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py new file mode 100644 index 00000000000..105e11c4b12 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py @@ -0,0 +1,39 @@ +"""Service compatibility table + +- Defines compatibility between services. + +Migration strategy: +- Composite primary key (`service_key`, `service_version`) is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + +from .base import metadata + +service_compatibility = sa.Table( + "service_compatibility", + metadata, + sa.Column( + "service_key", + sa.String, + nullable=False, + doc="Service key identifier", + ), + sa.Column( + "service_version", + sa.String, + nullable=False, + doc="Service version identifier", + ), + sa.Column( + "compatibility", + JSONB, + nullable=False, + server_default=sa.text("'{}'::jsonb"), + doc="Compatibility information in JSON format", + ), + sa.PrimaryKeyConstraint("service_key", "service_version"), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services_consume_filetypes.py b/packages/postgres-database/src/simcore_postgres_database/models/services_consume_filetypes.py index 65c6c8546b3..c82053364ac 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services_consume_filetypes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services_consume_filetypes.py @@ -1,23 +1,23 @@ """ - Establishes which services can consume a given filetype +Services consume filetypes table - The relation is N-N because - - a service could handle one or more filetypes and - - one filetype could be handled by one or more services +- Links services to the file types they consume. +- Establishes which services can consume a given filetype +- The relation is N-N because + - a service could handle one or more filetypes and + - one filetype could be handled by one or more services + +Migration strategy: +- Composite primary key (`service_key`, `filetype_id`) is unique and sufficient for migration. +- Ensure foreign key references to `services` and `filetypes` are valid in the target database. +- No additional changes are required; this table can be migrated as is. """ + import sqlalchemy as sa from ._common import RefActions from .base import metadata -# -# TODO: This information SHALL be defined in service metadata upon publication -# and the catalog service, using e.g. a background task, -# can automatically fill this table with services that elligable (e.g. shared with everybody) -# to consume given filetypes. Notice also that service "matching" will also be determined in a near -# future by more complex metadata -# - services_consume_filetypes = sa.Table( "services_consume_filetypes", metadata, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/tags.py b/packages/postgres-database/src/simcore_postgres_database/models/tags.py index da7c788e02d..6699dca6624 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/tags.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/tags.py @@ -1,3 +1,13 @@ +"""Tags table + +- Represents tags that can be associated with various entities. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from .base import metadata diff --git a/packages/postgres-database/src/simcore_postgres_database/models/tags_access_rights.py b/packages/postgres-database/src/simcore_postgres_database/models/tags_access_rights.py index b818c975817..300acd0b85c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/tags_access_rights.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/tags_access_rights.py @@ -1,3 +1,13 @@ +"""Tags access rights table + +- Defines access rights for tags. + +Migration strategy: +- Composite primary key (`tag_id`, `user_id`) is unique and sufficient for migration. +- Ensure foreign key references to `tags` and `users` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from ._common import RefActions, column_created_datetime, column_modified_datetime diff --git a/packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py b/packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py deleted file mode 100644 index 4a66e0be611..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/user_to_projects.py +++ /dev/null @@ -1,38 +0,0 @@ -import sqlalchemy as sa - -from ._common import RefActions -from .base import metadata -from .projects import projects -from .users import users - -# DEPRECATED!!!!!!!!!!!!!! DO NOT USE!!!!!!! -user_to_projects = sa.Table( - "user_to_projects", - metadata, - sa.Column("id", sa.BigInteger, nullable=False, primary_key=True), - sa.Column( - "user_id", - sa.BigInteger, - sa.ForeignKey( - users.c.id, - name="fk_user_to_projects_id_users", - ondelete=RefActions.CASCADE, - onupdate=RefActions.CASCADE, - ), - nullable=False, - ), - sa.Column( - "project_id", - sa.BigInteger, - sa.ForeignKey( - projects.c.id, - name="fk_user_to_projects_id_projects", - ondelete=RefActions.CASCADE, - onupdate=RefActions.CASCADE, - ), - nullable=False, - ), - # TODO: do not ondelete=cascase for project_id or it will delete SHARED PROJECT - # add instead sa.UniqueConstraint('user_id', 'project_id', name='user_project_uniqueness'), - # -) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/users.py b/packages/postgres-database/src/simcore_postgres_database/models/users.py index 7be2161ff86..2e5778521d6 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/users.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/users.py @@ -1,3 +1,13 @@ +"""Users table + +- Represents users in the system. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references (if any) are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + import sqlalchemy as sa from common_library.users_enums import UserRole, UserStatus from sqlalchemy.sql import expression @@ -10,6 +20,7 @@ "UserStatus", ) + users = sa.Table( "users", metadata, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py b/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py new file mode 100644 index 00000000000..3756d0613d5 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py @@ -0,0 +1,62 @@ +"""Users pre-registration details table + +- Stores details of users during the pre-registration phase. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references to `users` are valid in the target database. +- No additional changes are required; this table can be migrated as is. +""" + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import func + +from .base import metadata +from .users import users + +users_pre_registration_details = sa.Table( + "users_pre_registration_details", + metadata, + sa.Column( + "id", + sa.BigInteger, + nullable=False, + primary_key=True, + autoincrement=True, + doc="Unique identifier for the pre-registration detail", + ), + sa.Column( + "user_id", + sa.BigInteger, + sa.ForeignKey( + users.c.id, + name="fk_users_pre_registration_details_user_id", + ondelete="CASCADE", + onupdate="CASCADE", + ), + nullable=False, + doc="Reference to the user associated with this pre-registration detail", + ), + sa.Column( + "details", + JSONB, + nullable=False, + doc="JSONB column to store pre-registration details", + ), + sa.Column( + "created_at", + sa.DateTime(), + nullable=False, + server_default=func.now(), + doc="Timestamp when the pre-registration detail was created", + ), + sa.Column( + "updated_at", + sa.DateTime(), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + doc="Timestamp when the pre-registration detail was last updated", + ), +) From b8b949a0b9fe4184c87dbc478fa5ae0218b47abe Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 3 Apr 2025 22:11:26 +0200 Subject: [PATCH 2/6] docs --- .../simcore_postgres_database/models/users.py | 9 +++ .../models/users_pre_registration_details.py | 62 ------------------- 2 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/users.py b/packages/postgres-database/src/simcore_postgres_database/models/users.py index 2e5778521d6..553146ddd63 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/users.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/users.py @@ -6,6 +6,15 @@ - The primary key is `id`, which is unique and sufficient for migration. - Ensure foreign key references (if any) are valid in the target database. - No additional changes are required; this table can be migrated as is. + +Users pre-registration details table + +- Stores details of users during the pre-registration phase. + +Migration strategy: +- The primary key is `id`, which is unique and sufficient for migration. +- Ensure foreign key references to `users` are valid in the target database. +- No additional changes are required; this table can be migrated as is. """ import sqlalchemy as sa diff --git a/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py b/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py deleted file mode 100644 index 3756d0613d5..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/users_pre_registration_details.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Users pre-registration details table - -- Stores details of users during the pre-registration phase. - -Migration strategy: -- The primary key is `id`, which is unique and sufficient for migration. -- Ensure foreign key references to `users` are valid in the target database. -- No additional changes are required; this table can be migrated as is. -""" - -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.sql import func - -from .base import metadata -from .users import users - -users_pre_registration_details = sa.Table( - "users_pre_registration_details", - metadata, - sa.Column( - "id", - sa.BigInteger, - nullable=False, - primary_key=True, - autoincrement=True, - doc="Unique identifier for the pre-registration detail", - ), - sa.Column( - "user_id", - sa.BigInteger, - sa.ForeignKey( - users.c.id, - name="fk_users_pre_registration_details_user_id", - ondelete="CASCADE", - onupdate="CASCADE", - ), - nullable=False, - doc="Reference to the user associated with this pre-registration detail", - ), - sa.Column( - "details", - JSONB, - nullable=False, - doc="JSONB column to store pre-registration details", - ), - sa.Column( - "created_at", - sa.DateTime(), - nullable=False, - server_default=func.now(), - doc="Timestamp when the pre-registration detail was created", - ), - sa.Column( - "updated_at", - sa.DateTime(), - nullable=False, - server_default=func.now(), - onupdate=func.now(), - doc="Timestamp when the pre-registration detail was last updated", - ), -) From 8956fd7aa3f1a04f21241938b830eed475388f7e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 May 2025 21:13:51 +0200 Subject: [PATCH 3/6] wrong model --- .../models/service_compatibility.py | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py diff --git a/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py b/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py deleted file mode 100644 index 105e11c4b12..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/service_compatibility.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Service compatibility table - -- Defines compatibility between services. - -Migration strategy: -- Composite primary key (`service_key`, `service_version`) is unique and sufficient for migration. -- Ensure foreign key references (if any) are valid in the target database. -- No additional changes are required; this table can be migrated as is. -""" - -import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB - -from .base import metadata - -service_compatibility = sa.Table( - "service_compatibility", - metadata, - sa.Column( - "service_key", - sa.String, - nullable=False, - doc="Service key identifier", - ), - sa.Column( - "service_version", - sa.String, - nullable=False, - doc="Service version identifier", - ), - sa.Column( - "compatibility", - JSONB, - nullable=False, - server_default=sa.text("'{}'::jsonb"), - doc="Compatibility information in JSON format", - ), - sa.PrimaryKeyConstraint("service_key", "service_version"), -) From 16a08ea015cc2c6e4a11f5d62b5035d31d5956a2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 May 2025 22:02:39 +0200 Subject: [PATCH 4/6] cleanup --- .../models/services_compatibility.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py b/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py index d151b665885..b880bf8f008 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services_compatibility.py @@ -1,14 +1,14 @@ -""" Services table +"""Services table - - List of 3rd party services in the framework - - Services have a key, version, and access rights defined by group ids +- List of 3rd party services in the framework +- Services have a key, version, and access rights defined by group ids """ +from typing import NotRequired, Required import sqlalchemy as sa import typing_extensions from sqlalchemy.dialects.postgresql import JSONB -from typing_extensions import NotRequired, Required from ._common import ( RefActions, From b16a41b9adca1493285c2ad1fc4922026efc9788 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 May 2025 22:05:25 +0200 Subject: [PATCH 5/6] ruffed --- .../src/simcore_postgres_database/__init__.py | 2 +- .../simcore_postgres_database/aiopg_errors.py | 4 +-- .../src/simcore_postgres_database/cli.py | 6 ++--- .../src/simcore_postgres_database/settings.py | 2 +- .../sidecar_models.py | 11 ++++---- .../storage_models.py | 13 ++++----- .../simcore_postgres_database/utils_cli.py | 14 ++++------ .../utils_projects_metadata.py | 4 +-- .../webserver_models.py | 27 ++++++++++--------- 9 files changed, 39 insertions(+), 44 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/__init__.py b/packages/postgres-database/src/simcore_postgres_database/__init__.py index 9a34ac80a31..4bb7e3af5db 100644 --- a/packages/postgres-database/src/simcore_postgres_database/__init__.py +++ b/packages/postgres-database/src/simcore_postgres_database/__init__.py @@ -7,8 +7,8 @@ __all__: tuple[str, ...] = ( "metadata", - "webserver_models", "storage_models", + "webserver_models", ) # nopycln: file diff --git a/packages/postgres-database/src/simcore_postgres_database/aiopg_errors.py b/packages/postgres-database/src/simcore_postgres_database/aiopg_errors.py index 730d6f630ac..8b4f3d4db57 100644 --- a/packages/postgres-database/src/simcore_postgres_database/aiopg_errors.py +++ b/packages/postgres-database/src/simcore_postgres_database/aiopg_errors.py @@ -48,9 +48,9 @@ __all__: tuple[str, ...] = ( "CheckViolation", - "DatabaseError", - "DataError", "DBAPIError", + "DataError", + "DatabaseError", "ForeignKeyViolation", "IntegrityError", "InterfaceError", diff --git a/packages/postgres-database/src/simcore_postgres_database/cli.py b/packages/postgres-database/src/simcore_postgres_database/cli.py index 5fa3cf22025..7500b223f55 100644 --- a/packages/postgres-database/src/simcore_postgres_database/cli.py +++ b/packages/postgres-database/src/simcore_postgres_database/cli.py @@ -1,6 +1,4 @@ -""" command line interface for migration - -""" +"""command line interface for migration""" # pylint: disable=wildcard-import # pylint: disable=unused-wildcard-import @@ -127,7 +125,7 @@ def _test_swarm() -> dict: return cfg - except Exception as err: # pylint: disable=broad-except # noqa: PERF203 + except Exception as err: # pylint: disable=broad-except inline_msg = str(err).replace("\n", ". ") click.echo(f"<- {test.__name__} failed : {inline_msg}") diff --git a/packages/postgres-database/src/simcore_postgres_database/settings.py b/packages/postgres-database/src/simcore_postgres_database/settings.py index 58bcea4952e..8f491e1ab41 100644 --- a/packages/postgres-database/src/simcore_postgres_database/settings.py +++ b/packages/postgres-database/src/simcore_postgres_database/settings.py @@ -7,4 +7,4 @@ ] -__all__ = ["target_metadatas", "build_url"] +__all__ = ["build_url", "target_metadatas"] diff --git a/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py b/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py index e33181a251f..83c747a9ae8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py +++ b/packages/postgres-database/src/simcore_postgres_database/sidecar_models.py @@ -1,16 +1,17 @@ -""" Facade for sidecar service +"""Facade for sidecar service - Facade to direct access to models in the database by - the sidecar service +Facade to direct access to models in the database by +the sidecar service """ + from .models.comp_pipeline import StateType, comp_pipeline from .models.comp_tasks import comp_tasks __all__ = ( - "comp_tasks", + "StateType", "comp_pipeline", "comp_pipeline", - "StateType", + "comp_tasks", ) # nopycln: file diff --git a/packages/postgres-database/src/simcore_postgres_database/storage_models.py b/packages/postgres-database/src/simcore_postgres_database/storage_models.py index d4a48d37427..9a93ec02aff 100644 --- a/packages/postgres-database/src/simcore_postgres_database/storage_models.py +++ b/packages/postgres-database/src/simcore_postgres_database/storage_models.py @@ -1,8 +1,9 @@ -""" Facade for storage service (tables manager) +"""Facade for storage service (tables manager) - - Facade to direct access to models in the database by - the storage service +- Facade to direct access to models in the database by +the storage service """ + from .models.base import metadata from .models.file_meta_data import file_meta_data from .models.groups import groups, user_to_groups @@ -11,12 +12,12 @@ from .models.users import users __all__ = [ - "tokens", "file_meta_data", + "groups", "metadata", "projects", - "users", - "groups", + "tokens", "user_to_groups", + "users", ] # nopycln: file diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_cli.py b/packages/postgres-database/src/simcore_postgres_database/utils_cli.py index b5866f94962..453518eaa02 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_cli.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_cli.py @@ -57,17 +57,13 @@ def get_service_published_port(service_name: str) -> int: s for s in client.services.list() if service_name in getattr(s, "name", "") ] if not services: - raise RuntimeError( - "Cannot find published port for service '%s'. Probably services still not up" - % service_name - ) + msg = f"Cannot find published port for service '{service_name}'. Probably services still not up" + raise RuntimeError(msg) service_endpoint = services[0].attrs["Endpoint"] if "Ports" not in service_endpoint or not service_endpoint["Ports"]: - raise RuntimeError( - "Cannot find published port for service '%s' in endpoint. Probably services still not up" - % service_name - ) + msg = f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not up" + raise RuntimeError(msg) published_port = service_endpoint["Ports"][0]["PublishedPort"] return int(published_port) @@ -87,7 +83,7 @@ def load_cache(*, raise_if_error=False) -> dict: def reset_cache(): if os.path.exists(DISCOVERED_CACHE): os.remove(DISCOVERED_CACHE) - click.echo("Removed %s" % DISCOVERED_CACHE) + click.echo(f"Removed {DISCOVERED_CACHE}") def get_alembic_config_from_cache( diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py index 6451f1156ba..7418dee7486 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_metadata.py @@ -122,9 +122,7 @@ async def _project_has_any_child( get_stmt = sa.select(projects_metadata.c.project_uuid).where( projects_metadata.c.parent_project_uuid == f"{project_uuid}" ) - if await connection.scalar(get_stmt) is not None: - return True - return False + return await connection.scalar(get_stmt) is not None async def _compute_root_parent_from_parent( diff --git a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py index b62a2fd83fe..8daf64fb8c1 100644 --- a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py +++ b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py @@ -1,9 +1,10 @@ -""" Facade for webserver service +"""Facade for webserver service - Facade to direct access to models in the database by - the webserver service +Facade to direct access to models in the database by +the webserver service """ + from .models.api_keys import api_keys from .models.classifiers import group_classifiers from .models.comp_pipeline import StateType, comp_pipeline @@ -21,29 +22,29 @@ from .models.users import UserRole, UserStatus, users __all__ = ( + "DB_CHANNEL_NAME", + "ConfirmationAction", + "GroupType", + "NodeClass", + "ProjectType", + "StateType", + "UserRole", + "UserStatus", "api_keys", "comp_pipeline", "comp_tasks", - "ConfirmationAction", "confirmations", - "DB_CHANNEL_NAME", "group_classifiers", "groups", - "GroupType", - "NodeClass", "products", "projects", "projects_nodes", - "ProjectType", - "scicrunch_resources", - "StateType", "projects_tags", + "projects_to_wallet", + "scicrunch_resources", "tags", "tokens", "user_to_groups", - "UserRole", "users", - "UserStatus", - "projects_to_wallet", ) # nopycln: file From cad45dae3b88811992cb8b7849995e9656f78d09 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 1 May 2025 22:07:25 +0200 Subject: [PATCH 6/6] migration --- .../e33493b49140_drop_user_to_projects.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e33493b49140_drop_user_to_projects.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e33493b49140_drop_user_to_projects.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e33493b49140_drop_user_to_projects.py new file mode 100644 index 00000000000..8cd774c777b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e33493b49140_drop_user_to_projects.py @@ -0,0 +1,48 @@ +"""drop user_to_projects + +Revision ID: e33493b49140 +Revises: 0d52976dc616 +Create Date: 2025-05-01 20:06:49.642271+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e33493b49140" +down_revision = "0d52976dc616" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("user_to_projects") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "user_to_projects", + sa.Column("id", sa.BIGINT(), autoincrement=True, nullable=False), + sa.Column("user_id", sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column("project_id", sa.BIGINT(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + name="fk_user_to_projects_id_projects", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_user_to_projects_id_users", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="user_to_projects_pkey"), + ) + # ### end Alembic commands ###