remove model queries in alembic migration

backend/src/zimfarm_backend/migrations/versions/12096716adfd_add_offliner_definitions.py

@@ -6,18 +6,9 @@
 
 """
 
-import json
-from pathlib import Path
-
 import sqlalchemy as sa
 from alembic import op
-from sqlalchemy import update
 from sqlalchemy.dialects import postgresql
-from sqlalchemy.orm import Session
-
-from zimfarm_backend.common.schemas.offliners.models import OfflinerSpecSchema
-from zimfarm_backend.db.models import RequestedTask, Schedule, Task
-from zimfarm_backend.db.offliner_definition import create_offliner_definition
 
 # revision identifiers, used by Alembic.
 revision = "12096716adfd"
@@ -80,42 +71,6 @@ def upgrade() -> None:
         ["id"],
     )
 
-    data_dir = Path(__file__).parent.parent / "initial_offliner_definitions"
-
-    bind = op.get_bind()
-    session = Session(bind=bind)
-
-    for file in data_dir.glob("*.json"):
-        data = json.loads(file.read_text())
-
-        offliner_definition = create_offliner_definition(
-            session,
-            offliner=data["offliner_id"],
-            schema=OfflinerSpecSchema.model_validate(data),
-            version="initial",
-        )
-        # Make all the schedules point to the initial version
-        session.execute(
-            update(Schedule)
-            .where(
-                Schedule.config["offliner"]["offliner_id"].astext == data["offliner_id"]
-            )
-            .values({"offliner_definition_id": offliner_definition.id})
-        )
-        session.execute(
-            update(Task)
-            .where(Task.config["offliner"]["offliner_id"].astext == data["offliner_id"])
-            .values({"offliner_definition_id": offliner_definition.id})
-        )
-        session.execute(
-            update(RequestedTask)
-            .where(
-                RequestedTask.config["offliner"]["offliner_id"].astext
-                == data["offliner_id"]
-            )
-            .values({"offliner_definition_id": offliner_definition.id})
-        )
-
     # Reset the foreign keys to be non-nullable
     op.alter_column(
         "schedule",

backend/src/zimfarm_backend/migrations/versions/3e43525210c1_add_ci_secret_has_to_offliner.py

@@ -8,9 +8,6 @@
 
 import sqlalchemy as sa
 from alembic import op
-from sqlalchemy.orm import Session
-
-from zimfarm_backend.db.offliner import create_offliner
 
 # revision identifiers, used by Alembic.
 revision = "3e43525210c1"
@@ -23,112 +20,6 @@ def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("offliner", sa.Column("ci_secret_hash", sa.String(), nullable=True))
     # ### end Alembic commands ###
-    session = Session(bind=op.get_bind())
-    create_offliner(
-        session,
-        offliner_id="devdocs",
-        base_model="DashModel",
-        docker_image_name="openzim/devdocs",
-        command_name="devdocs2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="freecodecamp",
-        base_model="DashModel",
-        docker_image_name="openzim/freecodecamp",
-        command_name="freecodecamp2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="gutenberg",
-        base_model="DashModel",
-        docker_image_name="openzim/gutenberg",
-        command_name="gutenberg2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="sotoki",
-        base_model="DashModel",
-        docker_image_name="openzim/sotoki",
-        command_name="sotoki2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="wikihow",
-        base_model="DashModel",
-        docker_image_name="openzim/wikihow",
-        command_name="wikihow2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="ifixit",
-        base_model="DashModel",
-        docker_image_name="openzim/ifixit",
-        command_name="ifixit2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="mwoffliner",
-        base_model="DashModel",
-        docker_image_name="openzim/mwoffliner",
-        command_name="mwoffliner",
-    )
-    create_offliner(
-        session,
-        offliner_id="youtube",
-        base_model="DashModel",
-        docker_image_name="openzim/youtube",
-        command_name="youtube2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="ted",
-        base_model="DashModel",
-        docker_image_name="openzim/ted",
-        command_name="ted2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="openedx",
-        base_model="DashModel",
-        docker_image_name="openzim/openedx",
-        command_name="openedx2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="nautilus",
-        base_model="DashModel",
-        docker_image_name="openzim/nautilus",
-        command_name="nautiluszim",
-    )
-    create_offliner(
-        session,
-        offliner_id="zimit",
-        base_model="CamelModel",
-        docker_image_name="openzim/zimit",
-        command_name="zimit",
-    )
-    create_offliner(
-        session,
-        offliner_id="kolibri",
-        base_model="DashModel",
-        docker_image_name="openzim/kolibri",
-        command_name="kolibri2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="mindtouch",
-        base_model="DashModel",
-        docker_image_name="openzim/mindtouch",
-        command_name="mindtouch2zim",
-    )
-    create_offliner(
-        session,
-        offliner_id="phet",
-        base_model="DashModel",
-        docker_image_name="openzim/phet",
-        command_name="phet2zim",
-    )
 
 
 def downgrade() -> None:

backend/src/zimfarm_backend/migrations/versions/60c546babd23_add_schedule_history.py

@@ -8,12 +8,7 @@
 
 import sqlalchemy as sa
 from alembic import op
-from sqlalchemy import select
 from sqlalchemy.dialects import postgresql
-from sqlalchemy.orm import Session
-
-from zimfarm_backend.common import getnow
-from zimfarm_backend.db.models import Schedule, ScheduleHistory
 
 # revision identifiers, used by Alembic.
 revision = "60c546babd23"
@@ -23,8 +18,6 @@
 
 
 def upgrade() -> None:
-    bind = op.get_bind()
-    session = Session(bind=bind)
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         "schedule_history",
@@ -54,23 +47,6 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id", name=op.f("pk_schedule_history")),
     )
-    with session.begin():
-        for schedule in session.scalars(select(Schedule)):
-            history_entry = ScheduleHistory(
-                author="admin",
-                created_at=getnow(),
-                comment="Initial history created automatically",
-                config=schedule.config,
-                name=schedule.name,
-                category=schedule.category,
-                enabled=schedule.enabled,
-                language_code=schedule.language_code,
-                tags=schedule.tags,
-                periodicity=schedule.periodicity,
-                context=schedule.context,
-            )
-            schedule.history_entries.append(history_entry)
-
     op.create_index(op.f("ix_schedule_context"), "schedule", ["context"], unique=False)
     # ### end Alembic commands ###
 

backend/src/zimfarm_backend/routes/offliners/logic.py

@@ -4,14 +4,20 @@
 from fastapi import APIRouter, Body, Depends, Path, Query, Response
 from fastapi.responses import JSONResponse
 from sqlalchemy.orm import Session as OrmSession
-from werkzeug.security import check_password_hash
+from werkzeug.security import check_password_hash, generate_password_hash
 
-from zimfarm_backend.common.schemas.fields import LimitFieldMax200, SkipField
+from zimfarm_backend.common.roles import RoleEnum, get_role_for
+from zimfarm_backend.common.schemas.fields import (
+    LimitFieldMax200,
+    SkipField,
+)
 from zimfarm_backend.common.schemas.models import (
     calculate_pagination_metadata,
 )
 from zimfarm_backend.common.schemas.offliners.builder import build_offliner_model
 from zimfarm_backend.common.schemas.offliners.serializer import schema_to_flags
+from zimfarm_backend.db.models import User
+from zimfarm_backend.db.offliner import create_offliner as db_create_offliner
 from zimfarm_backend.db.offliner import get_all_offliners
 from zimfarm_backend.db.offliner import get_offliner as db_get_offliner
 from zimfarm_backend.db.offliner_definition import (
@@ -23,10 +29,11 @@
 from zimfarm_backend.db.offliner_definition import (
     get_offliner_versions as db_get_offliner_versions,
 )
-from zimfarm_backend.routes.dependencies import gen_dbsession
+from zimfarm_backend.routes.dependencies import gen_dbsession, get_current_user
 from zimfarm_backend.routes.http_errors import UnauthorizedError
 from zimfarm_backend.routes.models import ListResponse
 from zimfarm_backend.routes.offliners.models import (
+    OfflinerCreateSchema,
     OfflinerDefinitionCreateSchema,
 )
 
@@ -50,30 +57,29 @@
     )
 
 
-@router.get("/{offliner_id}")
-async def get_initial_offliner_version(
-    offliner_id: Annotated[str, Path()],
+@router.post("")
+async def create_offliner(
+    request: OfflinerCreateSchema,
     session: Annotated[OrmSession, Depends(gen_dbsession)],
-) -> JSONResponse:
-    """Get a specific offliner"""
-
-    # find the schema class that matches the offliner
-    offliner = db_get_offliner(session, offliner_id)
-    offliner_definition = db_get_offliner_definition(
-        session, offliner_id=offliner_id, version="initial"
+    current_user: User = Depends(get_current_user),
+) -> Response:
+    """Create an offliner"""
+    # TODO: Should we define new permissions for the current user that allows them to
+    # register new offliners? Because existing permissions overlap and it might not be
+    # feasible for someone who can create a schedule to create an offliner.
+    if not (current_user.scope and get_role_for(current_user.scope) == RoleEnum.ADMIN):
+        raise UnauthorizedError("You do not have permissions to create an offliner.")
+
+    db_create_offliner(
+        session,
+        offliner_id=request.offliner_id,
+        base_model=request.base_model,
+        docker_image_name=request.docker_image_name,
+        command_name=request.command_name,
+        ci_secret_hash=generate_password_hash(request.ci_secret_hash),
     )
-    schema_cls = build_offliner_model(offliner, offliner_definition.schema_)
 
-    flags = schema_to_flags(schema_cls)
-
-    return JSONResponse(
-        content={
-            "flags": [flag.model_dump(mode="json", by_alias=True) for flag in flags],
-            "help": (  # dynamic + sourced from backend because it might be custom
-                f"https://github.com/openzim/{offliner}/wiki/Frequently-Asked-Questions"
-            ),
-        }
-    )
+    return Response(status_code=HTTPStatus.CREATED)
 
 
 @router.get("/{offliner_id}/versions")

backend/src/zimfarm_backend/routes/offliners/models.py

@@ -1,8 +1,19 @@
+from typing import Literal
+
+from zimfarm_backend.common.enums import DockerImageName
 from zimfarm_backend.common.schemas import BaseModel
 from zimfarm_backend.common.schemas.fields import NotEmptyString
 from zimfarm_backend.common.schemas.offliners.models import OfflinerSpecSchema
 
 
+class OfflinerCreateSchema(BaseModel):
+    offliner_id: NotEmptyString
+    base_model: Literal["CamelModel", "DashModel"]
+    docker_image_name: DockerImageName
+    ci_secret_hash: NotEmptyString
+    command_name: NotEmptyString
+
+
 class OfflinerDefinitionCreateSchema(BaseModel):
     version: NotEmptyString
     ci_secret: NotEmptyString

backend/src/zimfarm_backend/routes/schedules/models.py

@@ -41,7 +41,7 @@ class ScheduleCreateSchema(BaseModel):
     periodicity: SchedulePeriodicity
     tags: list[NotEmptyString] = Field(default_factory=list)
     enabled: bool
-    version: str = "initial"  # version of offliner to use for validation
+    version: NotEmptyString
     config: dict[str, Any]  # will be validated in the route
     notification: ScheduleNotificationSchema | None = None
     context: NotEmptyString | None = None