Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,9 @@

"""

import json
from pathlib import Path

import sqlalchemy as sa
from alembic import op
from sqlalchemy import update
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import Session

from zimfarm_backend.common.schemas.offliners.models import OfflinerSpecSchema
from zimfarm_backend.db.models import RequestedTask, Schedule, Task
from zimfarm_backend.db.offliner_definition import create_offliner_definition

# revision identifiers, used by Alembic.
revision = "12096716adfd"
Expand Down Expand Up @@ -80,42 +71,6 @@ def upgrade() -> None:
["id"],
)

data_dir = Path(__file__).parent.parent / "initial_offliner_definitions"

bind = op.get_bind()
session = Session(bind=bind)

for file in data_dir.glob("*.json"):
data = json.loads(file.read_text())

offliner_definition = create_offliner_definition(
session,
offliner=data["offliner_id"],
schema=OfflinerSpecSchema.model_validate(data),
version="initial",
)
# Make all the schedules point to the initial version
session.execute(
update(Schedule)
.where(
Schedule.config["offliner"]["offliner_id"].astext == data["offliner_id"]
)
.values({"offliner_definition_id": offliner_definition.id})
)
session.execute(
update(Task)
.where(Task.config["offliner"]["offliner_id"].astext == data["offliner_id"])
.values({"offliner_definition_id": offliner_definition.id})
)
session.execute(
update(RequestedTask)
.where(
RequestedTask.config["offliner"]["offliner_id"].astext
== data["offliner_id"]
)
.values({"offliner_definition_id": offliner_definition.id})
)

# Reset the foreign keys to be non-nullable
op.alter_column(
"schedule",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@

import sqlalchemy as sa
from alembic import op
from sqlalchemy.orm import Session

from zimfarm_backend.db.offliner import create_offliner

# revision identifiers, used by Alembic.
revision = "3e43525210c1"
Expand All @@ -23,112 +20,6 @@ def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("offliner", sa.Column("ci_secret_hash", sa.String(), nullable=True))
# ### end Alembic commands ###
session = Session(bind=op.get_bind())
create_offliner(
session,
offliner_id="devdocs",
base_model="DashModel",
docker_image_name="openzim/devdocs",
command_name="devdocs2zim",
)
create_offliner(
session,
offliner_id="freecodecamp",
base_model="DashModel",
docker_image_name="openzim/freecodecamp",
command_name="freecodecamp2zim",
)
create_offliner(
session,
offliner_id="gutenberg",
base_model="DashModel",
docker_image_name="openzim/gutenberg",
command_name="gutenberg2zim",
)
create_offliner(
session,
offliner_id="sotoki",
base_model="DashModel",
docker_image_name="openzim/sotoki",
command_name="sotoki2zim",
)
create_offliner(
session,
offliner_id="wikihow",
base_model="DashModel",
docker_image_name="openzim/wikihow",
command_name="wikihow2zim",
)
create_offliner(
session,
offliner_id="ifixit",
base_model="DashModel",
docker_image_name="openzim/ifixit",
command_name="ifixit2zim",
)
create_offliner(
session,
offliner_id="mwoffliner",
base_model="DashModel",
docker_image_name="openzim/mwoffliner",
command_name="mwoffliner",
)
create_offliner(
session,
offliner_id="youtube",
base_model="DashModel",
docker_image_name="openzim/youtube",
command_name="youtube2zim",
)
create_offliner(
session,
offliner_id="ted",
base_model="DashModel",
docker_image_name="openzim/ted",
command_name="ted2zim",
)
create_offliner(
session,
offliner_id="openedx",
base_model="DashModel",
docker_image_name="openzim/openedx",
command_name="openedx2zim",
)
create_offliner(
session,
offliner_id="nautilus",
base_model="DashModel",
docker_image_name="openzim/nautilus",
command_name="nautiluszim",
)
create_offliner(
session,
offliner_id="zimit",
base_model="CamelModel",
docker_image_name="openzim/zimit",
command_name="zimit",
)
create_offliner(
session,
offliner_id="kolibri",
base_model="DashModel",
docker_image_name="openzim/kolibri",
command_name="kolibri2zim",
)
create_offliner(
session,
offliner_id="mindtouch",
base_model="DashModel",
docker_image_name="openzim/mindtouch",
command_name="mindtouch2zim",
)
create_offliner(
session,
offliner_id="phet",
base_model="DashModel",
docker_image_name="openzim/phet",
command_name="phet2zim",
)


def downgrade() -> None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,7 @@

import sqlalchemy as sa
from alembic import op
from sqlalchemy import select
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import Session

from zimfarm_backend.common import getnow
from zimfarm_backend.db.models import Schedule, ScheduleHistory

# revision identifiers, used by Alembic.
revision = "60c546babd23"
Expand All @@ -23,8 +18,6 @@


def upgrade() -> None:
bind = op.get_bind()
session = Session(bind=bind)
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"schedule_history",
Expand Down Expand Up @@ -54,23 +47,6 @@ def upgrade() -> None:
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_schedule_history")),
)
with session.begin():
for schedule in session.scalars(select(Schedule)):
history_entry = ScheduleHistory(
author="admin",
created_at=getnow(),
comment="Initial history created automatically",
config=schedule.config,
name=schedule.name,
category=schedule.category,
enabled=schedule.enabled,
language_code=schedule.language_code,
tags=schedule.tags,
periodicity=schedule.periodicity,
context=schedule.context,
)
schedule.history_entries.append(history_entry)

op.create_index(op.f("ix_schedule_context"), "schedule", ["context"], unique=False)
# ### end Alembic commands ###

Expand Down
54 changes: 30 additions & 24 deletions backend/src/zimfarm_backend/routes/offliners/logic.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,20 @@
from fastapi import APIRouter, Body, Depends, Path, Query, Response
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session as OrmSession
from werkzeug.security import check_password_hash
from werkzeug.security import check_password_hash, generate_password_hash

from zimfarm_backend.common.schemas.fields import LimitFieldMax200, SkipField
from zimfarm_backend.common.roles import RoleEnum, get_role_for
from zimfarm_backend.common.schemas.fields import (
LimitFieldMax200,
SkipField,
)
from zimfarm_backend.common.schemas.models import (
calculate_pagination_metadata,
)
from zimfarm_backend.common.schemas.offliners.builder import build_offliner_model
from zimfarm_backend.common.schemas.offliners.serializer import schema_to_flags
from zimfarm_backend.db.models import User
from zimfarm_backend.db.offliner import create_offliner as db_create_offliner
from zimfarm_backend.db.offliner import get_all_offliners
from zimfarm_backend.db.offliner import get_offliner as db_get_offliner
from zimfarm_backend.db.offliner_definition import (
Expand All @@ -23,10 +29,11 @@
from zimfarm_backend.db.offliner_definition import (
get_offliner_versions as db_get_offliner_versions,
)
from zimfarm_backend.routes.dependencies import gen_dbsession
from zimfarm_backend.routes.dependencies import gen_dbsession, get_current_user
from zimfarm_backend.routes.http_errors import UnauthorizedError
from zimfarm_backend.routes.models import ListResponse
from zimfarm_backend.routes.offliners.models import (
OfflinerCreateSchema,
OfflinerDefinitionCreateSchema,
)

Expand All @@ -50,30 +57,29 @@
)


@router.get("/{offliner_id}")
async def get_initial_offliner_version(
offliner_id: Annotated[str, Path()],
@router.post("")
async def create_offliner(
request: OfflinerCreateSchema,
session: Annotated[OrmSession, Depends(gen_dbsession)],
) -> JSONResponse:
"""Get a specific offliner"""

# find the schema class that matches the offliner
offliner = db_get_offliner(session, offliner_id)
offliner_definition = db_get_offliner_definition(
session, offliner_id=offliner_id, version="initial"
current_user: User = Depends(get_current_user),
) -> Response:
"""Create an offliner"""
# TODO: Should we define new permissions for the current user that allows them to

Check notice on line 67 in backend/src/zimfarm_backend/routes/offliners/logic.py

View check run for this annotation

codefactor.io / CodeFactor

backend/src/zimfarm_backend/routes/offliners/logic.py#L67

Unresolved comment '# TODO: Should we define new permissions for the current user that allows them to'. (C100)
# register new offliners? Because existing permissions overlap and it might not be
# feasible for someone who can create a schedule to create an offliner.
if not (current_user.scope and get_role_for(current_user.scope) == RoleEnum.ADMIN):
raise UnauthorizedError("You do not have permissions to create an offliner.")

db_create_offliner(
session,
offliner_id=request.offliner_id,
base_model=request.base_model,
docker_image_name=request.docker_image_name,
command_name=request.command_name,
ci_secret_hash=generate_password_hash(request.ci_secret_hash),
)
schema_cls = build_offliner_model(offliner, offliner_definition.schema_)

flags = schema_to_flags(schema_cls)

return JSONResponse(
content={
"flags": [flag.model_dump(mode="json", by_alias=True) for flag in flags],
"help": ( # dynamic + sourced from backend because it might be custom
f"https://github.com/openzim/{offliner}/wiki/Frequently-Asked-Questions"
),
}
)
return Response(status_code=HTTPStatus.CREATED)


@router.get("/{offliner_id}/versions")
Expand Down
11 changes: 11 additions & 0 deletions backend/src/zimfarm_backend/routes/offliners/models.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,19 @@
from typing import Literal

from zimfarm_backend.common.enums import DockerImageName
from zimfarm_backend.common.schemas import BaseModel
from zimfarm_backend.common.schemas.fields import NotEmptyString
from zimfarm_backend.common.schemas.offliners.models import OfflinerSpecSchema


class OfflinerCreateSchema(BaseModel):
offliner_id: NotEmptyString
base_model: Literal["CamelModel", "DashModel"]
docker_image_name: DockerImageName
ci_secret_hash: NotEmptyString
command_name: NotEmptyString


class OfflinerDefinitionCreateSchema(BaseModel):
version: NotEmptyString
ci_secret: NotEmptyString
Expand Down
2 changes: 1 addition & 1 deletion backend/src/zimfarm_backend/routes/schedules/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class ScheduleCreateSchema(BaseModel):
periodicity: SchedulePeriodicity
tags: list[NotEmptyString] = Field(default_factory=list)
enabled: bool
version: str = "initial" # version of offliner to use for validation
version: NotEmptyString
config: dict[str, Any] # will be validated in the route
notification: ScheduleNotificationSchema | None = None
context: NotEmptyString | None = None
Expand Down
Loading