Allow scheduling products for product increments

* Determine scheduling parameters (`DISTRI`, `VERSION`, `FLAVOR`, `ARCH`,
  `BUILD`) dynamically from the files in the directory listing of the
  download repository of the OBS/IBS project targeted by the product
  increment request
    * No longer rely on hard-coded parameters except for the OBS/IBS
      project
    * Change the meaning of the CLI parameters to act as filters
* Add additional flag to create a scheduled product if no jobs have been
  created yet

Related ticket: https://progress.opensuse.org/issues/190251

Author: Martchus

openqabot/args.py

@@ -242,46 +242,56 @@ def get_parser():
 
     cmdincrementapprove = commands.add_parser(
         "increment-approve",
-        help="Approve the specified product increment if tests passed",
+        help="Approve the most recent product increment for an OBS project if tests passed",
     )
     cmdincrementapprove.add_argument(
         "--obs-project",
         required=False,
         type=str,
         default="SUSE:SLFO:Products:SLES:16.0:TEST",
-        help="The project on OBS",
+        help="The project on OBS to monitor, schedule jobs for (if --schedule is specified) and approve (if all tests passd)",
     )
     cmdincrementapprove.add_argument(
         "--distri",
         required=False,
         type=str,
-        default="sle",
-        help="The DISTRI parameter of relevant scheduled products on openQA",
+        default="any",
+        help="Monitor and schedule only products with the specified DISTRI parameter",
     )
     cmdincrementapprove.add_argument(
         "--version",
         required=False,
         type=str,
-        default="15.99",
-        help="The VERSION parameter of relevant scheduled products on openQA",
+        default="any",
+        help="Monitor and schedule only products with the specified VERSION parameter",
     )
     cmdincrementapprove.add_argument(
         "--flavor",
         required=False,
         type=str,
-        default="Online-Increments",
-        help="The FLAVOR parameter of relevant scheduled products on openQA",
+        default="any",
+        help="Monitor and schedule only products with the specified FLAVOR parameter",
+    )
+    cmdincrementapprove.add_argument(
+        "--schedule",
+        action="store_true",
+        help="Schedule a new product (if none exists or if the most recent product has no jobs)",
+    )
+    cmdincrementapprove.add_argument(
+        "--reschedule",
+        action="store_true",
+        help="Always schedule a new product (even if one already exists)",
     )
     cmdincrementapprove.add_argument(
         "--accepted",
         action="store_true",
-        help="Consider accepted requests/reviews as well",
+        help="Consider accepted product increment requests as well",
     )
     cmdincrementapprove.add_argument(
         "--request-id",
         required=False,
         type=int,
-        help="Approve the specified request specifically",
+        help="Check/approve the specified request (instead of the most recent one)",
     )
     cmdincrementapprove.set_defaults(func=do_increment_approve, no_config=True)
 

openqabot/incrementapprover.py

@@ -1,7 +1,8 @@
 # Copyright SUSE LLC
 # SPDX-License-Identifier: MIT
 from argparse import Namespace
-from typing import Any, Dict, List, Tuple, Optional
+from typing import Any, Dict, List, Tuple, Optional, Set, NamedTuple
+import re
 from logging import getLogger
 from pprint import pformat
 
@@ -10,13 +11,27 @@
 
 from openqabot.openqa import openQAInterface
 
-from . import OBS_GROUP, OBS_URL
+from .errors import PostOpenQAError
+from .utils import retry10 as requests
+from . import OBS_GROUP, OBS_URL, OBS_DOWNLOAD_URL
 
 log = getLogger("bot.increment_approver")
 ok_results = set(("passed", "softfailed"))
 final_states = set(("done", "cancelled"))
 
 
+class BuildInfo(NamedTuple):
+    distri: str
+    product: str
+    version: str
+    flavor: str
+    arch: str
+    build: str
+
+    def __str__(self):
+        return f"{self.product}v{self.version} build {self.build}@{self.arch} of flavor {self.flavor}"
+
+
 class IncrementApprover:
     def __init__(self, args: Namespace) -> None:
         self.args = args
@@ -35,11 +50,11 @@ def _find_request_on_obs(self) -> Optional[osc.core.Request]:
                 OBS_GROUP,
                 args.obs_project,
             )
-            requests = osc.core.get_request_list(
+            obs_requests = osc.core.get_request_list(
                 OBS_URL, project=args.obs_project, req_state=relevant_states
             )
             relevant_request = None
-            for request in sorted(requests, reverse=True):
+            for request in sorted(obs_requests, reverse=True):
                 for review in request.reviews:
                     if review.by_group == OBS_GROUP and review.state in relevant_states:
                         relevant_request = request
@@ -54,31 +69,40 @@ def _find_request_on_obs(self) -> Optional[osc.core.Request]:
             )
         else:
             log.debug("Found request %s", relevant_request.id)
+            if hasattr(relevant_request.state, "to_xml"):
+                log.debug(relevant_request.to_str())
         return relevant_request
 
-    def _request_openqa_job_results(self) -> Dict[str, Dict[str, Dict[str, Any]]]:
-        log.debug("Checking openQA job results")
-        args = self.args
-        params = {"distri": args.distri, "version": args.version, "flavor": args.flavor}
+    def _request_openqa_job_results(
+        self, build_info: BuildInfo
+    ) -> Dict[str, Dict[str, Dict[str, Any]]]:
+        log.debug("Checking openQA job results for %s", build_info)
+        params = {
+            "distri": build_info.distri,
+            "version": build_info.version,
+            "flavor": build_info.flavor,
+            "arch": build_info.arch,
+            "build": build_info.build,
+        }
         res = self.client.get_scheduled_product_stats(params)
         log.debug("Job statistics:\n%s", pformat(res))
         return res
 
-    def _are_openqa_jobs_ready(self, res: Dict[str, Dict[str, Dict[str, Any]]]) -> bool:
-        args = self.args
+    def _check_openqa_jobs(
+        self, res: Dict[str, Dict[str, Dict[str, Any]]], build_info: BuildInfo
+    ) -> Optional[bool]:
         actual_states = set(res.keys())
         pending_states = actual_states - final_states
         if len(actual_states) == 0:
             log.info(
-                "Skipping approval, there are no relevant jobs on openQA for %s-%s-%s",
-                args.distri,
-                args.version,
-                args.flavor,
+                "Skipping approval, there are no relevant jobs on openQA for %s",
+                build_info,
             )
-            return False
+            return None
         if len(pending_states):
             log.info(
-                "Skipping approval, some jobs on openQA are in pending states (%s)",
+                "Skipping approval, some jobs on openQA for %s are in pending states (%s)",
+                build_info,
                 ", ".join(sorted(pending_states)),
             )
             return False
@@ -126,11 +150,72 @@ def _handle_approval(
         log.info(message)
         return 0
 
+    def _determine_build_info(self) -> Set[BuildInfo]:
+        # deduce DISTRI, VERSION, FLAVOR, ARCH and BUILD from the spdx files in the repo listing similar to the sync plugin,
+        # e.g. https://download.suse.de/download/ibs/SUSE:/SLFO:/Products:/SLES:/16.0:/TEST/product/?jsontable=1
+        path = self.args.obs_project.replace(":", ":/")
+        url = f"{OBS_DOWNLOAD_URL}/{path}/product/?jsontable=1"
+        rows = requests.get(url).json().get("data", [])
+        res = set()
+        args = self.args
+        for row in rows:
+            m = re.search(
+                "(?P<product>.*)-(?P<version>[^\\-]*?)-(?P<flavor>\\D+[^\\-]*?)-(?P<arch>[^\\-]*?)-Build(?P<build>.*?)\\.spdx.json",
+                row.get("name", ""),
+            )
+            if m:
+                product = m.group("product")
+                version = m.group("version")
+                flavor = m.group("flavor") + "-Increments"
+                arch = m.group("arch")
+                build = m.group("build")
+                if product.startswith("SLE"):
+                    distri = "sle"
+                else:
+                    continue  # skip unknown products
+                if (
+                    args.distri in ("any", distri)
+                    and args.flavor in ("any", flavor)
+                    and args.version in ("any", version)
+                ):
+                    res.add(BuildInfo(distri, product, version, flavor, arch, build))
+        return res
+
+    def _schedule_openqa_jobs(self, build_info: BuildInfo) -> int:
+        log.info("Scheduling jobs for %s", build_info)
+        if self.args.dry:
+            return 0
+        try:
+            self.client.post_job(  # create a scheduled product with build info from spdx file
+                {
+                    "DISTRI": build_info.distri,
+                    "VERSION": build_info.version,
+                    "FLAVOR": build_info.flavor,
+                    "ARCH": build_info.arch,
+                    "BUILD": build_info.build,
+                }
+            )
+            return 0
+        except PostOpenQAError:
+            return 1
+
     def __call__(self) -> int:
+        error_count = 0
         request = self._find_request_on_obs()
         if request is None:
-            return 0
-        res = self._request_openqa_job_results()
-        if not self._are_openqa_jobs_ready(res):
-            return 0
-        return self._handle_approval(request, *(self._evaluate_openqa_job_results(res)))
+            return error_count
+        for build_info in self._determine_build_info():
+            res = self._request_openqa_job_results(build_info)
+            if self.args.reschedule:
+                error_count += self._schedule_openqa_jobs(build_info)
+                continue
+            openqa_jobs_ready = self._check_openqa_jobs(res, build_info)
+            if openqa_jobs_ready is None and self.args.schedule:
+                error_count += self._schedule_openqa_jobs(build_info)
+                continue
+            if not openqa_jobs_ready:
+                continue
+            error_count += self._handle_approval(
+                request, *(self._evaluate_openqa_job_results(res))
+            )
+        return error_count

responses/test-product-repo.json

@@ -0,0 +1,12 @@
+{
+   "data": [
+      {"name": "SLES-16.0-Online-x86_64-Build139.1.spdx.json"},
+      {"name": "SLES-16.0-Online-aarch64-Build139.1.spdx.json"},
+      {"name": "SLES-16.0-Online-s390x-Build139.1.spdx.json"},
+      {"name": "SLES-16.0-Online-ppc64le-Build139.1.spdx.json"},
+      {"name": "SLES-SAP-16.0-Online-x86_64-Build139.1.spdx.json"},
+      {"name": "SLES-SAP-16.0-Online-aarch64-Build139.1.spdx.json"},
+      {"name": "SLES-SAP-16.0-Online-s390x-Build139.1.spdx.json"},
+      {"name": "SLES-SAP-16.0-Online-ppc64le-Build139.1.spdx.json"}
+   ]
+}