|
| 1 | +import argparse |
| 2 | +import base64 |
| 3 | +import logging |
| 4 | +import json |
| 5 | +import os |
| 6 | +import re |
| 7 | +import subprocess |
| 8 | +import sys |
| 9 | +import yaml |
| 10 | + |
| 11 | +from urllib.parse import urljoin |
| 12 | + |
| 13 | +from jinja2 import FileSystemLoader, Environment, StrictUndefined |
| 14 | + |
| 15 | + |
| 16 | +logger = logging.getLogger(__name__) |
| 17 | + |
| 18 | +LOGO_DOWNSTREAM_FILE = "img/downstream_logo.png" |
| 19 | +LOGO_UPSTREAM_FILE = "img/upstream_logo.png" |
| 20 | + |
| 21 | +# Default location for the image |
| 22 | +REGISTRY_HOST = "quay.io" |
| 23 | +REGISTRY_API_BASE = REGISTRY_HOST + "/api/v1/" |
| 24 | + |
| 25 | +CSO_REPO = "projectquay/container-security-operator" |
| 26 | +CSO_IMAGE = REGISTRY_HOST + "/" + CSO_REPO |
| 27 | +CSO_IMAGE_TAG = "master" |
| 28 | + |
| 29 | +CSO_CATALOG_REPO = "projectquay/cso-catalog" |
| 30 | +CSO_CATALOG_IMAGE = REGISTRY_HOST + "/" + CSO_CATALOG_REPO |
| 31 | +CSO_CATALOG_IMAGE_TAG = "master" |
| 32 | + |
| 33 | +# Default template values |
| 34 | +K8S_API_VERSION = "v1alpha1" |
| 35 | + |
| 36 | +# Jinja templates |
| 37 | +TEMPLATE_DIR = "templates" |
| 38 | +CSV_TEMPLATE_FILE = "container-security-operator.clusterserviceversion.yaml.jnj" |
| 39 | +CRD_TEMPLATE_FILES = [ |
| 40 | + "imagemanifestvulns.secscan.quay.redhat.com.crd.yaml.jnj" |
| 41 | +] |
| 42 | + |
| 43 | +# Output |
| 44 | +OUTPUT_MANIFEST_DIR = os.path.join("manifests", "container-security-operator") |
| 45 | +OUTPUT_CATALOG_FILE = "cso.catalogsource.yaml" |
| 46 | + |
| 47 | +MANIFEST_DIGEST_REGEX = re.compile(r"sha256:[a-z0-9]{64}") |
| 48 | +ARGUMENT_REGEX = re.compile(r"(-[\w])|(--[\w]+)") |
| 49 | +VERSION_REGEX = re.compile(r"^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$") |
| 50 | +MASTER_VERSION_REGEX = re.compile(r"^master$") |
| 51 | + |
| 52 | + |
| 53 | +def normalize_version(version): |
| 54 | + if VERSION_REGEX.match(version): |
| 55 | + return version[1:] |
| 56 | + return version |
| 57 | + |
| 58 | + |
| 59 | +def get_image_manifest_digest(image_ref, cmd="docker"): |
| 60 | + """ Return the repo and manifest digest of a given image reference. |
| 61 | + e.g quay.io/namespace/repo:tag -> (quay.io/namespace/repo, sha256:123456) |
| 62 | + """ |
| 63 | + if len(image_ref.split("@")) == 2: |
| 64 | + # Still pull for a digest ref, to make sure the image exists |
| 65 | + repo, digest = image_ref.split("@") |
| 66 | + pull_command = [cmd, "pull", repo+"@"+tag] |
| 67 | + inspect_command = [cmd, "inspect", repo+"@"+tag] |
| 68 | + else: |
| 69 | + repo, tag = image_ref.split(":") |
| 70 | + pull_command = [cmd, "pull", repo+":"+tag] |
| 71 | + inspect_command = [cmd, "inspect", repo+":"+tag] |
| 72 | + |
| 73 | + try: |
| 74 | + subprocess.run(pull_command, check=True) |
| 75 | + out = subprocess.run(inspect_command, check=True, capture_output=True) |
| 76 | + parsed = json.loads(out.stdout) |
| 77 | + repo_digests = parsed[0]["RepoDigests"] |
| 78 | + except subprocess.CalledProcessError as cpe: |
| 79 | + logger.error("Error running docker commands for image %s:%s - %s", repo, tag, cpe) |
| 80 | + return None, None |
| 81 | + except ValueError as ve: |
| 82 | + logger.error("Error parsing docker inspect output output - %s", ve) |
| 83 | + return None, None |
| 84 | + except Exception as e: |
| 85 | + logger.error("Error getting the manifest digest for image %s:%s - %s", repo, tag, e) |
| 86 | + return None, None |
| 87 | + |
| 88 | + repo_digests = list(filter(lambda repo_digest: repo_digest.startswith(repo),repo_digests)) |
| 89 | + if len(repo_digests) == 0: |
| 90 | + logger.error("Could not find the manifest digest for the given image %s:%s", repo, tag) |
| 91 | + return None, None |
| 92 | + |
| 93 | + manifest_digest = repo_digests[0].split("@")[-1] |
| 94 | + if not MANIFEST_DIGEST_REGEX.match(manifest_digest): |
| 95 | + logger.error("Unknown manifest digest format for %s:%s -> %s", repo_digest, manifest_digest) |
| 96 | + return None, None |
| 97 | + |
| 98 | + return repo, manifest_digest |
| 99 | + |
| 100 | + |
| 101 | +def get_b64_logo_from_file(filepath): |
| 102 | + with open(filepath, 'rb') as f: |
| 103 | + data = f.read() |
| 104 | + |
| 105 | + return base64.b64encode(data).decode("ascii") |
| 106 | + |
| 107 | + |
| 108 | +def parse_args(): |
| 109 | + def version_arg_type(arg_value, pat=re.compile(VERSION_REGEX)): |
| 110 | + if not pat.match(arg_value): |
| 111 | + if MASTER_VERSION_REGEX.match(arg_value): |
| 112 | + return arg_value |
| 113 | + |
| 114 | + if not pat.match("v"+arg_value): |
| 115 | + raise argparse.ArgumentTypeError |
| 116 | + |
| 117 | + return "v"+arg_value |
| 118 | + return arg_value |
| 119 | + |
| 120 | + desc = 'Generate CSVs for tagged versions.' |
| 121 | + parser = argparse.ArgumentParser(description=desc) |
| 122 | + parser.add_argument('version', help='Version to generate (SemVer). e.g v1.2.3', type=version_arg_type) |
| 123 | + parser.add_argument('previous_version', help='Previous version.', type=version_arg_type, nargs='?') |
| 124 | + parser.add_argument('--json', dest='yaml', help='Output json config (default).', action='store_false') |
| 125 | + parser.add_argument('--yaml', dest='yaml', help='Output yaml config.', action='store_true') |
| 126 | + parser.add_argument('--upstream', dest='downstream', help='Generate with upstream config.', action='store_false') |
| 127 | + parser.add_argument('--downstream', dest='downstream', help='Generate with downstream config.', action='store_true') |
| 128 | + parser.add_argument('--image', dest='image', help='Image to use in CSV.') |
| 129 | + parser.add_argument('--workdir', dest='workdir', help='Work directory', default=".") |
| 130 | + parser.add_argument('--output-dir', dest='output_dir', help='Output directory relative to the workdir', default="deploy") |
| 131 | + parser.add_argument('--skip-pull', dest='skip_pull', help='Skip pulling the image', action='store_true') |
| 132 | + parser.set_defaults(yaml=True) |
| 133 | + parser.set_defaults(downstream=False) |
| 134 | + parser.set_defaults(previous_version=None) |
| 135 | + parser.set_defaults(skip_pull=False) |
| 136 | + |
| 137 | + logger.debug('Parsing all args') |
| 138 | + _, unknown = parser.parse_known_args() |
| 139 | + |
| 140 | + added_args_keys = set() |
| 141 | + while (len(unknown) > 0 and ARGUMENT_REGEX.match(unknown[0]) and |
| 142 | + ARGUMENT_REGEX.match(unknown[0]).end() == len(unknown[0])): |
| 143 | + logger.info('Adding argument: %s', unknown[0]) |
| 144 | + added_args_keys.add(unknown[0].lstrip('-')) |
| 145 | + parser.add_argument(unknown[0]) |
| 146 | + _, unknown = parser.parse_known_args() |
| 147 | + |
| 148 | + logger.debug('Parsing final set of args') |
| 149 | + return parser.parse_args(), added_args_keys |
| 150 | + |
| 151 | + |
| 152 | +def main(): |
| 153 | + all_args, added_args_keys = parse_args() |
| 154 | + template_kwargs = {key: getattr(all_args, key, None) for key in added_args_keys} |
| 155 | + |
| 156 | + ENV = Environment(loader=FileSystemLoader(os.path.join(all_args.workdir, TEMPLATE_DIR)), undefined=StrictUndefined) |
| 157 | + ENV.filters['normalize_version'] = normalize_version |
| 158 | + |
| 159 | + logo = get_b64_logo_from_file(LOGO_DOWNSTREAM_FILE) if all_args.downstream else get_b64_logo_from_file(os.path.join(all_args.workdir,LOGO_UPSTREAM_FILE)) |
| 160 | + image_ref = all_args.image or CSO_IMAGE + ":" + CSO_IMAGE_TAG |
| 161 | + |
| 162 | + if not all_args.skip_pull: |
| 163 | + repo, image_manifest_digest = get_image_manifest_digest(image_ref) |
| 164 | + if not repo or not image_manifest_digest: |
| 165 | + sys.exit(1) |
| 166 | + |
| 167 | + container_image = repo + "@" + image_manifest_digest |
| 168 | + else: |
| 169 | + container_image = image_ref |
| 170 | + |
| 171 | + template_kwargs["version"] = all_args.version |
| 172 | + template_kwargs["previous_version"] = all_args.previous_version |
| 173 | + template_kwargs["logo"] = logo |
| 174 | + template_kwargs["container_image"] = container_image |
| 175 | + template_kwargs["k8s_api_version"] = template_kwargs.setdefault("k8s_api_version", K8S_API_VERSION) |
| 176 | + |
| 177 | + manifest_output_dir = os.path.join(all_args.output_dir, OUTPUT_MANIFEST_DIR, normalize_version(all_args.version)) |
| 178 | + os.makedirs(manifest_output_dir, exist_ok=True) |
| 179 | + generated_files = {} |
| 180 | + |
| 181 | + assert CSV_TEMPLATE_FILE.endswith(".clusterserviceversion.yaml.jnj") |
| 182 | + csv_template = ENV.get_template(CSV_TEMPLATE_FILE) |
| 183 | + generated_csv = csv_template.render(**template_kwargs) |
| 184 | + csv_filename = CSV_TEMPLATE_FILE.split(".") |
| 185 | + csv_filename.insert(1, all_args.version) |
| 186 | + csv_filename = ".".join(csv_filename[:-1]) |
| 187 | + generated_files[os.path.join(manifest_output_dir, csv_filename)] = generated_csv |
| 188 | + |
| 189 | + for crd_template_file in filter(lambda filename: filename.endswith(".crd.yaml.jnj"), CRD_TEMPLATE_FILES): |
| 190 | + crd_template = ENV.get_template(crd_template_file) |
| 191 | + generated_crd = crd_template.render(**template_kwargs) |
| 192 | + generated_files[os.path.join(manifest_output_dir, crd_template_file.rstrip(".jnj"))] = generated_crd |
| 193 | + |
| 194 | + if all_args.yaml: |
| 195 | + for filepath, content in generated_files.items(): |
| 196 | + with open(filepath, 'w') as f: |
| 197 | + f.write(content) |
| 198 | + else: |
| 199 | + for filepath, content in generated_files.items(): |
| 200 | + parsed = yaml.load(content, Loader=yaml.SafeLoader) |
| 201 | + with open(filepath.rstrip("yaml")+"json", 'w') as f: |
| 202 | + f.write(json.dumps(parsed, default=str)) |
| 203 | + |
| 204 | + |
| 205 | +if __name__ == "__main__": |
| 206 | + main() |
0 commit comments