From 91c0dca3c98edaee2dd04603e18984acbde53df4 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 19 Dec 2023 17:03:53 +0000 Subject: [PATCH 1/8] Add workflow for checking Octavia cert expiry --- roles/github/defaults/main.yml | 12 ++++ ...e-octavia-certificates-check-expiry.yml.j2 | 72 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index a6da44e..1842d9b 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -31,6 +31,8 @@ github_buildx_enable_provenance: false github_timeout: 360 +github_octavia_certificates_expiry_time: 30 + github_tempest_test_suites: | - default - tempest-full @@ -74,6 +76,7 @@ github_workflows: - "{{ github_run_infra_vm_host_package_update }}" - "{{ github_run_infra_vm_provision }}" - "{{ github_run_infra_vm_service_deploy }}" + - "{{ github_run_kolla_ansible_octavia_certificates }}" - "{{ github_run_network_connectivity_check }}" - "{{ github_run_overcloud_container_image_pull }}" - "{{ github_run_overcloud_database_backup }}" @@ -131,6 +134,15 @@ github_run_infra_vm_service_deploy: arguments: "{{ github_kayobe_task_arguments }}" concurrency_group: infra +github_run_kolla_ansible_octavia_certificates: + file_name: run-kolla-ansible-octavia-certificates-check-expiry.yml + use_bespoke: true + expiry_time: "{{ github_octavia_certificates_expiry_time }}" + trigger: + workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" + arguments: "{{ github_kayobe_task_arguments }}" + concurrency_group: overcloud + github_run_network_connectivity_check: file_name: run-network-connectivity-check.yml trigger: diff --git a/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 b/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 new file mode 100644 index 0000000..742ee3f --- /dev/null +++ b/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 @@ -0,0 +1,72 @@ +<%- if github_environment_selector == 'input' -%> +<%- set github_runs_on = github_runs_on + ['${{ inputs.kayobe_environment }}'] -%> +<%- set _ = workflow.update({"concurrency_group": "format('{0}-{1}', " + workflow.concurrency_group + ", '${{ inputs.kayobe_environment }}')" }) -%> +<%- set _ = github_default_registry.update({"url": "${{ vars[format('{0}_REGISTRY_URL', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"username": "${{ vars[format('{0}_REGISTRY_USERNAME', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_registry.update({"password": "${{ secrets[format('{0}_REGISTRY_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_AUTOMATION_SSH_PRIVATE_KEY": "${{ secrets[format('{0}_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_default_kayobe_arguments.update({"KAYOBE_VAULT_PASSWORD": "${{ secrets[format('{0}_KAYOBE_VAULT_PASSWORD', inputs.kayobe_environment)] }}" }) -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": '${{ inputs.kayobe_environment }}'}) -%> +<%- endif -%> +<%- if github_environment_selector == 'single' -%> +<%- set _ = github_kayobe_arguments.update({"KAYOBE_ENVIRONMENT": github_kayobe_environments | first}) -%> +<%- endif -%> +%% lookup('template', 'header.yml.j2') %% +jobs: + prepare-runner: + uses: ./.github/workflows/prepare-runner.yml + %% format_file_name(workflow.file_name) %%: + runs-on: %% github_runs_on %% + permissions: + contents: read + packages: %% 'read' if (github_registry.url | default(github_default_registry.url)) == 'ghcr.io' else 'none' %% + pull-requests: none + container: + image: %% github_registry.url | default(github_default_registry.url) %%/%% github_image_name %%:%% github_image_tag %% + credentials: + username: %% github_registry.username | default(github_default_registry.username) %% + password: %% github_registry.password | default(github_default_registry.password) %% + concurrency: + group: %% workflow.concurrency_group %% + cancel-in-progress: false + timeout-minutes: %% github_timeout %% + needs: prepare-runner + steps: +<% if github_checkout_hook | length >= 1 %> + %% github_checkout_hook | indent(width=6, first=false) %% +<% endif %> + - name: Checkout kayobe config + uses: actions/checkout@v3 + with: + submodules: true + path: kayobe-config + +<% if github_kayobe_hook | length >= 1 %> + %% github_kayobe_hook | indent(width=6, first=false) %% +<% endif %> + - name: Symlink source checkout to expected location + run: sudo ln -s $PWD/kayobe-config /src + + - name: Generate Kolla Ansible configuration + run: | + /src/.automation/pipeline/overcloud-service-configuration-generate.sh "/tmp/ignore" + env: +<% if github_environment_selector is not none %> + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' +<% endif %> + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' + KOLLA_TAGS: none + + - name: Check Octavia certificates expiry + run: | + /src/.automation/pipeline/kolla-ansible-run.sh "octavia-certificates --check-expiry %% workflow.expiry_time %%" + env: +<% if github_environment_selector is not none %> + KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' +<% endif %> + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: "%% github_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | default(github_default_kayobe_arguments.KAYOBE_AUTOMATION_SSH_PRIVATE_KEY) %%" + KAYOBE_VAULT_PASSWORD: "%% github_kayobe_arguments.KAYOBE_VAULT_PASSWORD | default(github_default_kayobe_arguments.KAYOBE_VAULT_PASSWORD) %%" + HOME: '%% github_kayobe_arguments.HOME | default(github_default_kayobe_arguments.HOME) %%' +<% if github_final_hook | length >= 1 +%> + %% github_final_hook | indent(width=6, first=false) -%% +<% endif %> From 322a01bebb9dd0b1c504339c41252a41a34532ed Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 20 Dec 2023 11:35:55 +0000 Subject: [PATCH 2/8] Set expiry time check from user input --- roles/github/defaults/main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 1842d9b..bac9a63 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -68,6 +68,17 @@ github_kayobe_environment_input: | options: {{ github_kayobe_environments }} {%- endif -%} +github_octavia_certificates_expiry_time_input: | + {%- if github_environment_selector == 'input' -%} + expiry_time: + description: | + Check if the Octavia Amphora certificates will expire within this + number of days (integer). + type: string + required: true + default: '{{ github_octavia_certificates_expiry_time }}' + {%- endif -%} + github_workflows: - "{{ github_prepare_runner }}" - "{{ github_build_kayobe_image }}" @@ -139,8 +150,7 @@ github_run_kolla_ansible_octavia_certificates: use_bespoke: true expiry_time: "{{ github_octavia_certificates_expiry_time }}" trigger: - workflow_dispatch: "{{ github_kayobe_dispatch_inputs }}" - arguments: "{{ github_kayobe_task_arguments }}" + workflow_dispatch: "{{ github_octavia_certificates_expiry_time_input }}" concurrency_group: overcloud github_run_network_connectivity_check: From 988463f5f8e5ed0b1bcec4bd0d52af0c4995d818 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 20 Dec 2023 13:58:03 +0000 Subject: [PATCH 3/8] Also include support for cron scheduling --- roles/github/README.md | 2 ++ roles/github/defaults/main.yml | 21 ++++++++++++------- ...e-octavia-certificates-check-expiry.yml.j2 | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 90d9292..9a8e217 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -71,6 +71,8 @@ github_buildx_inline_config: | `github_tempest_test_suites`: provide a list of load lists to be made available within the drop-down list for running tempest. Defaults to `default` and `tempest-full`. +If you are using the workflow `github_run_kolla_ansible_octavia_certificates` to check for the expiry of Octavia Amphora certificates, by default it will allow for manual execution via workflow_dispatch. You can change this to run automatically on a schedule by setting `github_octavia_certificates_schedule` to a cron schedule. If you want to change the number of days checked, set the variable `github_default_octavia_certificates_expiry_time`. + If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` `github_workflows:` is a list of dictionaries that contains each of the workflows described above. A given list element is made up of the following: diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index bac9a63..19c2ddf 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -31,7 +31,9 @@ github_buildx_enable_provenance: false github_timeout: 360 -github_octavia_certificates_expiry_time: 30 +github_default_octavia_certificates_expiry_time: 30 + +github_octavia_certificates_schedule: github_tempest_test_suites: | - default @@ -68,7 +70,7 @@ github_kayobe_environment_input: | options: {{ github_kayobe_environments }} {%- endif -%} -github_octavia_certificates_expiry_time_input: | +github_octavia_certificates_input: | {%- if github_environment_selector == 'input' -%} expiry_time: description: | @@ -76,7 +78,7 @@ github_octavia_certificates_expiry_time_input: | number of days (integer). type: string required: true - default: '{{ github_octavia_certificates_expiry_time }}' + default: '{{ github_default_octavia_certificates_expiry_time }}' {%- endif -%} github_workflows: @@ -87,7 +89,6 @@ github_workflows: - "{{ github_run_infra_vm_host_package_update }}" - "{{ github_run_infra_vm_provision }}" - "{{ github_run_infra_vm_service_deploy }}" - - "{{ github_run_kolla_ansible_octavia_certificates }}" - "{{ github_run_network_connectivity_check }}" - "{{ github_run_overcloud_container_image_pull }}" - "{{ github_run_overcloud_database_backup }}" @@ -145,12 +146,18 @@ github_run_infra_vm_service_deploy: arguments: "{{ github_kayobe_task_arguments }}" concurrency_group: infra +github_octavia_certificates_schedule_trigger: + schedule: + cron: "{{ github_octavia_certificates_schedule }}" + +github_octavia_certificates_workflow_dispatch_trigger: + workflow_dispatch: "{{ github_octavia_certificates_input }}" + github_run_kolla_ansible_octavia_certificates: file_name: run-kolla-ansible-octavia-certificates-check-expiry.yml use_bespoke: true - expiry_time: "{{ github_octavia_certificates_expiry_time }}" - trigger: - workflow_dispatch: "{{ github_octavia_certificates_expiry_time_input }}" + expiry_time: "{{ github_default_octavia_certificates_expiry_time }}" + trigger: "{{ github_octavia_certificates_schedule_trigger if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }}" concurrency_group: overcloud github_run_network_connectivity_check: diff --git a/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 b/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 index 742ee3f..7f9fddf 100644 --- a/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 +++ b/roles/github/templates/run-kolla-ansible-octavia-certificates-check-expiry.yml.j2 @@ -59,7 +59,7 @@ jobs: - name: Check Octavia certificates expiry run: | - /src/.automation/pipeline/kolla-ansible-run.sh "octavia-certificates --check-expiry %% workflow.expiry_time %%" + /src/.automation/pipeline/kolla-ansible-run.sh "octavia-certificates --check-expiry <%- if github_octavia_certificates_schedule %> %% workflow.expiry_time %% <%- else %> ${{ inputs.expiry_time }} <%- endif -%>" env: <% if github_environment_selector is not none %> KAYOBE_ENVIRONMENT: '%% github_kayobe_arguments.KAYOBE_ENVIRONMENT %%' From fd163a8f25cfea695b498157bcaf40a59680330d Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 16 Apr 2024 14:00:58 +0100 Subject: [PATCH 4/8] Linting fix --- roles/github/defaults/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 19c2ddf..f0e3597 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -157,7 +157,10 @@ github_run_kolla_ansible_octavia_certificates: file_name: run-kolla-ansible-octavia-certificates-check-expiry.yml use_bespoke: true expiry_time: "{{ github_default_octavia_certificates_expiry_time }}" - trigger: "{{ github_octavia_certificates_schedule_trigger if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }}" + trigger: >- + {{ github_octavia_certificates_schedule_trigger if + github_octavia_certificates_schedule else + github_octavia_certificates_workflow_dispatch_trigger }} concurrency_group: overcloud github_run_network_connectivity_check: From 134e9c3db4fc914cbf885dd31292aacabb76853d Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 13 May 2024 16:59:18 +0100 Subject: [PATCH 5/8] default+extra github_workflows --- roles/github/defaults/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index f0e3597..96d8613 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -79,9 +79,10 @@ github_octavia_certificates_input: | type: string required: true default: '{{ github_default_octavia_certificates_expiry_time }}' - {%- endif -%} -github_workflows: +github_workflows: "{{ github_workflows_default + github_workflows_extra }}" + +github_workflows_default: - "{{ github_prepare_runner }}" - "{{ github_build_kayobe_image }}" - "{{ github_run_kolla_config_diff }}" @@ -106,6 +107,8 @@ github_workflows: - "{{ github_run_seed_vm_provision }}" - "{{ github_run_tempest }}" +github_workflows_extra: [] + github_prepare_runner: file_name: prepare-runner.yml use_bespoke: true From f7562cdeb51532316c318669a43e6cbf0cccda61 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 13 May 2024 16:59:39 +0100 Subject: [PATCH 6/8] Allow dispatch when using scheudled --- roles/github/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 96d8613..29d5155 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -71,7 +71,6 @@ github_kayobe_environment_input: | {%- endif -%} github_octavia_certificates_input: | - {%- if github_environment_selector == 'input' -%} expiry_time: description: | Check if the Octavia Amphora certificates will expire within this @@ -161,7 +160,7 @@ github_run_kolla_ansible_octavia_certificates: use_bespoke: true expiry_time: "{{ github_default_octavia_certificates_expiry_time }}" trigger: >- - {{ github_octavia_certificates_schedule_trigger if + {{ [github_octavia_certificates_schedule_trigger, github_octavia_certificates_workflow_dispatch_trigger] if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }} concurrency_group: overcloud From 456c5c279348d1303bec48487abd55a065b9c3ad Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 23 May 2024 10:08:11 +0100 Subject: [PATCH 7/8] Correct jinja templating + update docs --- roles/github/README.md | 13 +++++++++++-- roles/github/defaults/main.yml | 5 +---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/roles/github/README.md b/roles/github/README.md index 9a8e217..eba1069 100644 --- a/roles/github/README.md +++ b/roles/github/README.md @@ -71,7 +71,7 @@ github_buildx_inline_config: | `github_tempest_test_suites`: provide a list of load lists to be made available within the drop-down list for running tempest. Defaults to `default` and `tempest-full`. -If you are using the workflow `github_run_kolla_ansible_octavia_certificates` to check for the expiry of Octavia Amphora certificates, by default it will allow for manual execution via workflow_dispatch. You can change this to run automatically on a schedule by setting `github_octavia_certificates_schedule` to a cron schedule. If you want to change the number of days checked, set the variable `github_default_octavia_certificates_expiry_time`. +If you are using the workflow `github_run_kolla_ansible_octavia_certificates` to check for the expiry of Octavia Amphora certificates, by default it will allow for manual execution via workflow_dispatch. You can also have this run automatically on a schedule by setting `github_octavia_certificates_schedule` to a cron schedule. If you want to change the number of days checked, set the variable `github_default_octavia_certificates_expiry_time`. If you wish to make more impactful changes such as which workflows are built and what they contain then see the list of dictionaries called `workflows` in `defaults/main.yml` @@ -89,7 +89,16 @@ If you wish to make more impactful changes such as which workflows are built and - `use_bespoke`: As mentioned in `path_override` some workflows require a more specific template to meet their needs. Setting this to true would create a workflow from a bespoke template if one exists within the roles `templates` directory. Note this is intended for **internal use only**. -The following will override `github_workflows` to ensure only `Build Kayobe Image` and `Run Kolla Config Diff` is generated. +By default, most workflows are built as they are part of the list `github_workflows_default`. If you want to add additional workflows, you can add them to the list `github_workflows_extra`. + +For example, you can set the following to additionally build `Run Kolla Ansible Octavia Certificates Check Expiry`. + +```yaml +github_workflows_extra: + - "{{ github_run_kolla_ansible_octavia_certificates }}" +``` + +If you don't want to build all workflows, the following will override `github_workflows` to ensure only `Build Kayobe Image` and `Run Kolla Config Diff` are generated. ```yaml github_workflows: diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index 29d5155..a4c4872 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -159,10 +159,7 @@ github_run_kolla_ansible_octavia_certificates: file_name: run-kolla-ansible-octavia-certificates-check-expiry.yml use_bespoke: true expiry_time: "{{ github_default_octavia_certificates_expiry_time }}" - trigger: >- - {{ [github_octavia_certificates_schedule_trigger, github_octavia_certificates_workflow_dispatch_trigger] if - github_octavia_certificates_schedule else - github_octavia_certificates_workflow_dispatch_trigger }} + trigger: "{{ github_octavia_certificates_workflow_dispatch_trigger | combine(github_octavia_certificates_schedule_trigger) if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }}" concurrency_group: overcloud github_run_network_connectivity_check: From 2b4d3de909e5bb05d66d5a3d2ec50dcbbdb9bd59 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 24 May 2024 09:18:06 +0100 Subject: [PATCH 8/8] Linter fix (again) --- roles/github/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/github/defaults/main.yml b/roles/github/defaults/main.yml index a4c4872..efbd1f0 100644 --- a/roles/github/defaults/main.yml +++ b/roles/github/defaults/main.yml @@ -159,7 +159,9 @@ github_run_kolla_ansible_octavia_certificates: file_name: run-kolla-ansible-octavia-certificates-check-expiry.yml use_bespoke: true expiry_time: "{{ github_default_octavia_certificates_expiry_time }}" - trigger: "{{ github_octavia_certificates_workflow_dispatch_trigger | combine(github_octavia_certificates_schedule_trigger) if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }}" + trigger: >- + {{ github_octavia_certificates_workflow_dispatch_trigger | combine(github_octavia_certificates_schedule_trigger) + if github_octavia_certificates_schedule else github_octavia_certificates_workflow_dispatch_trigger }} concurrency_group: overcloud github_run_network_connectivity_check: