From 41486f3c8e3de0cf5c33c1bf23cda2bbbf193c4d Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 29 Dec 2022 16:52:08 +0000 Subject: [PATCH 1/2] Add dind/pink CI test.yml scenario (requires registry) --- .github/workflows/test.yml | 58 ++++++++++++++++++- .../binderhub-chart+dind.yaml | 21 +++++++ .../binderhub-chart+pink.yaml | 22 +++++++ .../cm-insecure-registries-dind.yaml | 11 ++++ .../cm-insecure-registries-pink.yaml | 12 ++++ 5 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml create mode 100644 testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml create mode 100644 testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml create mode 100644 testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 835448050..2c2c5ba0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,8 @@ jobs: - main - auth - helm + test-variation: + - "" include: # Chart.yaml contains the chart's oldest supported k8s version, we # test against that and the oldest known supported helm cli version @@ -70,19 +72,56 @@ jobs: # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json upgrade-from: dev upgrade-from-extra-args: "" + - k3s-channel: v1.25 + test: helm + test-variation: dind + local-chart-extra-args: >- + --values testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml + --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ + --set registry.url=http://$REGISTRY_HOST + - k3s-channel: v1.25 + test: helm + test-variation: pink + local-chart-extra-args: >- + --values testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml + --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ + --set registry.url=http://$REGISTRY_HOST + + services: + registry: + image: docker.io/library/registry:latest + ports: + - 5000:5000 + steps: - uses: actions/checkout@v3 with: # chartpress requires the full history fetch-depth: 0 + - name: Set registry host + if: matrix.test-variation == 'dind' || matrix.test-variation == 'pink' + run: | + REGISTRY_HOST=$(hostname -I | awk '{print $1}'):5000 + echo REGISTRY_HOST="$REGISTRY_HOST" >> $GITHUB_ENV + + # Allow k3s to pull from private registry + # https://docs.k3s.io/installation/private-registry + sudo mkdir -p /etc/rancher/k3s/ + cat << EOF | sudo tee /etc/rancher/k3s/registries.yaml + mirrors: + "$REGISTRY_HOST": + endpoint: + - "http://$REGISTRY_HOST" + EOF + - uses: jupyterhub/action-k3s-helm@v3 with: k3s-channel: ${{ matrix.k3s-channel }} helm-version: ${{ matrix.helm-version }} metrics-enabled: false traefik-enabled: false - docker-enabled: true + docker-enabled: ${{ matrix.test-variation != 'dind' && matrix.test-variation != 'pink' }} - name: Setup OS level dependencies run: | @@ -145,9 +184,17 @@ jobs: run: | export DOCKER_BUILDKIT=1 + CHARTPRESS_ARGS= + if [ "${{ matrix.test-variation }}" = "dind" -o "${{ matrix.test-variation }}" = "pink" ]; then + CHARTPRESS_ARGS="--image-prefix localhost:5000/binderhub- --push" + + # Allow the pods to push to the non-https GitHub workflow registry + envsubst < testing/k8s-binder-k8s-hub/cm-insecure-registries-${{ matrix.test-variation }}.yaml | kubectl apply -f - + fi + # Use chartpress to create the helm chart and build its images helm dependency update ./helm-chart/binderhub - (cd helm-chart && chartpress) + (cd helm-chart && chartpress $CHARTPRESS_ARGS) git --no-pager diff --color=always - name: Generate values.schema.json from schema.yaml @@ -286,7 +333,12 @@ jobs: uses: jupyterhub/action-k8s-namespace-report@v1 if: always() with: - important-workloads: deploy/binder deploy/hub deploy/proxy + important-workloads: > + deploy/binder + deploy/hub + deploy/proxy + daemonset/binderhub-test-dind + daemonset/binderhub-test-pink # GitHub action reference: https://github.com/codecov/codecov-action - name: Upload coverage stats diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml new file mode 100644 index 000000000..f7bb9c039 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml @@ -0,0 +1,21 @@ +# Additional configuration for testing dind +# You must create configmap/insecure-registries-dind first to allow testing with an +# insecure http registry +# https://docs.docker.com/registry/insecure/ + +config: + BinderHub: + use_registry: true + +imageBuilderType: dind + +dind: + daemonset: + extraVolumeMounts: + - name: insecure-registries-dind + mountPath: /etc/docker/daemon.json + subPath: daemon.json + extraVolumes: + - name: insecure-registries-dind + configMap: + name: insecure-registries-dind diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml new file mode 100644 index 000000000..3ac0cf9c5 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml @@ -0,0 +1,22 @@ +# Additional configuration for testing podman +# You must create configmap/insecure-registries-pink first to allow testing with an +# insecure http registry + +config: + BinderHub: + use_registry: true + +imageBuilderType: pink + +pink: + daemonset: + extraArgs: + - --log-level=debug + extraVolumeMounts: + - name: insecure-registries-pink + mountPath: /etc/containers/registries.conf.d/100-insecure-registries-pink.conf + subPath: 100-insecure-registries-pink.conf + extraVolumes: + - name: insecure-registries-pink + configMap: + name: insecure-registries-pink diff --git a/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml b/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml new file mode 100644 index 000000000..bb147fd92 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml @@ -0,0 +1,11 @@ +# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-dind.yaml | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: insecure-registries-dind +data: + daemon.json: | + { + "insecure-registries": ["$REGISTRY_HOST"], + "debug": true + } diff --git a/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml b/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml new file mode 100644 index 000000000..14d5a54e4 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml @@ -0,0 +1,12 @@ +# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-pink.yaml | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: insecure-registries-pink +data: + 100-insecure-registries-pink.conf: | + # https://www.redhat.com/sysadmin/manage-container-registries + unqualified-search-registries = ["docker.io"] + [[registry]] + location="$REGISTRY_HOST" + insecure=true From ff061cbba59a9457a32495fe82d2008d650ae8eb Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 29 Dec 2022 17:55:12 +0000 Subject: [PATCH 2/2] Reduce the number of CI matrix tests --- .github/workflows/test.yml | 17 +++++++---------- .../binderhub-chart-config.yaml | 3 ++- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c2c5ba0e..6f245d540 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,6 @@ jobs: test: - main - auth - - helm test-variation: - "" include: @@ -66,26 +65,24 @@ jobs: - k3s-channel: v1.20 helm-version: v3.5.0 test: helm - - k3s-channel: latest - test: helm - test-variation: upgrade - # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json - upgrade-from: dev - upgrade-from-extra-args: "" - - k3s-channel: v1.25 - test: helm test-variation: dind local-chart-extra-args: >- --values testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ --set registry.url=http://$REGISTRY_HOST - - k3s-channel: v1.25 + - k3s-channel: latest test: helm test-variation: pink local-chart-extra-args: >- --values testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ --set registry.url=http://$REGISTRY_HOST + - k3s-channel: latest + test: helm + test-variation: upgrade + # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json + upgrade-from: dev + upgrade-from-extra-args: "" services: registry: diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml index 17f398182..9b3d470c6 100644 --- a/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml +++ b/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml @@ -38,7 +38,8 @@ ingress: # against which we haven't. We currently only test this through the use of # lint-and-validate-values.yaml and setting this value explicitly to make sure # our rendered templates are valid against a k8s api-server. -imageBuilderType: "host" +# This is already the default +# imageBuilderType: "host" # NOTE: This is a mirror of the jupyterhub section in # jupyterhub-chart-config.yaml in testing/local-binder-k8s-hub, keep these