diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 835448050..6f245d540 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,8 @@ jobs: test: - main - auth - - helm + test-variation: + - "" include: # Chart.yaml contains the chart's oldest supported k8s version, we # test against that and the oldest known supported helm cli version @@ -64,25 +65,60 @@ jobs: - k3s-channel: v1.20 helm-version: v3.5.0 test: helm + test-variation: dind + local-chart-extra-args: >- + --values testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml + --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ + --set registry.url=http://$REGISTRY_HOST + - k3s-channel: latest + test: helm + test-variation: pink + local-chart-extra-args: >- + --values testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml + --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ + --set registry.url=http://$REGISTRY_HOST - k3s-channel: latest test: helm test-variation: upgrade # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json upgrade-from: dev upgrade-from-extra-args: "" + + services: + registry: + image: docker.io/library/registry:latest + ports: + - 5000:5000 + steps: - uses: actions/checkout@v3 with: # chartpress requires the full history fetch-depth: 0 + - name: Set registry host + if: matrix.test-variation == 'dind' || matrix.test-variation == 'pink' + run: | + REGISTRY_HOST=$(hostname -I | awk '{print $1}'):5000 + echo REGISTRY_HOST="$REGISTRY_HOST" >> $GITHUB_ENV + + # Allow k3s to pull from private registry + # https://docs.k3s.io/installation/private-registry + sudo mkdir -p /etc/rancher/k3s/ + cat << EOF | sudo tee /etc/rancher/k3s/registries.yaml + mirrors: + "$REGISTRY_HOST": + endpoint: + - "http://$REGISTRY_HOST" + EOF + - uses: jupyterhub/action-k3s-helm@v3 with: k3s-channel: ${{ matrix.k3s-channel }} helm-version: ${{ matrix.helm-version }} metrics-enabled: false traefik-enabled: false - docker-enabled: true + docker-enabled: ${{ matrix.test-variation != 'dind' && matrix.test-variation != 'pink' }} - name: Setup OS level dependencies run: | @@ -145,9 +181,17 @@ jobs: run: | export DOCKER_BUILDKIT=1 + CHARTPRESS_ARGS= + if [ "${{ matrix.test-variation }}" = "dind" -o "${{ matrix.test-variation }}" = "pink" ]; then + CHARTPRESS_ARGS="--image-prefix localhost:5000/binderhub- --push" + + # Allow the pods to push to the non-https GitHub workflow registry + envsubst < testing/k8s-binder-k8s-hub/cm-insecure-registries-${{ matrix.test-variation }}.yaml | kubectl apply -f - + fi + # Use chartpress to create the helm chart and build its images helm dependency update ./helm-chart/binderhub - (cd helm-chart && chartpress) + (cd helm-chart && chartpress $CHARTPRESS_ARGS) git --no-pager diff --color=always - name: Generate values.schema.json from schema.yaml @@ -286,7 +330,12 @@ jobs: uses: jupyterhub/action-k8s-namespace-report@v1 if: always() with: - important-workloads: deploy/binder deploy/hub deploy/proxy + important-workloads: > + deploy/binder + deploy/hub + deploy/proxy + daemonset/binderhub-test-dind + daemonset/binderhub-test-pink # GitHub action reference: https://github.com/codecov/codecov-action - name: Upload coverage stats diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml new file mode 100644 index 000000000..f7bb9c039 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml @@ -0,0 +1,21 @@ +# Additional configuration for testing dind +# You must create configmap/insecure-registries-dind first to allow testing with an +# insecure http registry +# https://docs.docker.com/registry/insecure/ + +config: + BinderHub: + use_registry: true + +imageBuilderType: dind + +dind: + daemonset: + extraVolumeMounts: + - name: insecure-registries-dind + mountPath: /etc/docker/daemon.json + subPath: daemon.json + extraVolumes: + - name: insecure-registries-dind + configMap: + name: insecure-registries-dind diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml new file mode 100644 index 000000000..3ac0cf9c5 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml @@ -0,0 +1,22 @@ +# Additional configuration for testing podman +# You must create configmap/insecure-registries-pink first to allow testing with an +# insecure http registry + +config: + BinderHub: + use_registry: true + +imageBuilderType: pink + +pink: + daemonset: + extraArgs: + - --log-level=debug + extraVolumeMounts: + - name: insecure-registries-pink + mountPath: /etc/containers/registries.conf.d/100-insecure-registries-pink.conf + subPath: 100-insecure-registries-pink.conf + extraVolumes: + - name: insecure-registries-pink + configMap: + name: insecure-registries-pink diff --git a/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml b/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml index 17f398182..9b3d470c6 100644 --- a/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml +++ b/testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml @@ -38,7 +38,8 @@ ingress: # against which we haven't. We currently only test this through the use of # lint-and-validate-values.yaml and setting this value explicitly to make sure # our rendered templates are valid against a k8s api-server. -imageBuilderType: "host" +# This is already the default +# imageBuilderType: "host" # NOTE: This is a mirror of the jupyterhub section in # jupyterhub-chart-config.yaml in testing/local-binder-k8s-hub, keep these diff --git a/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml b/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml new file mode 100644 index 000000000..bb147fd92 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml @@ -0,0 +1,11 @@ +# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-dind.yaml | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: insecure-registries-dind +data: + daemon.json: | + { + "insecure-registries": ["$REGISTRY_HOST"], + "debug": true + } diff --git a/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml b/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml new file mode 100644 index 000000000..14d5a54e4 --- /dev/null +++ b/testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml @@ -0,0 +1,12 @@ +# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-pink.yaml | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: insecure-registries-pink +data: + 100-insecure-registries-pink.conf: | + # https://www.redhat.com/sysadmin/manage-container-registries + unqualified-search-registries = ["docker.io"] + [[registry]] + location="$REGISTRY_HOST" + insecure=true