diff --git a/config/staging.yaml b/config/staging.yaml index 065ea6e44..f39211fe2 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -7,21 +7,29 @@ binderhub: hub_url: https://hub.gke2.staging.mybinder.org badge_base_url: https://staging.mybinder.org image_prefix: gcr.io/binderhub-288415/r2d-staging-g5b5b759- + # image_prefix: us-central1-docker.pkg.dev/binderhub-288415/staging/r2d-2023-04- sticky_builds: true build_memory_limit: "2G" + # DockerRegistry: + # token_url: "https://us-central1-docker.pkg.dev/v2/token" + + # registry: + # url: "https://us-central1-docker.pkg.dev" + # username: "_json_key" + extraEnv: EVENT_LOG_NAME: "binderhub-staging-events-text" resources: requests: - cpu: 0.25 + cpu: 0.1 memory: 512Mi hpa: enabled: true maxReplicas: 3 minReplicas: 1 - targetCPU: 50 # 50% of cpu request, so 0.125 CPU + targetCPU: 90 # 90% of cpu request, so 90m CPU dind: resources: @@ -30,7 +38,7 @@ binderhub: memory: 1Gi limits: cpu: "1" - memory: 2.5Gi + memory: 1Gi ingress: hosts: @@ -41,14 +49,14 @@ binderhub: hub: resources: requests: - memory: 512Mi - cpu: null + memory: 100M + cpu: "10m" singleuser: memory: - guarantee: 256M + guarantee: 100M limit: 256M cpu: - guarantee: 0.1 + guarantee: .01 limit: 0.5 ingress: hosts: @@ -61,7 +69,20 @@ binderhub: - hub.gke2.staging.mybinder.org scheduling: userPlaceholder: - replicas: 2 + replicas: 1 + + proxy: + chp: + resources: + requests: + memory: 100M + cpu: 10m + +minesweeper: + resources: + requests: + cpu: 5m + memory: 100Mi grafana: ingress: @@ -101,8 +122,16 @@ prometheus: ingress-nginx: controller: + replicaCount: 2 service: loadBalancerIP: 35.222.35.25 + resources: + requests: + cpu: 10m + memory: 100Mi + limits: + cpu: 500m + memory: 500Mi static: ingress: @@ -137,7 +166,9 @@ analyticsPublisher: events: logName: binderhub-staging-events-text sourceBucket: binder-staging-events-raw-export - + resources: + requests: + cpu: 10m # TODO: copy to values cloudCosts: sourceBucket: binder-billing-archive @@ -153,6 +184,13 @@ gcsProxy: federationRedirect: host: staging.mybinder.org enabled: true + resources: + requests: + cpu: 10m + memory: 100Mi + limits: + cpu: 0.2 + memory: 200Mi hosts: gke: url: https://gke2.staging.mybinder.org diff --git a/mybinder/templates/analytics-publisher/deployment.yaml b/mybinder/templates/analytics-publisher/deployment.yaml index a7228ec33..5aa7c2b27 100644 --- a/mybinder/templates/analytics-publisher/deployment.yaml +++ b/mybinder/templates/analytics-publisher/deployment.yaml @@ -50,12 +50,6 @@ spec: - name: config mountPath: /etc/analytics-publisher readOnly: true - resources: - requests: - cpu: 0.2 - memory: 200Mi - limits: - cpu: 0.2 - memory: 300Mi + resources: {{ .Values.analyticsPublisher.resources | toJson }} {{- end }} diff --git a/mybinder/templates/federation-redirect/deployment.yaml b/mybinder/templates/federation-redirect/deployment.yaml index 677d31a42..27bc08e17 100644 --- a/mybinder/templates/federation-redirect/deployment.yaml +++ b/mybinder/templates/federation-redirect/deployment.yaml @@ -54,13 +54,10 @@ spec: - name: config mountPath: /etc/federation-redirect readOnly: true + {{- with .Values.federationRedirect.resources }} resources: - requests: - cpu: 0.2 - memory: 200Mi - limits: - cpu: 0.2 - memory: 300Mi + {{- . | toYaml | nindent 10 }} + {{- end }} tolerations: - key: "node.kubernetes.io/unschedulable" operator: "Exists" diff --git a/mybinder/templates/matomo/deployment.yaml b/mybinder/templates/matomo/deployment.yaml index 3523efd38..86f212bcb 100644 --- a/mybinder/templates/matomo/deployment.yaml +++ b/mybinder/templates/matomo/deployment.yaml @@ -59,7 +59,10 @@ spec: volumeMounts: - name: matomo-config-rw mountPath: /var/www/html/config/ - resources: {{ toJson .Values.matomo.resources }} + {{- with .Values.matomo.resources }} + resources: + {{- . | toYaml | nindent 10 }} + {{- end }} - name: nginx image: nginx:1.15 ports: diff --git a/mybinder/templates/minesweeper/daemonset.yaml b/mybinder/templates/minesweeper/daemonset.yaml index d865c90aa..aeff536a9 100644 --- a/mybinder/templates/minesweeper/daemonset.yaml +++ b/mybinder/templates/minesweeper/daemonset.yaml @@ -57,7 +57,8 @@ spec: add: - KILL {{- with .Values.minesweeper.resources }} - resources: {{ toJson . }} + resources: + {{- . | toYaml | nindent 10 }} {{- end }} volumeMounts: - name: config diff --git a/mybinder/values.yaml b/mybinder/values.yaml index b2f584651..e9979f81a 100644 --- a/mybinder/values.yaml +++ b/mybinder/values.yaml @@ -542,14 +542,28 @@ analyticsPublisher: fileName: cloud-costs.jsonl kind: csv nodeSelector: {} + resources: + requests: + cpu: 0.2 + memory: 200Mi + limits: + cpu: 0.2 + memory: 300Mi # this is defined in secrets/ for the OVH cluster eventsArchiver: serviceAccountKey: "" federationRedirect: - host: mybinder.org enabled: false + host: mybinder.org + resources: + requests: + cpu: 0.2 + memory: 200Mi + limits: + cpu: 0.2 + memory: 300Mi image: name: jupyterhub/mybinder.org-federation-redirect tag: "set-by-chartpress" diff --git a/terraform/gcp/modules/mybinder/resource.tf b/terraform/gcp/modules/mybinder/resource.tf index 1b165e429..8e780cf41 100644 --- a/terraform/gcp/modules/mybinder/resource.tf +++ b/terraform/gcp/modules/mybinder/resource.tf @@ -16,7 +16,7 @@ locals { }, binderhub-builder = { display_name = "Storage access for ${var.name} image builder", - role = "roles/storage.admin", + role = var.use_artifact_registry ? "roles/artifactregistry.createOnPushWriter" : "roles/storage.admin", }, } # add -staging to events prefix, but don't include 'prod' in prod events @@ -25,6 +25,13 @@ locals { events_log_prefix = var.name == "prod" ? "binderhub" : "binderhub-${var.name}" } +resource "google_artifact_registry_repository" "repo" { + location = var.registry_location != null ? var.registry_location : data.google_client_config.provider.region + repository_id = var.name + description = "${var.name} container registry" + format = "DOCKER" +} + resource "google_container_cluster" "cluster" { name = var.name location = var.gke_location != null ? var.gke_location : data.google_client_config.provider.zone diff --git a/terraform/gcp/modules/mybinder/variables.tf b/terraform/gcp/modules/mybinder/variables.tf index e6f95b666..26eab5e2e 100644 --- a/terraform/gcp/modules/mybinder/variables.tf +++ b/terraform/gcp/modules/mybinder/variables.tf @@ -14,7 +14,19 @@ variable "gke_master_version" { variable "gke_location" { type = string - description = "GKE location for cluster if different, e.g. us-central1 for regional cluster" + description = "GKE location for cluster if different from provider zone, e.g. us-central1 for regional cluster" + default = null +} + +variable "use_artifact_registry" { + type = bool + description = "Use artifact registry instead of legacy container registry" + default = false +} + +variable "registry_location" { + type = string + description = "Registry location for cluster if different from provider region, e.g. us for multi-region" default = null } diff --git a/terraform/gcp/staging/main.tf b/terraform/gcp/staging/main.tf index 071862c51..f1285eb2c 100644 --- a/terraform/gcp/staging/main.tf +++ b/terraform/gcp/staging/main.tf @@ -12,31 +12,36 @@ provider "google" { } locals { - gke_version = "1.19.14-gke.1900" + gke_version = "1.24.10-gke.2300" } module "mybinder" { - source = "../modules/mybinder" - name = "staging" - gke_master_version = local.gke_version - federation_members = [] + source = "../modules/mybinder" + name = "staging" + gke_master_version = local.gke_version + use_artifact_registry = false + federation_members = [] } # define node pools here, too hard to encode with variables -resource "google_container_node_pool" "pool" { - name = "pool-2020-09" +resource "google_container_node_pool" "pool-1" { + name = "pool-2023-04" cluster = module.mybinder.cluster_name autoscaling { min_node_count = 1 - max_node_count = 4 + max_node_count = 3 } version = local.gke_version node_config { - machine_type = "n1-standard-4" - disk_size_gb = 500 + # e2-medium is 2cpu, 8GB shared-core + # only 1 CPU allocatable, though, and k8s itself needs most of that + # e2-standard-2 is 2x as expensive + # but 2 e2-standard-2 is $100/month + machine_type = "e2-standard-2" + disk_size_gb = 100 disk_type = "pd-standard" # https://www.terraform.io/docs/providers/google/r/container_cluster.html#oauth_scopes-1 oauth_scopes = [ @@ -65,7 +70,7 @@ resource "google_container_node_pool" "pool" { output "private_keys" { value = module.mybinder.private_keys - description = "GCP serice account keys" + description = "GCP service account keys" sensitive = true }