diff --git a/manifests/kustomize/base/cache/kustomization.yaml b/manifests/kustomize/base/cache/kustomization.yaml index 56c40afbe4e..2c2001ed0bd 100644 --- a/manifests/kustomize/base/cache/kustomization.yaml +++ b/manifests/kustomize/base/cache/kustomization.yaml @@ -2,10 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cache-deployment.yaml - - cache-service.yaml - cache-role.yaml - cache-rolebinding.yaml - cache-sa.yaml + - cache-service.yaml commonLabels: app: cache-server images: diff --git a/manifests/kustomize/base/cache/postgresql/cache-deployment.yaml b/manifests/kustomize/base/cache/postgresql/cache-deployment.yaml new file mode 100644 index 00000000000..8e6cc316cee --- /dev/null +++ b/manifests/kustomize/base/cache/postgresql/cache-deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cache-server + labels: + app: cache-server +spec: + replicas: 1 + selector: + matchLabels: + app: cache-server + template: + metadata: + labels: + app: cache-server + spec: + containers: + - name: server + image: gcr.io/ml-pipeline/cache-server:dummy + env: + - name: DEFAULT_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: pgx + - name: DBCONFIG_DB_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheDb + - name: DBCONFIG_HOST_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresDbHost + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresDbPort + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # If you update WEBHOOK_PORT, also change the value of the + # containerPort "webhook-api" to match. + - name: WEBHOOK_PORT + value: "8443" + args: ["--db_driver=$(DBCONFIG_DRIVER)", + "--db_host=$(DBCONFIG_HOST_NAME)", + "--db_port=$(DBCONFIG_PORT)", + "--db_name=$(DBCONFIG_DB_NAME)", + "--db_user=$(DBCONFIG_USER)", + "--db_password=$(DBCONFIG_PASSWORD)", + "--namespace_to_watch=$(NAMESPACE_TO_WATCH)", + "--listen_port=$(WEBHOOK_PORT)", + ] + imagePullPolicy: Always + ports: + - containerPort: 8443 + name: webhook-api + volumeMounts: + - name: webhook-tls-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: webhook-tls-certs + secret: + secretName: webhook-server-tls + serviceAccountName: kubeflow-pipelines-cache diff --git a/manifests/kustomize/base/cache/postgresql/kustomization.yaml b/manifests/kustomize/base/cache/postgresql/kustomization.yaml new file mode 100644 index 00000000000..89adcab9d98 --- /dev/null +++ b/manifests/kustomize/base/cache/postgresql/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cache-deployment.yaml + - ../cache-role.yaml + - ../cache-rolebinding.yaml + - ../cache-sa.yaml + - ../cache-service.yaml +commonLabels: + app: cache-server +images: + - name: gcr.io/ml-pipeline/cache-server + newTag: 2.0.0 diff --git a/manifests/kustomize/base/installs/generic/kustomization.yaml b/manifests/kustomize/base/installs/generic/kustomization.yaml index a3786b20772..4ef5fd2471f 100644 --- a/manifests/kustomize/base/installs/generic/kustomization.yaml +++ b/manifests/kustomize/base/installs/generic/kustomization.yaml @@ -45,4 +45,4 @@ vars: fieldref: fieldpath: data.defaultPipelineRoot configurations: -- params.yaml + - params.yaml diff --git a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml index 47ce6b687d4..cd50fe5dce0 100644 --- a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml +++ b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml @@ -12,8 +12,11 @@ data: namespace: `kubectl rollout restart deployment -n `. appName: pipeline appVersion: 2.0.1 - dbHost: mysql - dbPort: "3306" + dbHost: mysql # relic to be removed after release + dbPort: "3306" # relic to be removed after release + dbType: mysql + mysqlHost: mysql + mysqlPort: "3306" mlmdDb: metadb cacheDb: cachedb pipelineDb: mlpipeline diff --git a/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml b/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml new file mode 100644 index 00000000000..bd435cc7122 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml @@ -0,0 +1,48 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +bases: +- ../../../pipeline/postgres +- ../../../cache/postgres +- ../../../cache-deployer +resources: +- pipeline-install-config.yaml +- postgres-secret.yaml +vars: +- name: kfp-namespace + objref: + kind: Deployment + apiVersion: apps/v1 + name: ml-pipeline + fieldref: + fieldpath: metadata.namespace +- name: kfp-app-name + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.appName +- name: kfp-app-version + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.appVersion +- name: kfp-artifact-bucket-name + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.bucketName +- name: kfp-default-pipeline-root + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.defaultPipelineRoot +configurations: + - params.yaml diff --git a/manifests/kustomize/base/installs/generic/postgres/params.yaml b/manifests/kustomize/base/installs/generic/postgres/params.yaml new file mode 100644 index 00000000000..9b8a7efd529 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/params.yaml @@ -0,0 +1,10 @@ +# Allow Kustomize var to replace following fields. +varReference: +- path: data/config + kind: ConfigMap +- path: data/defaultPipelineRoot + kind: ConfigMap +- path: metadata/name + kind: Application +- path: spec/descriptor/version + kind: Application diff --git a/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml new file mode 100644 index 00000000000..f5d87c67971 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pipeline-install-config +data: + warning: | + 1. Do not use kubectl to edit this configmap, because some values are used + during kustomize build. Instead, change the configmap and apply the entire + kustomize manifests again. + 2. After updating the configmap, some deployments may need to be restarted + until the changes take effect. A quick way to restart all deployments in a + namespace: `kubectl rollout restart deployment -n `. + appName: pipeline + appVersion: 2.0.0 + dbHost: postgres # relic to be removed after release + dbPort: "5432" # relic to be removed after release + dbType: postgres + postgresHost: postgres + postgresPort: "5432" + mlmdDb: metadb + cacheDb: cachedb + pipelineDb: mlpipeline + bucketName: mlpipeline + ## defaultPipelineRoot: Optional. Default pipeline root in v2 compatible mode. + ## https://www.kubeflow.org/docs/components/pipelines/sdk/v2/v2-compatibility/ + ## + ## If the field is not set, kfp-launcher configmaps won't be created and + ## v2 compatible mode defaults to minio://mlpipeline/v2/artifacts as pipeline + ## root. + ## + ## When not in Kubeflow Pipelines multi-user mode, the config works as you + ## would normally expect. + ## + ## In Kubeflow Pipelines multi-user mode, the config creates default + ## kfp-launcher configmaps in each user's namespace. Users can edit the + ## kfp-launcher configmap's defaultPipelineRoot field afterwards to configure + ## namespace-specific default pipeline root. The namespace specific changes in + ## kfp-launcher configmap won't be overridden by pipeline-install-config. + ## + ## Caveat: when you update the config from a non-empty value, only new + ## namespaces get the updated config by default. Owners of existing namespaces + ## must delete the kfp-launcher configmap to get the new default config value. + ## + ## Examples: + ## defaultPipelineRoot: minio://mlpipeline/v2/artifacts + ## defaultPipelineRoot: gs://your-bucket/path/to/artifacts + ## defaultPipelineRoot: s3://your-bucket/path/to/artifacts + ## + ## V2 Compatible Mode Feature stage: + ## [Beta](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#beta) + defaultPipelineRoot: "" + ## autoUpdatePipelineDefaultVersion: States if the pipeline version + ## should be updated by defult for a versioned pipeline or not when a new + ## version is uploaded. This sets the deployment wide definition. + autoUpdatePipelineDefaultVersion: "true" + ## cronScheduleTimezone: States the timezone which should be used for + ## the cron scheduler. If not specified the local timezone of the + ## cluster will be used. Valid values are UTC, Local or values according to + ## the IANA Time Zone database, such as "America/New_York" and "Asia/Shanghai". + ## Feature stage: + ## [Alpha](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#alpha) + cronScheduleTimezone: "UTC" + ## cacheImage is the image that the mutating webhook will use to patch + ## cached steps with. Will be used to echo a message announcing that + ## the cached step result will be used. If not set it will default to + ## 'gcr.io/google-containers/busybox' + cacheImage: "gcr.io/google-containers/busybox" + ## cacheNodeRestrictions the dummy container runing if output is cached + ## will run with the same affinity and node selector as the default pipeline + ## step. This is defaulted to 'false' to allow the pod to be scheduled on + ## any node and avoid defaulting to specific nodes. Allowed values are: + ## 'false' and 'true'. + cacheNodeRestrictions: "false" + ## MAXIMUM_CACHE_STALENESS configures caching according to + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/. + ## Larger than MAXIMUM_CACHE_STALENESS per pipeline user set values are + ## reduced to MAXIMUM_CACHE_STALENESS. + ## The administrator of the storage backend can rely on it to delete old cache + ## artifacts. + MAXIMUM_CACHE_STALENESS: "" + ## MAXIMUM_CACHE_STALENESS: "P30D" + ## DEFAULT_CACHE_STALENESS configures caching according to + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/. + ## This value is used if the user did not set a value in the pipeline. + DEFAULT_CACHE_STALENESS: "" + ## DEFAULT_CACHE_STALENESS: "P7D" + ## ConMaxLifeTime will set the connection max lifetime for MySQL + ## this is very important to setup when using external databases. + ## See this issue for more details: https://github.com/kubeflow/pipelines/issues/5329 + ## Note: this value should be a string that can be parsed by `time.ParseDuration`. + ## If this value doesn't include a unit abbreviation, the units will be assumed + ## to be nanoseconds. + ConMaxLifeTime: "120s" diff --git a/manifests/kustomize/base/installs/generic/postgres/postgres-secret.yaml b/manifests/kustomize/base/installs/generic/postgres/postgres-secret.yaml new file mode 100644 index 00000000000..734ce0b5f50 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/postgres-secret.yaml @@ -0,0 +1,7 @@ +kind: Secret +apiVersion: v1 +metadata: + name: postgres-secret +stringData: + username: user + password: "password" diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index fea23811107..d296a72b835 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -33,6 +33,7 @@ spec: configMapKeyRef: name: pipeline-install-config key: bucketName + # relic variables - name: DBCONFIG_USER valueFrom: secretKeyRef: @@ -58,11 +59,44 @@ spec: configMapKeyRef: name: pipeline-install-config key: dbPort + # end of relic variables - name: DBCONFIG_CONMAXLIFETIME valueFrom: configMapKeyRef: name: pipeline-install-config key: ConMaxLifeTime + - name: DB_DRIVER_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbType + # MySQL Config + - name: DBCONFIG_MYSQLCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mysqlPort + # end of MySQL Config - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: diff --git a/manifests/kustomize/base/pipeline/postgres/kustomization.yaml b/manifests/kustomize/base/pipeline/postgres/kustomization.yaml new file mode 100644 index 00000000000..08e991d9729 --- /dev/null +++ b/manifests/kustomize/base/pipeline/postgres/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: + - ../metadata-writer +resources: + - ml-pipeline-apiserver-deployment.yaml + - ../ml-pipeline-apiserver-role.yaml + - ../ml-pipeline-apiserver-rolebinding.yaml + - ../ml-pipeline-apiserver-sa.yaml + - ../ml-pipeline-apiserver-service.yaml + - ../ml-pipeline-persistenceagent-deployment.yaml + - ../ml-pipeline-persistenceagent-role.yaml + - ../ml-pipeline-persistenceagent-rolebinding.yaml + - ../ml-pipeline-persistenceagent-sa.yaml + - ../ml-pipeline-scheduledworkflow-deployment.yaml + - ../ml-pipeline-scheduledworkflow-role.yaml + - ../ml-pipeline-scheduledworkflow-rolebinding.yaml + - ../ml-pipeline-scheduledworkflow-sa.yaml + - ../ml-pipeline-ui-deployment.yaml + - ../ml-pipeline-ui-configmap.yaml + - ../ml-pipeline-ui-role.yaml + - ../ml-pipeline-ui-rolebinding.yaml + - ../ml-pipeline-ui-sa.yaml + - ../ml-pipeline-ui-service.yaml + - ../ml-pipeline-viewer-crd-role.yaml + - ../ml-pipeline-viewer-crd-rolebinding.yaml + - ../ml-pipeline-viewer-crd-deployment.yaml + - ../ml-pipeline-viewer-crd-sa.yaml + - ../ml-pipeline-visualization-deployment.yaml + - ../ml-pipeline-visualization-sa.yaml + - ../ml-pipeline-visualization-service.yaml + - ../pipeline-runner-role.yaml + - ../pipeline-runner-rolebinding.yaml + - ../pipeline-runner-sa.yaml + - ../container-builder-sa.yaml + - ../viewer-sa.yaml + - ../kfp-launcher-configmap.yaml +images: + - name: gcr.io/ml-pipeline/api-server + newTag: 2.0.0 + - name: gcr.io/ml-pipeline/persistenceagent + newTag: 2.0.0 + - name: gcr.io/ml-pipeline/scheduledworkflow + newTag: 2.0.0 + - name: gcr.io/ml-pipeline/frontend + newTag: 2.0.0 + - name: gcr.io/ml-pipeline/viewer-crd-controller + newTag: 2.0.0 + - name: gcr.io/ml-pipeline/visualization-server + newTag: 2.0.0 diff --git a/manifests/kustomize/base/pipeline/postgres/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/postgres/ml-pipeline-apiserver-deployment.yaml new file mode 100644 index 00000000000..0cbd4bd7291 --- /dev/null +++ b/manifests/kustomize/base/pipeline/postgres/ml-pipeline-apiserver-deployment.yaml @@ -0,0 +1,161 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ml-pipeline + name: ml-pipeline +spec: + selector: + matchLabels: + app: ml-pipeline + template: + metadata: + labels: + app: ml-pipeline + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + spec: + containers: + - env: + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: bucketName + # relic variables + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: DBCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbHost + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbPort + # end of relic variables + - name: DBCONFIG_CONMAXLIFETIME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbType + # PostgreSQL Config + - name: DBCONFIG_POSTGRESQLCONFIG_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: username + - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: password + - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_POSTGRESQLCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresHost + - name: DBCONFIG_POSTGRESQLCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresPort + # end of PostgreSQL variables + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + image: gcr.io/ml-pipeline/api-server:dummy + imagePullPolicy: IfNotPresent + name: ml-pipeline-api-server + ports: + - name: http + containerPort: 8888 + - name: grpc + containerPort: 8887 + readinessProbe: + exec: + command: + - wget + - -q # quiet + - -S # show server response + - -O + - "-" # Redirect output to stdout + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + livenessProbe: + exec: + command: + - wget + - -q # quiet + - -S # show server response + - -O + - "-" # Redirect output to stdout + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + # This startup probe provides up to a 60 second grace window before the + # liveness probe takes over to accomodate the occasional database + # migration. + startupProbe: + exec: + command: + - wget + - -q # quiet + - -S # show server response + - -O + - "-" # Redirect output to stdout + - http://localhost:8888/apis/v1beta1/healthz + failureThreshold: 12 + periodSeconds: 5 + timeoutSeconds: 2 + resources: + requests: + cpu: 250m + memory: 500Mi + serviceAccountName: ml-pipeline diff --git a/manifests/kustomize/env/dev/postgresql/kustomization.yaml b/manifests/kustomize/env/dev/postgresql/kustomization.yaml new file mode 100644 index 00000000000..25aec96ef09 --- /dev/null +++ b/manifests/kustomize/env/dev/postgresql/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + # Application controller is used to provide Google Cloud Console integration. + - ../../../third-party/application + - ../../../base/application + - ../../platform-agnostic-postgresql + - ../../gcp/inverse-proxy + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources +namespace: kubeflow diff --git a/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml new file mode 100644 index 00000000000..99e03a3e6cd --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base/installs/generic/postgres + - ../../base/metadata/base + - ../../third-party/argo/installs/namespace + - ../../third-party/minio/base + - ../../third-party/postgresql/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow