PostEra on AWK EKS

.github/workflows/full_kubeflow_integration_test.yaml

@@ -128,6 +128,11 @@ jobs:
         fi
         kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '
 
+    - name: Install lib2to3 for KFP V1 SDK with Python3.12 (lib2to3 has been removed from python 3.12)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3-lib2to3
+
     - name: V1 Pipeline Test
       run: |
         pip3 install "kfp>=1.8.22,<2.0.0"
@@ -136,7 +141,7 @@ jobs:
 
     - name: V2 Pipeline Test
       run: |
-        pip3 install -U "kfp>=2.13.0"
+        pip3 install -U "kfp>=2.14.3"
         TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
         python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}"
 

.github/workflows/pipeline_swfs_test.yaml

@@ -81,6 +81,11 @@ jobs:
     - name: Port forward
       run: ./tests/port_forward_gateway.sh
 
+    - name: Install lib2to3 for KFP V1 SDK with Python3.12 (lib2to3 has been removed from python 3.12)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3-lib2to3
+
     - name: List and deploy test pipeline with V1 API
       run: |
         pip3 install "kfp>=1.8.22,<2.0.0"
@@ -93,14 +98,14 @@ jobs:
 
     - name: List and deploy test pipeline with V2 API
       run: |
-        pip3 install kfp==2.13.0
+        pip3 install kfp==2.14.3
         KF_PROFILE=kubeflow-user-example-com
         TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
         python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}"
 
     - name: Fail to list pipelines with unauthorized ServiceAccount Token (V2 API)
       run: |
-        pip3 install kfp==2.13.0
+        pip3 install kfp==2.14.3
         KF_PROFILE=kubeflow-user-example-com
         TOKEN="$(kubectl -n default create token default)"
         python3 tests/pipeline_v2_test.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}"

.github/workflows/pipeline_test.yaml

@@ -72,6 +72,11 @@ jobs:
     - name: Port forward
       run: ./tests/port_forward_gateway.sh
 
+    - name: Install lib2to3 for KFP V1 SDK with Python3.12 (lib2to3 has been removed from python 3.12)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3-lib2to3
+
     - name: V1 Pipeline Test
       run: |
         pip3 install "kfp>=1.8.22,<2.0.0"
@@ -80,7 +85,7 @@ jobs:
 
     - name: V2 Pipeline Test
       run: |
-        pip3 install "kfp>=2.13.0"
+        pip3 install "kfp>=2.14.3"
         TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
         python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}"
 

README.md

@@ -69,7 +69,7 @@ This repository periodically synchronizes all official Kubeflow components from
 | Katib | applications/katib/upstream | [v0.18.0](https://github.com/kubeflow/katib/tree/v0.18.0/manifests/v1beta1) | 13m | 476Mi | 10GB |
 | KServe | applications/kserve/kserve | [v0.15.0](https://github.com/kserve/kserve/releases/tag/v0.15.0/install/v0.15.0) | 600m | 1200Mi | 0GB |
 | KServe Models Web Application | applications/kserve/models-web-app | [v0.14.0](https://github.com/kserve/models-web-app/tree/v0.14.0/config) | 6m | 259Mi  | 0GB |
-| Kubeflow Pipelines | applications/pipeline/upstream | [2.5.0](https://github.com/kubeflow/pipelines/tree/2.5.0/manifests/kustomize) | 970m | 3552Mi | 35GB |
+| Kubeflow Pipelines | applications/pipeline/upstream | [2.14.3](https://github.com/kubeflow/pipelines/tree/2.14.3/manifests/kustomize) | 970m | 3552Mi | 35GB |
 | Kubeflow Model Registry | applications/model-registry/upstream | [v0.2.19](https://github.com/kubeflow/model-registry/tree/v0.2.19/manifests/kustomize) | 510m | 2112Mi | 20GB |
 | Spark Operator | applications/spark/spark-operator | [2.2.0](https://github.com/kubeflow/spark-operator/tree/v2.2.0) | 9m | 41Mi | 0GB |
 | Istio | common/istio | [1.26.1](https://github.com/istio/istio/releases/tag/1.26.1) | 750m | 2364Mi | 0GB |

applications/katib/upstream/installs/katib-cert-manager/katib-config.yaml

@@ -24,29 +24,92 @@ runtime:
   suggestions:
     - algorithmName: random
       image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: tpe
       image: ghcr.io/kubeflow/katib/suggestion-hyperopt:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: grid
       image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: hyperband
       image: ghcr.io/kubeflow/katib/suggestion-hyperband:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: bayesianoptimization
       image: ghcr.io/kubeflow/katib/suggestion-skopt:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: cmaes
       image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: sobol
       image: ghcr.io/kubeflow/katib/suggestion-goptuna:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: multivariate-tpe
       image: ghcr.io/kubeflow/katib/suggestion-optuna:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: enas
       image: ghcr.io/kubeflow/katib/suggestion-enas:v0.18.0
+      serviceAccountName: default-editor
       resources:
+        requests:
+          memory: 500Mi
         limits:
-          memory: 400Mi
+          memory: 500Mi
     - algorithmName: darts
       image: ghcr.io/kubeflow/katib/suggestion-darts:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
     - algorithmName: pbt
       image: ghcr.io/kubeflow/katib/suggestion-pbt:v0.18.0
+      serviceAccountName: default-editor
+      resources:
+        requests:
+          memory: 500Mi
+        limits:
+          memory: 500Mi
       persistentVolumeClaimSpec:
         accessModes:
           - ReadWriteMany

applications/katib/upstream/installs/katib-cert-manager/kustomization.yaml

@@ -11,8 +11,8 @@ resources:
   - ../../components/crd/
   # Katib DB manager.
   - ../../components/db-manager/
-  # Katib DB mysql.
-  - ../../components/mysql/
+#  # Katib DB mysql.
+#  - ../../components/mysql/
   # Katib UI.
   - ../../components/ui/
   # Katib webhooks.
@@ -32,7 +32,12 @@ images:
 
 patchesStrategicMerge:
   - patches/katib-cert-injection.yaml
-
+  - patches/db-manager.yaml
+# Modify katib-mysql-secrets with parameters for the DB.
+#secretGenerator:
+#  - name: katib-mysql-secrets
+#    envs:
+#      - secrets.env
 vars:
   - fieldref:
       fieldPath: metadata.namespace

applications/katib/upstream/installs/katib-cert-manager/patches/db-manager.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: katib-db-manager
+  namespace: kubeflow
+spec:
+  template:
+    spec:
+      containers:
+        - name: katib-db-manager
+          env:
+            - name: DB_NAME
+              value: mysql
+            - name: DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: katib-mysql-secrets
+                  key: DB_USER
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: katib-mysql-secrets
+                  key: DB_PASSWORD
+            - name: KATIB_MYSQL_DB_DATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: katib-mysql-secrets
+                  key: KATIB_MYSQL_DB_DATABASE
+            - name: KATIB_MYSQL_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: katib-mysql-secrets
+                  key: KATIB_MYSQL_DB_HOST
+            - name: KATIB_MYSQL_DB_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: katib-mysql-secrets
+                  key: KATIB_MYSQL_DB_PORT

applications/katib/upstream/installs/katib-cert-manager/secrets.env

@@ -0,0 +1,5 @@
+KATIB_MYSQL_DB_DATABASE=katib
+KATIB_MYSQL_DB_HOST=changeme_rds_instance_endpoint
+KATIB_MYSQL_DB_PORT=3306
+DB_USER=kubeflow
+DB_PASSWORD=changeme

applications/pipeline/upstream/OWNERS

@@ -1,6 +1,7 @@
 approvers:
-  - gkcalat
-  - zijianjoy
+  - chensun
+  - HumairAK
 reviewers:
-  - gkcalat
-  - zijianjoy
+  - mprahl
+  - droctothorpe
+  - hbelmiro

applications/pipeline/upstream/base/cache-deployer/kustomization.yaml

@@ -6,7 +6,7 @@ resources:
   - cache-deployer-deployment.yaml
 images:
   - name: ghcr.io/kubeflow/kfp-cache-deployer
-    newTag: 2.5.0
+    newTag: 2.14.3
 labels:
   - includeSelectors: true
     pairs:

applications/pipeline/upstream/base/cache/kustomization.yaml

@@ -8,7 +8,7 @@ resources:
   - cache-service.yaml
 images:
   - name: ghcr.io/kubeflow/kfp-cache-server
-    newTag: 2.5.0
+    newTag: 2.14.3
 labels:
   - includeSelectors: true
     pairs:

applications/pipeline/upstream/base/crds/pipelines.kubeflow.org_pipelines.yaml

@@ -41,9 +41,9 @@ spec:
             properties:
               description:
                 type: string
+              displayName:
+                type: string
             type: object
         type: object
     served: true
     storage: true
-    subresources:
-      status: {}

applications/pipeline/upstream/base/crds/pipelines.kubeflow.org_pipelineversions.yaml

@@ -43,10 +43,16 @@ spec:
                 type: string
               description:
                 type: string
+              displayName:
+                type: string
               pipelineName:
                 type: string
               pipelineSpec:
                 x-kubernetes-preserve-unknown-fields: true
+              pipelineSpecURI:
+                type: string
+              platformSpec:
+                x-kubernetes-preserve-unknown-fields: true
             required:
             - pipelineSpec
             type: object

applications/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml

@@ -11,7 +11,7 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.5.0
+  appVersion: 2.14.3
   dbHost: mysql # relic to be removed after release
   dbPort: "3306" # relic to be removed after release
   dbType: mysql
@@ -63,8 +63,8 @@ data:
   ## cacheImage is the image that the mutating webhook will use to patch
   ## cached steps with. Will be used to echo a message announcing that 
   ## the cached step result will be used. If not set it will default to 
-  ## 'registry.k8s.io/busybox'
-  cacheImage: "registry.k8s.io/busybox"
+  ## 'ghcr.io/containerd/busybox'
+  cacheImage: "ghcr.io/containerd/busybox"
   ## cacheNodeRestrictions the dummy container runing if output is cached
   ## will run with the same affinity and node selector as the default pipeline
   ## step. This is defaulted to 'false' to allow the pod to be scheduled on 

applications/pipeline/upstream/base/installs/generic/postgres/pipeline-install-config.yaml

@@ -63,8 +63,8 @@ data:
   ## cacheImage is the image that the mutating webhook will use to patch
   ## cached steps with. Will be used to echo a message announcing that 
   ## the cached step result will be used. If not set it will default to 
-  ## 'registry.k8s.io/busybox'
-  cacheImage: "registry.k8s.io/busybox"
+  ## 'ghcr.io/containerd/busybox'
+  cacheImage: "ghcr.io/containerd/busybox"
   ## cacheNodeRestrictions the dummy container runing if output is cached
   ## will run with the same affinity and node selector as the default pipeline
   ## step. This is defaulted to 'false' to allow the pod to be scheduled on 

applications/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/deployment.yaml

@@ -22,7 +22,7 @@ spec:
             drop:
             - ALL
         image: public.ecr.aws/docker/library/python:3.12
-        command: ["python", "/hooks/sync.py"]
+        command: ["sh", "-c", "export HOME=/tmp && pip install --no-cache-dir --user pyyaml && export PYTHONPATH=$PYTHONPATH:/tmp/.local/lib/python3.12/site-packages && python /hooks/sync.py"]
         envFrom:
         - configMapRef:
             name: kubeflow-pipelines-profile-controller-env
@@ -38,16 +38,6 @@ spec:
               optional: true
               name: pipeline-install-config
               key: defaultPipelineRoot
-        - name: MINIO_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: mlpipeline-minio-artifact
-              key: accesskey
-        - name: MINIO_SECRET_KEY
-          valueFrom:
-            secretKeyRef:
-              name: mlpipeline-minio-artifact
-              key: secretkey
         volumeMounts:
         - name: hooks
           mountPath: /hooks