From 01e3388668f126a9ac1b4214986064d6e8910bad Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 2 Nov 2022 10:37:37 +0000
Subject: [PATCH 1/9] namespace

---
 helm-chart/kuberay-apiserver/templates/role.yaml              | 2 +-
 helm-chart/kuberay-apiserver/templates/rolebinding.yaml       | 4 ++--
 .../kuberay-operator/templates/ray_rayjob_editor_role.yaml    | 2 +-
 .../kuberay-operator/templates/ray_rayjob_viewer_role.yaml    | 2 +-
 .../templates/ray_rayservice_editor_role.yaml                 | 2 +-
 .../templates/ray_rayservice_viewer_role.yaml                 | 2 +-
 helm-chart/kuberay-operator/templates/role.yaml               | 2 +-
 helm-chart/kuberay-operator/templates/rolebinding.yaml        | 4 ++--
 8 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/helm-chart/kuberay-apiserver/templates/role.yaml b/helm-chart/kuberay-apiserver/templates/role.yaml
index bc57e3cd530..961f015f13b 100644
--- a/helm-chart/kuberay-apiserver/templates/role.yaml
+++ b/helm-chart/kuberay-apiserver/templates/role.yaml
@@ -1,7 +1,7 @@
 
 {{- if .Values.rbacEnable }}
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   labels:
     app.kubernetes.io/name: {{ .Values.name }}
diff --git a/helm-chart/kuberay-apiserver/templates/rolebinding.yaml b/helm-chart/kuberay-apiserver/templates/rolebinding.yaml
index b7401266023..7ba5c672e30 100644
--- a/helm-chart/kuberay-apiserver/templates/rolebinding.yaml
+++ b/helm-chart/kuberay-apiserver/templates/rolebinding.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.rbacEnable }}
-kind: ClusterRoleBinding
+kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   labels:
@@ -10,7 +10,7 @@ subjects:
   name: {{ .Values.serviceAccount.name  }}
   namespace: {{ .Release.Namespace }}
 roleRef:
-  kind: ClusterRole
+  kind: Role
   name: {{ include "kuberay-apiserver.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
diff --git a/helm-chart/kuberay-operator/templates/ray_rayjob_editor_role.yaml b/helm-chart/kuberay-operator/templates/ray_rayjob_editor_role.yaml
index 090286ab33a..c9869fe18dc 100644
--- a/helm-chart/kuberay-operator/templates/ray_rayjob_editor_role.yaml
+++ b/helm-chart/kuberay-operator/templates/ray_rayjob_editor_role.yaml
@@ -1,6 +1,6 @@
 # permissions for end users to edit rayjobs.
 {{- if .Values.rbacEnable }}
-kind: ClusterRole
+kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   labels:
diff --git a/helm-chart/kuberay-operator/templates/ray_rayjob_viewer_role.yaml b/helm-chart/kuberay-operator/templates/ray_rayjob_viewer_role.yaml
index 332147cf7fd..e420d452aa6 100644
--- a/helm-chart/kuberay-operator/templates/ray_rayjob_viewer_role.yaml
+++ b/helm-chart/kuberay-operator/templates/ray_rayjob_viewer_role.yaml
@@ -1,6 +1,6 @@
 # permissions for end users to view rayjobs.
 {{- if .Values.rbacEnable }}
-kind: ClusterRole
+kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   labels:
diff --git a/helm-chart/kuberay-operator/templates/ray_rayservice_editor_role.yaml b/helm-chart/kuberay-operator/templates/ray_rayservice_editor_role.yaml
index ea89c9a804e..8614e343320 100644
--- a/helm-chart/kuberay-operator/templates/ray_rayservice_editor_role.yaml
+++ b/helm-chart/kuberay-operator/templates/ray_rayservice_editor_role.yaml
@@ -1,7 +1,7 @@
 # permissions for end users to edit rayservices.
 {{- if .Values.rbacEnable }}
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   name: rayservice-editor-role
 rules:
diff --git a/helm-chart/kuberay-operator/templates/ray_rayservice_viewer_role.yaml b/helm-chart/kuberay-operator/templates/ray_rayservice_viewer_role.yaml
index ba934485cfd..b8710390201 100644
--- a/helm-chart/kuberay-operator/templates/ray_rayservice_viewer_role.yaml
+++ b/helm-chart/kuberay-operator/templates/ray_rayservice_viewer_role.yaml
@@ -1,7 +1,7 @@
 # permissions for end users to view rayservices.
 {{- if .Values.rbacEnable }}
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   name: rayservice-viewer-role
 rules:
diff --git a/helm-chart/kuberay-operator/templates/role.yaml b/helm-chart/kuberay-operator/templates/role.yaml
index 11a0262e767..2c087a76ac6 100644
--- a/helm-chart/kuberay-operator/templates/role.yaml
+++ b/helm-chart/kuberay-operator/templates/role.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.rbacEnable }}
-kind: ClusterRole
+kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   labels:
diff --git a/helm-chart/kuberay-operator/templates/rolebinding.yaml b/helm-chart/kuberay-operator/templates/rolebinding.yaml
index cfa1d0cf80c..bf7c5db1690 100644
--- a/helm-chart/kuberay-operator/templates/rolebinding.yaml
+++ b/helm-chart/kuberay-operator/templates/rolebinding.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.rbacEnable }}
-kind: ClusterRoleBinding
+kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   labels:
@@ -10,7 +10,7 @@ subjects:
   name: {{ .Values.serviceAccount.name  }}
   namespace: {{ .Release.Namespace }}
 roleRef:
-  kind: ClusterRole
+  kind: Role
   name: {{ include "kuberay-operator.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
 {{- end }}

From 5f46f07062965dba8c1570f3df0d04f1b2fa6fe5 Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 2 Nov 2022 11:24:29 +0000
Subject: [PATCH 2/9] namespace fixes

---
 .../kuberay-operator/templates/deployment.yaml       |  3 +++
 helm-chart/kuberay-operator/templates/role.yaml      | 12 ++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml
index f41959adb91..0ee8d884832 100644
--- a/helm-chart/kuberay-operator/templates/deployment.yaml
+++ b/helm-chart/kuberay-operator/templates/deployment.yaml
@@ -33,6 +33,9 @@ spec:
           volumeMounts: []
           command:
             - /manager
+          args:
+           - --watch-namespace"
+           - {{ .Values.watchNamespace }}
           ports:
             - name: http
               containerPort: 8080
diff --git a/helm-chart/kuberay-operator/templates/role.yaml b/helm-chart/kuberay-operator/templates/role.yaml
index 2c087a76ac6..8788ed1d461 100644
--- a/helm-chart/kuberay-operator/templates/role.yaml
+++ b/helm-chart/kuberay-operator/templates/role.yaml
@@ -27,6 +27,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

From fa3d0455298afcea93d8b626600fcbb8e97b525e Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 2 Nov 2022 11:34:53 +0000
Subject: [PATCH 3/9] fix

---
 helm-chart/kuberay-operator/templates/deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml
index 0ee8d884832..e19f82c8632 100644
--- a/helm-chart/kuberay-operator/templates/deployment.yaml
+++ b/helm-chart/kuberay-operator/templates/deployment.yaml
@@ -34,7 +34,7 @@ spec:
           command:
             - /manager
           args:
-           - --watch-namespace"
+           - --watch-namespace
            - {{ .Values.watchNamespace }}
           ports:
             - name: http

From e0179da3f0c530ccf661914973200e62298bc09e Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 2 Nov 2022 17:34:22 +0000
Subject: [PATCH 4/9] push script

---
 scripts/helm-push.sh | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100755 scripts/helm-push.sh

diff --git a/scripts/helm-push.sh b/scripts/helm-push.sh
new file mode 100755
index 00000000000..2ad821d78e5
--- /dev/null
+++ b/scripts/helm-push.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eu -x
+helm package ./helm-chart/kuberay-apiserver ./helm-chart/kuberay-operator
+helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
+helm push kuberay-apiserver-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
\ No newline at end of file

From 22489258c2653f0113c255123b0afd0a63a93bef Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Fri, 11 Nov 2022 10:37:46 +0000
Subject: [PATCH 5/9] port raycluster

---
 .../templates/raycluster-cluster.yaml           | 15 ++++++++++++++-
 helm-chart/ray-cluster/templates/service.yaml   | 17 +++++++++++++++++
 helm-chart/ray-cluster/values.yaml              | 14 +++++++++++++-
 3 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 helm-chart/ray-cluster/templates/service.yaml

diff --git a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml
index 6d22ca0f490..d83eed99b50 100644
--- a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml
+++ b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml
@@ -14,6 +14,19 @@ spec:
   {{- if .Values.head.enableInTreeAutoscaling }}
   enableInTreeAutoscaling: {{ .Values.head.enableInTreeAutoscaling }}
   {{- end }}
+  # autoscalerOptions is an OPTIONAL field specifying configuration overrides for the Ray autoscaler.
+  # The example configuration shown below below represents the DEFAULT values.
+  autoscalerOptions:
+    # upscalingMode is "Default" or "Aggressive."
+    # Default: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster.
+    # Aggressive: Upscaling is not rate-limited.
+    upscalingMode: Default
+    # idleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources.
+    idleTimeoutSeconds: 300
+    # image optionally overrides the autoscaler's container image.
+    image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+    # imagePullPolicy optionally overrides the autoscaler container's image pull policy.
+    imagePullPolicy: {{ .Values.image.pullPolicy }}
   headGroupSpec:
     serviceType: {{ .Values.service.type }}
     rayStartParams:
@@ -117,7 +130,7 @@ spec:
       {{ $key }}: {{ $val | quote }}
     {{- end }}
     replicas: {{ .Values.worker.replicas }}
-    minReplicas: {{ .Values.worker.miniReplicas | default 1 }}
+    minReplicas: {{ .Values.worker.miniReplicas | default 0 }}
     maxReplicas: {{ .Values.worker.maxiReplicas | default 2147483647 }}
     groupName: {{ .Values.worker.groupName }}
     template:
diff --git a/helm-chart/ray-cluster/templates/service.yaml b/helm-chart/ray-cluster/templates/service.yaml
new file mode 100644
index 00000000000..57373fcc321
--- /dev/null
+++ b/helm-chart/ray-cluster/templates/service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "ray-cluster.fullname" . }}-svc
+spec:
+  ports:
+    - name: ray-client
+      protocol: TCP
+      port: 10001
+      targetPort: 10001
+    - name: dashboard
+      protocol: TCP
+      port: 8265
+      targetPort: 8265
+  selector:
+    ray.io/identifier: {{ include "ray-cluster.fullname" . }}-head
+  type: ClusterIP
diff --git a/helm-chart/ray-cluster/values.yaml b/helm-chart/ray-cluster/values.yaml
index 3afede136e6..3e54ad9e99f 100644
--- a/helm-chart/ray-cluster/values.yaml
+++ b/helm-chart/ray-cluster/values.yaml
@@ -1,6 +1,11 @@
 # Default values for ray-cluster.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
+annotations:
+  # Treat this as a helm hook to force re-creation until bug is fixed:
+  # https://github.com/ray-project/kuberay/issues/558
+  "helm.sh/hook": post-install,post-upgrade
+  "helm.sh/hook-delete-policy": before-hook-creation
 
 image:
   repository: rayproject/ray
@@ -14,6 +19,7 @@ imagePullSecrets: []
   # - name: an-existing-secret
 
 head:
+  enableInTreeAutoscaling: true
   groupName: headgroup
   replicas: 1
   type: head
@@ -50,7 +56,13 @@ head:
     - mountPath: /tmp/ray
       name: log-volume
   sidecarContainers: []
-
+  ports:
+    - containerPort: 8265
+      protocol: TCP
+    - containerPort: 10001
+      protocol: TCP
+    - containerPort: 6379
+      protocol: TCP
 
 worker:
   # If you want to disable the default workergroup

From 05b7ed18a286f14209ff326cb877ec8e16bff886 Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Fri, 11 Nov 2022 11:36:52 +0000
Subject: [PATCH 6/9] raycluster

---
 .gitignore                         |  4 +++-
 helm-chart/ray-cluster/Chart.yaml  |  2 +-
 helm-chart/ray-cluster/values.yaml |  1 -
 scripts/helm-push.sh               | 11 ++++++++---
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4b6f9fee7dd..84270ef426d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,4 +43,6 @@
 # Dependency directories (remove the comment below to include it)
 **/vendor/
 
-.ipynb_checkpoints
\ No newline at end of file
+.ipynb_checkpoints
+
+*tgz
\ No newline at end of file
diff --git a/helm-chart/ray-cluster/Chart.yaml b/helm-chart/ray-cluster/Chart.yaml
index 48850503c50..889c55a5bd1 100644
--- a/helm-chart/ray-cluster/Chart.yaml
+++ b/helm-chart/ray-cluster/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v1
 appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: ray-cluster
-version: 0.3.0
+version: 0.3.0-tbt-1
 icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png
diff --git a/helm-chart/ray-cluster/values.yaml b/helm-chart/ray-cluster/values.yaml
index 3e54ad9e99f..a5e094cad81 100644
--- a/helm-chart/ray-cluster/values.yaml
+++ b/helm-chart/ray-cluster/values.yaml
@@ -29,7 +29,6 @@ head:
     port: '6379'
     redis-password: 'LetMeInRay'  # Deprecated since Ray 1.11 due to GCS bootstrapping enabled
     dashboard-host: '0.0.0.0'
-    num-cpus: '1'  # can be auto-completed from the limits
     node-ip-address: $MY_POD_IP  # auto-completed as the head pod IP
     block: 'true'
   containerEnv:
diff --git a/scripts/helm-push.sh b/scripts/helm-push.sh
index 2ad821d78e5..798a3010df9 100755
--- a/scripts/helm-push.sh
+++ b/scripts/helm-push.sh
@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
 
 set -eu -x
-helm package ./helm-chart/kuberay-apiserver ./helm-chart/kuberay-operator
-helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
-helm push kuberay-apiserver-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
\ No newline at end of file
+helm package \
+  ./helm-chart/kuberay-apiserver \
+  ./helm-chart/kuberay-operator \
+  ./helm-chart/ray-cluster
+  
+# helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
+# helm push kuberay-apiserver-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
+# helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
\ No newline at end of file

From 9fb7c23f83ae584c4272e47a34b34c8d91139ee7 Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Fri, 11 Nov 2022 12:19:19 +0000
Subject: [PATCH 7/9] push

---
 scripts/helm-push.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/helm-push.sh b/scripts/helm-push.sh
index 798a3010df9..019634447c8 100755
--- a/scripts/helm-push.sh
+++ b/scripts/helm-push.sh
@@ -1,11 +1,15 @@
 #!/usr/bin/env bash
 
 set -eu -x
+
+version=0.3.0-tbt-"$(git rev-parse --short HEAD)"
+
 helm package \
+  --version "${version}" \
   ./helm-chart/kuberay-apiserver \
   ./helm-chart/kuberay-operator \
   ./helm-chart/ray-cluster
   
-# helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
-# helm push kuberay-apiserver-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
-# helm push kuberay-operator-0.3.0.tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
\ No newline at end of file
+helm push kuberay-operator-"${version}".tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
+helm push kuberay-apiserver-"${version}".tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
+helm push ray-cluster-"${version}".tgz oci://ghcr.io/treebeardtech/kuberay/helm-chart
\ No newline at end of file

From 3fe079639aa957e5ecb41963c24ac574586f1ec3 Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 23 Nov 2022 11:26:51 +0000
Subject: [PATCH 8/9] add apiserver node selectors/tolerations

---
 .../kuberay-apiserver/templates/deployment.yaml     | 13 ++++++++++++-
 .../kuberay-operator/templates/deployment.yaml      |  8 ++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/helm-chart/kuberay-apiserver/templates/deployment.yaml b/helm-chart/kuberay-apiserver/templates/deployment.yaml
index 7e7996efe6e..f884d50c594 100644
--- a/helm-chart/kuberay-apiserver/templates/deployment.yaml
+++ b/helm-chart/kuberay-apiserver/templates/deployment.yaml
@@ -25,4 +25,15 @@ spec:
         {{- toYaml .Values.containerPort | nindent 8 }}
         resources:
           {{- toYaml .Values.resources | nindent 10 }}
-          
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
\ No newline at end of file
diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml
index a28f5dc3876..6deeb65b8fc 100644
--- a/helm-chart/kuberay-operator/templates/deployment.yaml
+++ b/helm-chart/kuberay-operator/templates/deployment.yaml
@@ -63,11 +63,11 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-    {{- with .Values.affinity }}
+      {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
-    {{- end }}
-    {{- with .Values.tolerations }}
+      {{- end }}
+      {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-    {{- end }}
+      {{- end }}

From 816e86a14d9a96cef9694b503d95c5a58e894c6c Mon Sep 17 00:00:00 2001
From: alex-treebeard <alex@treebeard.io>
Date: Wed, 23 Nov 2022 11:38:17 +0000
Subject: [PATCH 9/9] default ns

---
 helm-chart/kuberay-operator/templates/deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml
index 6deeb65b8fc..150abe999ea 100644
--- a/helm-chart/kuberay-operator/templates/deployment.yaml
+++ b/helm-chart/kuberay-operator/templates/deployment.yaml
@@ -37,7 +37,7 @@ spec:
             - /manager
           args:
            - --watch-namespace
-           - {{ .Values.watchNamespace }}
+           - {{ (default .Values.watchNamespace .Release.Namespace) }}
           ports:
             - name: http
               containerPort: 8080