diff --git a/helm/adelphi/templates/cassandra-clusters.yaml b/helm/adelphi/templates/cassandra-clusters.yaml index 6923cd6..0fc0c86 100644 --- a/helm/adelphi/templates/cassandra-clusters.yaml +++ b/helm/adelphi/templates/cassandra-clusters.yaml @@ -25,8 +25,10 @@ spec: spec: allowMultipleNodesPerWorker: {{ .Values.allowMultipleNodesPerWorker }} clusterName: {{ .Values.source.clusterName }} - serverType: cassandra + serverType: {{ .Values.source.type }} serverVersion: {{ .Values.source.version }} + dseWorkloads: + graphEnabled: {{ .Values.source.graphEnabled }} managementApiAuth: insecure: {} # Limit each pod to a fixed 1 CPU cores and 2 GB of RAM. @@ -51,7 +53,11 @@ spec: authenticator: org.apache.cassandra.auth.PasswordAuthenticator authorizer: org.apache.cassandra.auth.CassandraAuthorizer role_manager: org.apache.cassandra.auth.CassandraRoleManager + {{- if or (eq .Values.target.type "dse") (semverCompare "4.0.0" .Values.target.version) }} + jvm-server-options: + {{- else }} jvm-options: + {{- end }} initial_heap_size: "800M" max_heap_size: "800M" - name: start-target-cluster @@ -72,11 +78,13 @@ spec: spec: allowMultipleNodesPerWorker: {{ .Values.allowMultipleNodesPerWorker }} clusterName: {{ .Values.target.clusterName }} - serverType: cassandra + serverType: {{ .Values.target.type }} serverVersion: {{ .Values.target.version }} {{- if .Values.target.git_identifier }} serverImage: localhost:30000/adelphi/management-api-for-apache-cassandra {{- end }} + dseWorkloads: + graphEnabled: {{ .Values.target.graphEnabled }} managementApiAuth: insecure: {} # Limit each pod to a fixed 1 CPU cores and 2 GB of RAM. @@ -101,6 +109,11 @@ spec: authenticator: org.apache.cassandra.auth.PasswordAuthenticator authorizer: org.apache.cassandra.auth.CassandraAuthorizer role_manager: org.apache.cassandra.auth.CassandraRoleManager + {{- if or (eq .Values.target.type "dse") (semverCompare "4.0.0" .Values.target.version) }} jvm-server-options: + {{- else }} + jvm-options: + {{- end }} initial_heap_size: "800M" - max_heap_size: "800M" \ No newline at end of file + max_heap_size: "800M" + \ No newline at end of file diff --git a/helm/adelphi/templates/cloudgate-proxy-service.yaml b/helm/adelphi/templates/cloudgate-proxy-service.yaml new file mode 100644 index 0000000..bdf9097 --- /dev/null +++ b/helm/adelphi/templates/cloudgate-proxy-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: cloudgate-proxy-svc + namespace: {{ .Values.namespace }} +spec: + selector: + app: cloudgate-proxy + ports: + - name: cql + protocol: TCP + port: 9042 + targetPort: 9042 \ No newline at end of file diff --git a/helm/adelphi/templates/cloudgate-proxy.yaml b/helm/adelphi/templates/cloudgate-proxy.yaml new file mode 100644 index 0000000..81b678a --- /dev/null +++ b/helm/adelphi/templates/cloudgate-proxy.yaml @@ -0,0 +1,94 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: cloudgate-proxy-template + namespace: {{ .Values.namespace }} + annotations: + "helm.sh/hook": pre-install +spec: + templates: + - name: start-cloudgate-proxy + resource: + action: create + manifest: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cloudgate-deployment + ownerReferences: + - apiVersion: argoproj.io/v1alpha1 + blockOwnerDeletion: true + kind: Workflow + name: "{{`{{workflow.name}}`}}" + uid: "{{`{{workflow.uid}}`}}" + spec: + replicas: 3 + selector: + matchLabels: + app: cloudgate-proxy + template: + metadata: + labels: + app: cloudgate-proxy + spec: + serviceAccountName: cass-operator + initContainers: + - name: get-ips + image: gcr.io/cloud-builders/kubectl:latest + command: ["sh", "-c", "-e"] + args: + - get_ips() + { + ips=`kubectl get pods -n cass-operator -l app=cloudgate-proxy -o=wide | awk '{print $6}' | grep -Ev "none|IP"`; + }; + count_replicas() + { + count=`echo $ips | wc -w`; + }; + get_ips; + count_replicas; + while [ $count -ne 3 ]; do sleep 1 && get_ips && count_replicas; done; + echo $ips; + join_ips=`echo $ips | sed "s/ /,/g"`; + kubectl create configmap zdm-proxy-ips -n {{ .Values.namespace }} --from-literal=PROXY_ADDRESSES=$join_ips --dry-run=client -o yaml | kubectl apply -f -; + containers: + - name: cloudgate + image: datastax/cloudgate-proxy + ports: + - containerPort: 9042 + hostPort: 9042 + protocol: TCP + env: + - name: PROXY_ADDRESSES + valueFrom: + configMapKeyRef: + name: zdm-proxy-ips + key: PROXY_ADDRESSES + - name: ORIGIN_CASSANDRA_USERNAME + value: "{{ .Values.source.clusterName }}-superuser" + - name: ORIGIN_CASSANDRA_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.source.clusterName }}-superuser + key: password + - name: TARGET_CASSANDRA_USERNAME + value: "{{ .Values.target.clusterName }}-superuser" + - name: TARGET_CASSANDRA_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.target.clusterName }}-superuser + key: password + - name: ORIGIN_CASSANDRA_CONTACT_POINTS + value: "{{ .Values.source.clusterName }}-{{ .Values.source.dc }}-service" + - name: ORIGIN_CASSANDRA_PORT + value: "9042" + - name: TARGET_CASSANDRA_CONTACT_POINTS + value: "{{ .Values.target.clusterName }}-{{ .Values.target.dc }}-service" + - name: TARGET_CASSANDRA_PORT + value: "9042" + - name: PROXY_QUERY_ADDRESS + value: "0.0.0.0" + - name: PROXY_QUERY_PORT + value: "9042" + - name: LOG_LEVEL + value: "DEBUG" \ No newline at end of file diff --git a/helm/adelphi/templates/nosqlbench-graph-configmap.yaml b/helm/adelphi/templates/nosqlbench-graph-configmap.yaml new file mode 100644 index 0000000..68457e2 --- /dev/null +++ b/helm/adelphi/templates/nosqlbench-graph-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nosqlbench-graph-configmap + namespace: {{ .Values.namespace }} +data: + graph_workload.yaml: |- +{{ .Values.nosqlbench.graph.workload | indent 4}} \ No newline at end of file diff --git a/helm/adelphi/templates/nosqlbench-jobs.yaml b/helm/adelphi/templates/nosqlbench-jobs.yaml index 8285878..71b2746 100644 --- a/helm/adelphi/templates/nosqlbench-jobs.yaml +++ b/helm/adelphi/templates/nosqlbench-jobs.yaml @@ -111,4 +111,111 @@ spec: emptyDir: {} - name: results-pv persistentVolumeClaim: - claimName: nosqlbench-target-pvc \ No newline at end of file + claimName: nosqlbench-target-pvc + - name: nosqlbench + inputs: + parameters: + - name: job + - name: username + - name: secret + - name: hostname + container: + image: nosqlbench/nosqlbench:4.15.45 + args: [ + "--show-stacktraces", + "--report-csv-to=/results_{{`{{inputs.parameters.job}}`}}", + "--report-graphite-to=graphite-service:9109", + "/workspace/nosqlbench_{{`{{inputs.parameters.job}}`}}.yaml", + "{{`{{inputs.parameters.job}}`}}", + "scenarioname={{`{{inputs.parameters.job}}`}}", + "hosts={{`{{inputs.parameters.hostname}}`}}", + "username={{`{{inputs.parameters.username}}`}}", + "passfile=/secret/password", + "driver=cqld3" + ] + dnsConfig: + options: + - name: ndots + value: "1" + volumeMounts: + - name: secret-volume + mountPath: /secret + readOnly: true + - name: workspace + mountPath: /workspace + - name: results-pv + mountPath: /results + initContainers: + - name: extract-schema + image: python:3.10.0b1-alpine3.13 + command: ["sh", "-c", "-e"] + args: + - pip install adelphi; + adelphi --hosts="{{`{{inputs.parameters.hostname}}`}}" --username="{{`{{inputs.parameters.username}}`}}" --password="`cat /secret/password`" --keyspaces="{{ .Values.gemini_schema_keyspace }}" --no-anonymize export-nb --rampup-cycles="{{ .Values.nosqlbench_cycles }}" --main-cycles="{{ .Values.nosqlbench_cycles }}" > /workspace/nosqlbench_{{`{{inputs.parameters.job}}`}}.yaml; + cat /workspace/nosqlbench_{{`{{inputs.parameters.job}}`}}.yaml; + volumeMounts: + - name: workspace + mountPath: /workspace + - name: secret-volume + mountPath: /secret + readOnly: true + volumes: + - name: secret-volume + secret: + secretName: "{{`{{inputs.parameters.secret}}`}}" + items: + - key: password + path: password + - name: workspace + emptyDir: {} + - name: results-pv + persistentVolumeClaim: + claimName: nosqlbench-pvc + - name: nosqlbench-graph + inputs: + parameters: + - name: job + - name: username + - name: secret + - name: hostname + container: + image: nosqlbench/nosqlbench:4.17.6 + args: [ + "--show-stacktraces", + "--report-csv-to=/results_{{`{{inputs.parameters.job}}`}}", + "--report-graphite-to=graphite-service:9109", + "/nosqlbench/graph_workload.yaml", + "hosts={{`{{inputs.parameters.hostname}}`}}", + "username={{`{{inputs.parameters.username}}`}}", + "passfile=/secret/password", + "localdc=dc1" + ] + dnsConfig: + options: + - name: ndots + value: "1" + volumeMounts: + - name: secret-volume + mountPath: /secret + readOnly: true + - name: workspace + mountPath: /workspace + - name: results-pv + mountPath: /results + - name: nosqlbench-graph + mountPath: /nosqlbench + volumes: + - name: secret-volume + secret: + secretName: "{{`{{inputs.parameters.secret}}`}}" + items: + - key: password + path: password + - name: workspace + emptyDir: {} + - name: results-pv + persistentVolumeClaim: + claimName: nosqlbench-pvc + - name: nosqlbench-graph + configMap: + name: nosqlbench-graph-configmap \ No newline at end of file diff --git a/helm/adelphi/templates/pvc.yaml b/helm/adelphi/templates/pvc.yaml index 87d25ae..6487827 100644 --- a/helm/adelphi/templates/pvc.yaml +++ b/helm/adelphi/templates/pvc.yaml @@ -39,6 +39,19 @@ spec: --- apiVersion: v1 kind: PersistentVolumeClaim +metadata: + name: nosqlbench-pvc + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.storageClassName }} + resources: + requests: + storage: 20Mi +--- +apiVersion: v1 +kind: PersistentVolumeClaim metadata: name: cassandra-diff-pvc namespace: {{ .Values.namespace }} diff --git a/helm/adelphi/templates/schema.yaml b/helm/adelphi/templates/schema.yaml new file mode 100644 index 0000000..dbfa034 --- /dev/null +++ b/helm/adelphi/templates/schema.yaml @@ -0,0 +1,35 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: schema-job + namespace: {{ .Values.namespace }} + annotations: + "helm.sh/hook": pre-install +spec: + templates: + - name: configure-schema + inputs: + parameters: + - name: username + - name: secret + - name: hostname + - name: schema + retryStrategy: + limit: 10 + container: + image: ubuntu:20.04 + env: + - name: password + valueFrom: + secretKeyRef: + name: "{{`{{inputs.parameters.secret}}`}}" + key: password + command: ["sh", "-c", "-e"] + args: + - apt update; + apt install python3-pip -y; + pip3 install -U cqlsh; + echo "Configuring schema on the source cluster..."; + echo "{{`{{inputs.parameters.schema}}`}}"; + echo "{{`{{inputs.parameters.schema}}`}}" | cqlsh --protocol-version=4 -u {{`{{inputs.parameters.username}}`}} -p $password {{`{{inputs.parameters.hostname}}`}}; + echo "Done"; \ No newline at end of file diff --git a/helm/adelphi/templates/workflow-adelphi.yaml b/helm/adelphi/templates/workflow-adelphi.yaml index 7ac6e0a..69c7da4 100644 --- a/helm/adelphi/templates/workflow-adelphi.yaml +++ b/helm/adelphi/templates/workflow-adelphi.yaml @@ -5,6 +5,9 @@ metadata: spec: entrypoint: execute serviceAccountName: cass-operator + {{- if ne .Values.workflow "cassandra" }} + suspend: {} + {{- end }} templates: - name: execute dag: diff --git a/helm/adelphi/templates/workflow-zdm-graph.yaml b/helm/adelphi/templates/workflow-zdm-graph.yaml new file mode 100644 index 0000000..bdf50d7 --- /dev/null +++ b/helm/adelphi/templates/workflow-zdm-graph.yaml @@ -0,0 +1,127 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: workflow-zdm-graph-{{ .Release.Revision }} +spec: + entrypoint: execute + serviceAccountName: cass-operator + {{- if ne .Values.workflow "zdm-graph" }} + suspend: {} + {{- end }} + templates: + - name: execute + dag: + tasks: + # START DOCKER REGISTRY + - name: start-registry + templateRef: + name: registry + template: registry-template + + # BUILD CASSANDRA FROM SOURCE + - name: build-cassandra + depends: "start-registry" + when: "{{ not (empty .Values.target.git_identifier) }}" + arguments: + parameters: + - name: git_identifier + value: "{{ .Values.target.git_identifier }}" + templateRef: + name: cassandra-image + template: cassandra-image-template + + # BUILD CASSANDRA MANAGEMENT API + - name: build-cassandra-mgmt-api + depends: "build-cassandra" + when: "{{ not (empty .Values.target.git_identifier) }}" + templateRef: + name: cassandra-mgmt-image + template: cassandra-mgmt-image-template + + # START SOURCE CASSANDRA CLUSTER + - name: cassandra-source + templateRef: + name: cassandra-clusters + template: start-source-cluster + + # START TARGET CASSANDRA CLUSTER + - name: cassandra-target + depends: "build-cassandra-mgmt-api" + templateRef: + name: cassandra-clusters + template: start-target-cluster + + # WAIT FOR CASSANDRA CLUSTERS + - name: cassandra-ready + depends: "cassandra-source && cassandra-target" + templateRef: + name: cassandra-status + template: ready + + # START CLOUDGATE PROXY + - name: start-cloudgate + depends: "cassandra-ready" + templateRef: + name: cloudgate-proxy-template + template: start-cloudgate-proxy + + # CREATE SCHEMA + - name: configure-schema + arguments: + parameters: + - name: username + value: "{{ .Values.target.clusterName }}-superuser" + - name: secret + value: "{{ .Values.target.clusterName }}-superuser" + - name: hostname + value: "cloudgate-proxy-svc.{{ .Values.namespace }}.svc.cluster.local" + - name: schema + value: "{{ .Values.cql_schema }}" + depends: "start-cloudgate" + templateRef: + name: schema-job + template: configure-schema + + # RUN NOSQLBENCH WORKLOAD THROUGH THE ZDM PROXY + - name: nosqlbench + arguments: + parameters: + - name: job + value: "zdm-graph" + - name: username + value: "{{ .Values.target.clusterName }}-superuser" + - name: secret + value: "{{ .Values.target.clusterName }}-superuser" + - name: hostname + value: "cloudgate-proxy-svc.{{ .Values.namespace }}.svc.cluster.local" + depends: "configure-schema" + templateRef: + name: nosqlbench-job + template: nosqlbench-graph + + # COMPARE CLUSTERS DATA WITH CASSANDRA-DIFF + - name: run-diff + depends: "nosqlbench" + templateRef: + name: spark-job + template: spark-job-template + - name: collect-diff-results + depends: "run-diff" + templateRef: + name: cassandra-diff + template: collect-results + # - name: gemini-oracle-sut + # depends: "collect-diff-results" + # templateRef: + # name: gemini-job + # template: gemini-oracle-sut + # - name: gemini-results + # depends: "gemini-oracle-sut" + # templateRef: + # name: gemini-results + # template: collect-results + - name: start-results-server + depends: "collect-diff-results" + templateRef: + name: results + template: results-server \ No newline at end of file diff --git a/helm/adelphi/templates/workflow-zdm.yaml b/helm/adelphi/templates/workflow-zdm.yaml new file mode 100644 index 0000000..0ee009c --- /dev/null +++ b/helm/adelphi/templates/workflow-zdm.yaml @@ -0,0 +1,127 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: workflow-zdm-{{ .Release.Revision }} +spec: + entrypoint: execute + serviceAccountName: cass-operator + {{- if ne .Values.workflow "zdm" }} + suspend: {} + {{- end }} + templates: + - name: execute + dag: + tasks: + # START DOCKER REGISTRY + - name: start-registry + templateRef: + name: registry + template: registry-template + + # BUILD CASSANDRA FROM SOURCE + - name: build-cassandra + depends: "start-registry" + when: "{{ not (empty .Values.target.git_identifier) }}" + arguments: + parameters: + - name: git_identifier + value: "{{ .Values.target.git_identifier }}" + templateRef: + name: cassandra-image + template: cassandra-image-template + + # BUILD CASSANDRA MANAGEMENT API + - name: build-cassandra-mgmt-api + depends: "build-cassandra" + when: "{{ not (empty .Values.target.git_identifier) }}" + templateRef: + name: cassandra-mgmt-image + template: cassandra-mgmt-image-template + + # START SOURCE CASSANDRA CLUSTER + - name: cassandra-source + templateRef: + name: cassandra-clusters + template: start-source-cluster + + # START TARGET CASSANDRA CLUSTER + - name: cassandra-target + depends: "build-cassandra-mgmt-api" + templateRef: + name: cassandra-clusters + template: start-target-cluster + + # WAIT FOR CASSANDRA CLUSTERS + - name: cassandra-ready + depends: "cassandra-source && cassandra-target" + templateRef: + name: cassandra-status + template: ready + + # START CLOUDGATE PROXY + - name: start-cloudgate + depends: "cassandra-ready" + templateRef: + name: cloudgate-proxy-template + template: start-cloudgate-proxy + + # CREATE SCHEMA + - name: configure-schema + arguments: + parameters: + - name: username + value: "{{ .Values.target.clusterName }}-superuser" + - name: secret + value: "{{ .Values.target.clusterName }}-superuser" + - name: hostname + value: "cloudgate-proxy-svc.{{ .Values.namespace }}.svc.cluster.local" + - name: schema + value: "{{ .Values.cql_schema }}" + depends: "start-cloudgate" + templateRef: + name: schema-job + template: configure-schema + + # RUN NOSQLBENCH WORKLOAD THROUGH THE ZDM PROXY + - name: nosqlbench + arguments: + parameters: + - name: job + value: "zdm" + - name: username + value: "{{ .Values.target.clusterName }}-superuser" + - name: secret + value: "{{ .Values.target.clusterName }}-superuser" + - name: hostname + value: "cloudgate-proxy-svc.{{ .Values.namespace }}.svc.cluster.local" + depends: "configure-schema" + templateRef: + name: nosqlbench-job + template: nosqlbench + + # COMPARE CLUSTERS DATA WITH CASSANDRA-DIFF + - name: run-diff + depends: "nosqlbench" + templateRef: + name: spark-job + template: spark-job-template + - name: collect-diff-results + depends: "run-diff" + templateRef: + name: cassandra-diff + template: collect-results + # - name: gemini-oracle-sut + # depends: "collect-diff-results" + # templateRef: + # name: gemini-job + # template: gemini-oracle-sut + # - name: gemini-results + # depends: "gemini-oracle-sut" + # templateRef: + # name: gemini-results + # template: collect-results + - name: start-results-server + depends: "collect-diff-results" + templateRef: + name: results + template: results-server \ No newline at end of file diff --git a/helm/adelphi/values.yaml b/helm/adelphi/values.yaml index d4bf9e5..17b8866 100644 --- a/helm/adelphi/values.yaml +++ b/helm/adelphi/values.yaml @@ -1,3 +1,5 @@ +workflow: cassandra + namespace: cass-operator clusterSize: 1 # number of nodes per cluster allowMultipleNodesPerWorker: false @@ -7,13 +9,15 @@ storageClassName: local-path source: dc: dc1 clusterName: source - version: "3.11.7" + version: "3.11.10" + type: cassandra cql_schema: target: dc: dc2 clusterName: target version: "4.0.0" + type: cassandra git_url: git_identifier: cql_schema: @@ -21,6 +25,10 @@ target: gemini_test_duration: 1m nosqlbench_cycles: 1000 +nosqlbench: + graph: + workload: + cassandra_diff_config: # This is how many parts we split the full token range in. # Each of these splits is then compared between the clusters diff --git a/helm/adelphi/zdm-graph-values.yaml b/helm/adelphi/zdm-graph-values.yaml new file mode 100644 index 0000000..7cc6eef --- /dev/null +++ b/helm/adelphi/zdm-graph-values.yaml @@ -0,0 +1,156 @@ +############################ +# Config overrides for ZDM +# with DSE Graph in GKE +############################ + +workflow: zdm-graph + +storageClassName: standard + +clusterSize: 3 + +source: + type: dse + version: "6.8.0" + graphEnabled: true + +target: + type: dse + version: "6.8.0" + graphEnabled: true + +nosqlbench_cycles: "10000" +gemini_schema_keyspace: datastax_banking_iot + +nosqlbench: + graph: + workload: |- + description: creates local graphs which resemble a wagon-wheel topology, using + DSE Graph, version 6.8 or newer + + scenarios: + drop-graph: run driver=cqld4 graphname=graph_wheels tags=block:drop-graph cycles===UNDEF + creategraph: run driver=cqld4 graphname=graph_wheels tags=phase:create-graph cycles===UNDEF + creategraph-classic: run driver=cqld4 graphname=graph_wheels tags=block:create-graph-classic cycles===UNDEF + schema: run driver=cqld4 graphname=graph_wheels tags=phase:graph-schema cycles===UNDEF + disable-verify: run driver=cqld4 graphname=graph_wheels tags=phase:disable-verify cycles===UNDEF + rampup: run driver==cqld4 graphname=graph_wheels tags=phase:rampup cycles=1000 + fluent: run driver=cqld4 graphname=graph_wheels tags=block:fluent cycles=10 + default: + creategraph: run driver=cqld4 graphname=graph_wheels tags=phase:create-graph cycles===UNDEF + schema: run driver=cqld4 graphname=graph_wheels tags=phase:graph-schema cycles===UNDEF + rampup: run driver==cqld4 graphname=graph_wheels tags=phase:rampup cycles=10000 + devmode: run driver=cqld4 graphname=graph_wheels tags=name:dev-mode + prodmode: run driver=cqld4 graphname=graph_wheels tags=name:dev-mode + + bindings: + sessionid: ToEpochTimeUUID()->java.util.UUID; ToString(); + deviceid: Add(200000); Div(<>); ToEpochTimeUUID()->java.util.UUID; ToString(); + type: WeightedStrings('phone:10;computer:10;') + os: WeightedStrings('android:6;ios:4;linux:2;osx:7;windows:3') + osversion: WeightedStrings('nougat:3;oreo:1;jellybean:2;4:1;4c:1;5:1;5c:1;trusty:1;xenial:1;yosemite:1;el capitan:2;sierra:3;high sierra:1;7:1;10:2') + ipaddress: Combinations('1;7;0-3;.;0-2;0-2;0-5;.;0-2;0-2;0-5') + createdtime: Add(1505256898) + diag_ten_pct: WeightedLongs('1:1;0:9') + diag_one_pct: WeightedLongs('1:1;0:99') + + blocks: + drop-graph: + statements: + drop-graph: + type: gremlin + script: "system.graph('<>').ifExists().drop();" + create-graph-classic: + statements: + creategraph: + type: gremlin + script: >- + system.graph('<>') + .classicEngine() + .create() + create-graph: + tags: + phase: create-graph + statements: + creategraph: + type: gremlin + script: >- + system.graph('<>').ifNotExists().create() + create-schema: + tags: + phase: graph-schema + statements: + graph-schema: + type: gremlin + graphname: <> + script: >- + schema.vertexLabel('session') + .ifNotExists() + .partitionBy('sessionid', Uuid) + .property('ipaddress', Text) + .property('deviceid', Uuid) + .property('createdtime', Bigint) + .create(); + + schema.vertexLabel('device') + .ifNotExists() + .partitionBy('deviceid', Uuid) + .property('type', Text) + .property('os', Text) + .property('osversion', Text) + .create(); + + schema.edgeLabel('using') + .ifNotExists() + .from('session') + .to('device') + .create() + dev-mode: + tags: + phase: dev-mode + statements: + dev-mode: + type: gremlin + graphname: <> + script: >- + schema.config().option('graph.schema_mode').set('Development'); + prod-mode: + tags: + phase: prod-mode + statements: + prod-mode: + type: gremlin + graphname: <> + script: >- + schema.config().option('graph.schema_mode').set('Production'); + rampup: + tags: + phase: rampup + statements: + main-add: + type: gremlin + diag: "{diag_one_pct}" + graphname: <> + script: >- + device = g.addV('device') + .property('deviceid', '{deviceid}' as UUID) + .property('type', '{type}') + .property('os', '{os}') + .property('osversion', '{osversion}') + .as('d') + .addV('session') + .property('sessionid', '{sessionid}' as UUID) + .property('ipaddress', '{ipaddress}') + .property('deviceid', '{deviceid}' as UUID) + .property('createdtime', {createdtime}) + .as('s') + .addE('using').from('s').to('d'); + fluent: + statements: + read: + type: fluent + graphname: <> + imports: + - "org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__" + fluent: >- + g.V().hasLabel("device").has("deviceid", UUID.fromString({deviceid})) \ No newline at end of file diff --git a/helm/adelphi/zdm-values.yaml b/helm/adelphi/zdm-values.yaml new file mode 100644 index 0000000..ce87c35 --- /dev/null +++ b/helm/adelphi/zdm-values.yaml @@ -0,0 +1,53 @@ +############################ +# Config overrides for ZDM +# in GKE +############################ + +workflow: zdm + +storageClassName: standard + +clusterSize: 3 + +target: + version: "3.11.10" + +nosqlbench_cycles: "1000000" +gemini_schema_keyspace: datastax_banking_iot + +cql_schema: |- + create keyspace if not exists datastax_banking_iot WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1' }; + + use datastax_banking_iot; + + create table if not exists transactions( + cc_no text, + year int, + transaction_time timestamp, + transaction_id text, + user_id text, + location text, + items map, + merchant text, + amount double, + status text, + notes text, + tags set, + PRIMARY KEY ((cc_no, year), transaction_time) + ) WITH CLUSTERING ORDER BY (transaction_time desc); + + create table if not exists latest_transactions( + cc_no text, + transaction_time timestamp, + transaction_id text, + user_id text, + location text, + items map, + merchant text, + amount double, + status text, + notes text, + tags set, + PRIMARY KEY (cc_no, transaction_time) + ) WITH CLUSTERING ORDER BY (transaction_time desc) + and default_time_to_live = 2592000;