diff --git a/docker/build-images.sh b/docker/build-images.sh index ed07b64..28b0da9 100755 --- a/docker/build-images.sh +++ b/docker/build-images.sh @@ -17,3 +17,4 @@ fi docker build --tag spark:3.0.0 -f $SPARK_DOCKERFILE spark docker build --tag cassandra-diff:4c9bc4 cassandra-diff docker build --tag gemini:1.7.3 gemini +docker build --tag harry:f569b9 harry \ No newline at end of file diff --git a/docker/harry/Dockerfile b/docker/harry/Dockerfile new file mode 100644 index 0000000..551a5dc --- /dev/null +++ b/docker/harry/Dockerfile @@ -0,0 +1,26 @@ +FROM maven:3-openjdk-11 as builder + +# cassandra-harry doesn't have an official Apache distribution yet; +# we have to build it from the source, here we pin it to a known commit hash +ENV GIT_ID="f569b94cdfc41b241e970880a3d4dbbce92c652a" + +RUN git clone https://github.com/grighetto/cassandra-harry.git cassandra-harry; \ + cd cassandra-harry; \ + git checkout $GIT_ID + +WORKDIR /cassandra-harry +RUN mvn -pl '!harry-integration' clean install -DskipTests + +FROM adoptopenjdk/openjdk11 + +RUN apt-get update +RUN apt-get install -y gettext-base + +WORKDIR /cassandra-harry + +COPY --from=builder /cassandra-harry/harry-integration-external/target/harry-integration-external-0.0.1-SNAPSHOT.jar . + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["sh", "/entrypoint.sh"] \ No newline at end of file diff --git a/docker/harry/entrypoint.sh b/docker/harry/entrypoint.sh new file mode 100644 index 0000000..c9e98d4 --- /dev/null +++ b/docker/harry/entrypoint.sh @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +echo $1 +CONFIG_FILE=$1 +cat $CONFIG_FILE +OUTPUT_DIR=$2 +CLUSTER_STATE_DIR=/cassandra-harry/cluster-state + +# Replace env vars into a new config file +envsubst < $CONFIG_FILE > /cassandra-harry/config.yaml +CONFIG_FILE=/cassandra-harry/config.yaml + +java -ea \ + -Xms4g \ + -Xmx4g \ + -XX:MaxRAM=4g \ + -XX:MaxMetaspaceSize=384M \ + -XX:MetaspaceSize=128M \ + -XX:SoftRefLRUPolicyMSPerMB=0 \ + -XX:MaxDirectMemorySize=2g \ + -Dcassandra.memtable_row_overhead_computation_step=100 \ + -Djdk.attach.allowAttachSelf=true \ + -XX:+HeapDumpOnOutOfMemoryError \ + -XX:-UseBiasedLocking \ + -XX:+UseTLAB \ + -XX:+ResizeTLAB \ + -XX:+UseNUMA \ + -XX:+PerfDisableSharedMem \ + -XX:+UseConcMarkSweepGC \ + -XX:+CMSParallelRemarkEnabled \ + -XX:SurvivorRatio=8 \ + -XX:MaxTenuringThreshold=1 \ + -XX:CMSInitiatingOccupancyFraction=75 \ + -XX:+UseCMSInitiatingOccupancyOnly \ + -XX:CMSWaitDuration=10000 \ + -XX:+CMSParallelInitialMarkEnabled \ + -XX:+CMSEdenChunksRecordAlways \ + -XX:+CMSClassUnloadingEnabled \ + -XX:+UseCondCardMark \ + -XX:OnOutOfMemoryError=kill \ + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-exports java.base/sun.nio.ch=ALL-UNNAMED \ + --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED \ + --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED \ + --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED \ + --add-exports java.sql/java.sql=ALL-UNNAMED \ + --add-opens java.base/java.lang.module=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ + --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ + --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ + -Dharry.root=${CLUSTER_STATE_DIR} \ + -Dlogback.configurationFile=logback-dtest.xml \ + -jar harry-integration-external-0.0.1-SNAPSHOT.jar \ + ${CONFIG_FILE} + +if [ $? -ne 0 ]; then + if [ -e "failure.dump" ]; then + echo "Creating failure dump..." + RUN="run-$(date +%Y%m%d%H%M%S)-${RANDOM}" + mkdir ${OUTPUT_DIR} + mkdir ${OUTPUT_DIR}/cluster-state + mv ${CLUSTER_STATE_DIR}* ${OUTPUT_DIR}/cluster-state + mv operation.log ${OUTPUT_DIR}/ + mv failure.dump ${OUTPUT_DIR}/ + mv run.yaml ${OUTPUT_DIR}/ + fi +fi diff --git a/docker/push-images.sh b/docker/push-images.sh index c2d2097..edbb0b8 100755 --- a/docker/push-images.sh +++ b/docker/push-images.sh @@ -10,3 +10,7 @@ docker push $REPO/cassandra-diff:4c9bc4 # Push gemini image docker tag gemini:1.7.3 $REPO/gemini:1.7.3 docker push $REPO/gemini:1.7.3 + +# Push harry image +docker tag harry:f569b9 $REPO/harry:f569b9 +docker push $REPO/harry:f569b9 \ No newline at end of file diff --git a/helm/adelphi/templates/harry-configmap.yaml b/helm/adelphi/templates/harry-configmap.yaml new file mode 100644 index 0000000..196a024 --- /dev/null +++ b/helm/adelphi/templates/harry-configmap.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: harry-configmap + namespace: {{ .Values.namespace }} +data: + harry.yaml: | + seed: 1596731732524 + + # Default schema provider generates random schema + schema_provider: + default: {} + + drop_schema: false + create_schema: true + truncate_table: false + + # Clock is a component responsible for mapping _logical_ timestamps to _real-time_ ones. + # + # When reproducing test failures, and for validation purposes, a snapshot of such clock can + # be taken to map a real-time timestamp from the value retrieved from the database in order + # to map it back to the logical timestamp of the operation that wrote this value. + clock: + approximate_monotonic: + history_size: 7300 + epoch_length: 1 + epoch_time_unit: "SECONDS" + + # Runner is a is a component that schedules operations that change the cluster (system under test) + # and model state. + runner: + concurrent: + concurrency: 2 + partition_visitors: + - mutating: + row_visitor: + mutating: {} + + run_time: 2 + run_time_unit: "MINUTES" + + # System under test: a Cassandra node or cluster. Default implementation is in_jvm (in-jvm DTest cluster). + # Harry also supports external clusters. + system_under_test: + external: + contact_points: ${hostname} + port: ${port} + username: ${username} + password: ${password} + + # Partition descriptor selector controls how partitions is selected based on the current logical + # timestamp. Default implementation is a sliding window of partition descriptors that will visit + # one partition after the other in the window `slide_after_repeats` times. After that will + # retire one partition descriptor, and pick one instead of it. + partition_descriptor_selector: + default: + window_size: 10 + slide_after_repeats: 100 + + # Clustering descriptor selector controls how clusterings are picked within the partition: + # how many rows there can be in a partition, how many rows will be visited for a logical timestamp, + # how many operations there will be in batch, what kind of operations there will and how often + # each kind of operation is going to occur. + clustering_descriptor_selector: + default: + modifications_per_lts: + type: "constant" + constant: 10 + rows_per_modification: + type: "constant" + constant: 10 + operation_kind_weights: + WRITE: 97 + DELETE_RANGE: 1 + DELETE_ROW: 1 + DELETE_COLUMN: 1 + column_mask_bitsets: null + max_partition_size: 100 + + metric_reporter: + default: {} diff --git a/helm/adelphi/templates/harry-jobs.yaml b/helm/adelphi/templates/harry-jobs.yaml new file mode 100644 index 0000000..97ba46c --- /dev/null +++ b/helm/adelphi/templates/harry-jobs.yaml @@ -0,0 +1,79 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: harry-job + namespace: {{ .Values.namespace }} + annotations: + "helm.sh/hook": pre-install +spec: + templates: + - name: harry-source + container: + image: adelphitools/harry:f569b9 + env: + - name: hostname + value: {{ .Values.source.clusterName }}-{{ .Values.source.dc }}-service + - name: port + value: "9042" + - name: username + value: {{ .Values.source.clusterName }}-superuser + - name: password + valueFrom: + secretKeyRef: + name: {{ .Values.source.clusterName }}-superuser + key: password + args: [ + "/config/harry.yaml", + "/results" + ] + dnsConfig: + options: + - name: ndots + value: "1" + volumeMounts: + - name: results-pv + mountPath: /results + - name: config + mountPath: /config + volumes: + - name: results-pv + persistentVolumeClaim: + claimName: harry-source-pvc + - name: config + configMap: + name: harry-configmap + - name: harry-target + container: + image: adelphitools/harry:f569b9 + env: + - name: hostname + value: {{ .Values.target.clusterName }}-{{ .Values.target.dc }}-service + - name: port + value: "9042" + - name: username + value: {{ .Values.target.clusterName }}-superuser + - name: password + valueFrom: + secretKeyRef: + name: {{ .Values.target.clusterName }}-superuser + key: password + args: [ + "/config/harry.yaml", + "/results" + ] + dnsConfig: + options: + - name: ndots + value: "1" + volumeMounts: + - name: results-pv + mountPath: /results + - name: config + mountPath: /config + volumes: + - name: results-pv + persistentVolumeClaim: + claimName: harry-target-pvc + - name: config + configMap: + name: harry-configmap \ No newline at end of file diff --git a/helm/adelphi/templates/pvc.yaml b/helm/adelphi/templates/pvc.yaml index 87d25ae..0ddde8b 100644 --- a/helm/adelphi/templates/pvc.yaml +++ b/helm/adelphi/templates/pvc.yaml @@ -49,4 +49,30 @@ spec: resources: requests: storage: 20Mi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: harry-source-pvc + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.storageClassName }} + resources: + requests: + storage: 20Mi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: harry-target-pvc + namespace: {{ .Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.storageClassName }} + resources: + requests: + storage: 20Mi --- \ No newline at end of file diff --git a/helm/adelphi/templates/results-server.yaml b/helm/adelphi/templates/results-server.yaml index 12ece0c..2b48476 100644 --- a/helm/adelphi/templates/results-server.yaml +++ b/helm/adelphi/templates/results-server.yaml @@ -31,6 +31,10 @@ spec: mountPath: /etc/nginx/conf.d - name: gemini mountPath: /results/gemini + - name: harry-source + mountPath: /results/harry-source + - name: harry-target + mountPath: /results/harry-target - name: nosqlbench-source mountPath: /results/nosqlbench-source - name: nosqlbench-target @@ -44,6 +48,12 @@ spec: - name: gemini persistentVolumeClaim: claimName: gemini-pvc + - name: harry-source + persistentVolumeClaim: + claimName: harry-source-pvc + - name: harry-target + persistentVolumeClaim: + claimName: harry-target-pvc - name: nosqlbench-source persistentVolumeClaim: claimName: nosqlbench-source-pvc diff --git a/helm/adelphi/templates/workflow-adelphi.yaml b/helm/adelphi/templates/workflow-adelphi.yaml index 4378d34..ce50776 100644 --- a/helm/adelphi/templates/workflow-adelphi.yaml +++ b/helm/adelphi/templates/workflow-adelphi.yaml @@ -58,13 +58,23 @@ spec: templateRef: name: schema-job template: configure-schema - - name: nosqlbench-source + - name: harry-source + depends: "configure-schema" + templateRef: + name: harry-job + template: harry-source + - name: harry-target depends: "configure-schema" + templateRef: + name: harry-job + template: harry-target + - name: nosqlbench-source + depends: "harry-source && harry-target" templateRef: name: nosqlbench-job template: nosqlbench-source - name: nosqlbench-target - depends: "configure-schema" + depends: "harry-source && harry-target" templateRef: name: nosqlbench-job template: nosqlbench-target