From e270f2b7643461b6941d4ed66dc6fde91ec1ca53 Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Tue, 13 Mar 2018 15:08:54 +0600 Subject: [PATCH 1/7] Update Aerospike client to version 4 --- .../com/yahoo/ycsb/db/AerospikeClient.java | 24 +++++++++++++++---- pom.xml | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java index bd3b6cf02a..2f8b7755fc 100644 --- a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java +++ b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java @@ -22,6 +22,8 @@ import com.aerospike.client.Key; import com.aerospike.client.Record; import com.aerospike.client.policy.ClientPolicy; +import com.aerospike.client.policy.CommitLevel; +import com.aerospike.client.policy.ConsistencyLevel; import com.aerospike.client.policy.Policy; import com.aerospike.client.policy.RecordExistsAction; import com.aerospike.client.policy.WritePolicy; @@ -69,11 +71,25 @@ public void init() throws DBException { int port = Integer.parseInt(props.getProperty("as.port", DEFAULT_PORT)); int timeout = Integer.parseInt(props.getProperty("as.timeout", DEFAULT_TIMEOUT)); + boolean linearizeRead = props.getProperty("as.linearizeRead", "true").equals("true"); + String consistencyLevel = props.getProperty("as.consistencyLevel", "CONSISTENCY_ONE"); // or _ALL + String commitLevel = props.getProperty("as.commitLevel", "COMMIT_MASTER"); // or _ALL - readPolicy.timeout = timeout; - insertPolicy.timeout = timeout; - updatePolicy.timeout = timeout; - deletePolicy.timeout = timeout; + readPolicy.setTimeout(timeout); + insertPolicy.setTimeout(timeout); + updatePolicy.setTimeout(timeout); + deletePolicy.setTimeout(timeout); + + try { + insertPolicy.commitLevel = CommitLevel.valueOf(commitLevel); + updatePolicy.commitLevel = CommitLevel.valueOf(commitLevel); + deletePolicy.commitLevel = CommitLevel.valueOf(commitLevel); + insertPolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); + updatePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); + deletePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); + } catch(IllegalArgumentException e) { + throw e; + } ClientPolicy clientPolicy = new ClientPolicy(); diff --git a/pom.xml b/pom.xml index 6e7629df83..97bf233559 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ LICENSE file. 2.3.1 1.6.5 2.0.5 - 3.1.2 + 4.1.4 5.5.3 6.4.1 2.7.3 From 3cc528c1cb58bf6ecd3e7b4b1e5a22b4713a6982 Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Mon, 26 Mar 2018 17:17:41 +0600 Subject: [PATCH 2/7] Remove comment from aerospike client --- .../com/yahoo/ycsb/db/AerospikeClient.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java index 2f8b7755fc..b610fecad4 100644 --- a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java +++ b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java @@ -71,19 +71,32 @@ public void init() throws DBException { int port = Integer.parseInt(props.getProperty("as.port", DEFAULT_PORT)); int timeout = Integer.parseInt(props.getProperty("as.timeout", DEFAULT_TIMEOUT)); - boolean linearizeRead = props.getProperty("as.linearizeRead", "true").equals("true"); - String consistencyLevel = props.getProperty("as.consistencyLevel", "CONSISTENCY_ONE"); // or _ALL - String commitLevel = props.getProperty("as.commitLevel", "COMMIT_MASTER"); // or _ALL + boolean linearizeRead = props.getProperty("as.linearizeRead", "false") + .equals("true"); + String consistencyLevel = props.getProperty("as.consistencyLevel", + "CONSISTENCY_ONE"); // or _ALL + String commitLevel = props.getProperty("as.commitLevel", + "COMMIT_MASTER"); // or _ALL + int maxRetries = Integer.parseInt(props.getProperty("as.insertMaxRetries", "0")); readPolicy.setTimeout(timeout); insertPolicy.setTimeout(timeout); updatePolicy.setTimeout(timeout); deletePolicy.setTimeout(timeout); + insertPolicy.maxRetries = maxRetries; + readPolicy.linearizeRead = linearizeRead; + updatePolicy.linearizeRead = linearizeRead; + try { insertPolicy.commitLevel = CommitLevel.valueOf(commitLevel); updatePolicy.commitLevel = CommitLevel.valueOf(commitLevel); deletePolicy.commitLevel = CommitLevel.valueOf(commitLevel); + } catch(IllegalArgumentException e) { + throw e; + } + + try { insertPolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); updatePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); deletePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); From 3c02e09998fa5eecd42506c1e1f25231bac029a7 Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Wed, 11 Apr 2018 11:29:37 +0600 Subject: [PATCH 3/7] VB-00000: Add support for TLS --- .../src/main/java/com/yahoo/ycsb/db/AerospikeClient.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java index b610fecad4..6e8ba34d09 100644 --- a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java +++ b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java @@ -26,6 +26,7 @@ import com.aerospike.client.policy.ConsistencyLevel; import com.aerospike.client.policy.Policy; import com.aerospike.client.policy.RecordExistsAction; +import com.aerospike.client.policy.TlsPolicy; import com.aerospike.client.policy.WritePolicy; import com.yahoo.ycsb.ByteArrayByteIterator; import com.yahoo.ycsb.ByteIterator; @@ -78,6 +79,8 @@ public void init() throws DBException { String commitLevel = props.getProperty("as.commitLevel", "COMMIT_MASTER"); // or _ALL int maxRetries = Integer.parseInt(props.getProperty("as.insertMaxRetries", "0")); + boolean tlsEnable = props.getProperty("as.tlsEnable", "false") + .equals("true"); readPolicy.setTimeout(timeout); insertPolicy.setTimeout(timeout); @@ -96,7 +99,7 @@ public void init() throws DBException { throw e; } - try { + try { insertPolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); updatePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); deletePolicy.consistencyLevel = ConsistencyLevel.valueOf(consistencyLevel); @@ -105,6 +108,10 @@ public void init() throws DBException { } ClientPolicy clientPolicy = new ClientPolicy(); + + if (tlsEnable) { + clientPolicy.tlsPolicy = new TlsPolicy(); + } if (user != null && password != null) { clientPolicy.user = user; From ab70d54344ac363ee7b5d15787078a48754ac62c Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Wed, 11 Apr 2018 17:47:32 +0600 Subject: [PATCH 4/7] VB-000000: Fix tls issue --- .../src/main/java/com/yahoo/ycsb/db/AerospikeClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java index 6e8ba34d09..fc71c3401f 100644 --- a/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java +++ b/aerospike/src/main/java/com/yahoo/ycsb/db/AerospikeClient.java @@ -19,6 +19,7 @@ import com.aerospike.client.AerospikeException; import com.aerospike.client.Bin; +import com.aerospike.client.Host; import com.aerospike.client.Key; import com.aerospike.client.Record; import com.aerospike.client.policy.ClientPolicy; @@ -118,9 +119,11 @@ public void init() throws DBException { clientPolicy.password = password; } + Host[] hosts = Host.parseHosts(host, port); + try { client = - new com.aerospike.client.AerospikeClient(clientPolicy, host, port); + new com.aerospike.client.AerospikeClient(clientPolicy, hosts); } catch (AerospikeException e) { throw new DBException(String.format("Error while creating Aerospike " + "client for %s:%d.", host, port), e); From a9cba0942a1cecb21857080650d12ef8cd9645db Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Fri, 20 Apr 2018 13:01:48 +0600 Subject: [PATCH 5/7] VB-000000: Add README --- aerospike/README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/aerospike/README.md b/aerospike/README.md index c5bb13ef82..46f4f0d7f5 100644 --- a/aerospike/README.md +++ b/aerospike/README.md @@ -56,3 +56,85 @@ Run the workload test: ./bin/ycsb run aerospike -s -P workloads/workloada >outputRun.txt +This is patched version and it contains next things. + +## Lineate changes + +### New arguments + * `as.tlsEnable` - TLS (default: `false`) + * `as.commitLevel` - AP mode commit level: `COMMIT_MASTER`, or `COMMIT_ALL` (default: `COMMIT_MASTER`) + * `as.consistencyLevel` - AP mode commit level: `CONSISTENCY_ONE`, or `CONSISTENCY_ALL` (default: `CONSISTENCY_ONE`) + * `as.linearizeRead` - SC mode session consistency (default: false) + +### Changes + * `timeout` now is socket timeout + * client bumped version 4 + +### Sample run scripts + +```SH +#!/bin/bash +YCSB="/root/ycsb-aerospike-binding-0.14.0-SNAPSHOT" +#YCSB="/opt/ycsb" +REPORTS="/opt/app/_reports" +YCSB_BIN="${YCSB}/bin/ycsb" + +WORKLOAD="$(pwd)/workloads/visa_workload_1" +HOST="sl73ovndbd01:cluster-visa:3000" + +NAMESPACE=ycsb-ap +#NAMESPACE=ycsb-mem + +THREADS=48 + +OUTPUT_NAME=`date '+%Y-%m-%d_%H-%M'_run_`${NAMESPACE} +mkdir -p "${REPORTS}/${OUTPUT_NAME}" +OUTPUT="${REPORTS}/${OUTPUT_NAME}" +cd "$YCSB" + +JVM_ARGS='' #-Djavax.net.ssl.trustStore=/root/aerospike/keystore.jks' #'-Dcom.aerospike.client.policy.linerizeRead=true' + +for (( i=0; i < 4; i++ )) +do + nohup ${YCSB_BIN} run aerospike -s -P ${WORKLOAD} \ + -jvm-args=${JVM_ARGS} \ + -p as.host=${HOST} -p as.namespace=${NAMESPACE} -p as.timeout=2000 \ + -p maxexecutiontime=1200 -p as.linearizeRead=false \ + -p as.tlsEnable=false \ + -p as.consistencyLevel=CONSISTENCY_ALL -p as.commitLevel=COMMIT_ALL \ + -p measurementtype=raw -p measurement.raw.output_file=${OUTPUT}/run$i.log \ + -threads ${THREADS} -target 20000 > ${OUTPUT}/transactions$i.dat & +done +``` + +```SH +#!/bin/bash +YCSB="/root/ycsb-aerospike-binding-0.14.0-SNAPSHOT" +#YCSB="/opt/ycsb" +REPORTS="/opt/app/_reports" +YCSB_BIN="${YCSB}/bin/ycsb" + +WORKLOAD="$(pwd)/workloads/visa_workload_1" +HOST=sl73ovndbd01.visa.com + +NAMESPACE=ycsb-sc +#NAMESPACE=ycsb-mem + +THREADS=24 + +OUTPUT_NAME=`date '+%Y-%m-%d_%H-%M'_load_`${NAMESPACE} +mkdir -p "${REPORTS}/${OUTPUT_NAME}" +OUTPUT="${REPORTS}/${OUTPUT_NAME}" +cd "$YCSB" + +# -target - by default maximum +${YCSB_BIN} load aerospike -s -P ${WORKLOAD} \ + -p as.host=${HOST} -p as.namespace=${NAMESPACE}\ + -p as.insertMaxRetries=2 -p as.timeout=12000 \ + -p as.linearizeRead=false \ + -jvm-args='-Xms100G -Xmx100G -XX:MaxGCPauseMillis=100 -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:G1NewSizePercent=1 -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCDateStamps -Xloggc:/var/log/ycsb-gc.log' \ + -threads ${THREADS} > ${OUTPUT}/load.dat + + +# -p measurementtype=raw -p measurement.raw.output_file=${OUTPUT}/load.log \ +``` From 11706bc5a06b6293fdc5e57fff77b85867bab74a Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Tue, 22 May 2018 21:28:32 +0600 Subject: [PATCH 6/7] change names --- aerospike/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aerospike/README.md b/aerospike/README.md index 46f4f0d7f5..46a03c7792 100644 --- a/aerospike/README.md +++ b/aerospike/README.md @@ -79,8 +79,8 @@ YCSB="/root/ycsb-aerospike-binding-0.14.0-SNAPSHOT" REPORTS="/opt/app/_reports" YCSB_BIN="${YCSB}/bin/ycsb" -WORKLOAD="$(pwd)/workloads/visa_workload_1" -HOST="sl73ovndbd01:cluster-visa:3000" +WORKLOAD="$(pwd)/workloads/workload_1" +HOST="host:cluster:3000" NAMESPACE=ycsb-ap #NAMESPACE=ycsb-mem @@ -114,8 +114,8 @@ YCSB="/root/ycsb-aerospike-binding-0.14.0-SNAPSHOT" REPORTS="/opt/app/_reports" YCSB_BIN="${YCSB}/bin/ycsb" -WORKLOAD="$(pwd)/workloads/visa_workload_1" -HOST=sl73ovndbd01.visa.com +WORKLOAD="$(pwd)/workloads/workload_1" +HOST=host NAMESPACE=ycsb-sc #NAMESPACE=ycsb-mem From 068a1fc40b6a82a71e4c14011115029cf88c5bda Mon Sep 17 00:00:00 2001 From: Dmitry Nikitin Date: Tue, 22 May 2018 21:29:32 +0600 Subject: [PATCH 7/7] change names --- aerospike/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aerospike/README.md b/aerospike/README.md index 46a03c7792..cdf29f1e56 100644 --- a/aerospike/README.md +++ b/aerospike/README.md @@ -58,7 +58,7 @@ Run the workload test: This is patched version and it contains next things. -## Lineate changes +## Changes ### New arguments * `as.tlsEnable` - TLS (default: `false`)