From 8ea98ff43900754da208d7d7173bc7f951b5594f Mon Sep 17 00:00:00 2001 From: Alexander Patrikalakis Date: Sun, 26 Feb 2017 12:46:55 +0900 Subject: [PATCH 1/2] Dockerize gremlin-server.sh & JanusGraph with docker plugin Signed-off-by: Alexander Patrikalakis --- .gitignore | 1 + AUTHORS.txt | 1 + BUILDING.md | 29 ++++++++++++++ README.md | 8 +++- .../janusgraph-dist-hadoop-2/Dockerfile | 13 ++++++ janusgraph-dist/pom.xml | 28 +++++++++++++ .../gremlin-server-berkeleyje.yaml | 40 +++++++++++++++++++ ...janusgraph-berkeleyje-es-server.properties | 14 +++++++ pom.xml | 10 +++-- 9 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile create mode 100644 janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml create mode 100644 janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties diff --git a/.gitignore b/.gitignore index c3a1f7c21b..ae14e15839 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ target/ /output/ /scripts/ /elasticsearch/ +/docker/*.zip # When executing tests in alphabetical order, Maven generates temporary # files with names like this: # diff --git a/AUTHORS.txt b/AUTHORS.txt index 9b6573ad95..5fbb36b051 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -11,6 +11,7 @@ # Please keep the list sorted. +Amazon Aurelius DataStax Dylan Bethune-Waddell diff --git a/BUILDING.md b/BUILDING.md index c6b9f91755..1ebe1d9c33 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -30,6 +30,35 @@ To build with only the TinkerPop tests\*: mvn clean install -Dtest.skip.tp=false -DskipTests=true ``` +To build and run a docker image with JanusGraph and Gremlin Server, configured +to run the berkeleyje backend with an embedded ElasticSearch instance: + +```bash +mvn clean package -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true +cd janusgraph-hbase-parent/janusgraph-hbase-core && mvn install -DskipTests=true && cd ../.. +cd janusgraph-dist && mvn install -Pjanusgraph-docker -DskipTests=true docker:build +docker run -d -p 8182:8182 --name janusgraph janusgraph/server:latest +``` + +To connect to the server in the same container on the console: + +```bash +docker exec -i -t janusgraph /var/janusgraph/bin/gremlin.sh +``` + +Then you can interact with the graph on the console through the `:remote` interface: + +```groovy +gremlin> :remote connect tinkerpop.server conf/remote.yaml +==>Configured localhost/127.0.0.1:8182 +gremlin> :remote console +==>All scripts will now be sent to Gremlin Server - [localhost/127.0.0.1:8182] - type ':remote console' to return to local mode +gremlin> GraphOfTheGodsFactory.load(graph) +==>null +gremlin> g = graph.traversal() +==>graphtraversalsource[standardjanusgraph[berkeleyje:db/berkeley], standard] +``` + ## Building on Eclipse IDE Note that this has only been tested on Eclipse Neon.2 Release (4.6.2) with m2e (1.7.0.20160603-1933) and m2e-wtp (1.3.1.20160831-1005) plugin. diff --git a/README.md b/README.md index 4a29419388..4dc2fa212c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ ![JanusGraph logo](janusgraph.png) -JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Graph_database) optimized for storing and querying large graphs with billions of vertices and edges distributed across a multi-machine cluster. JanusGraph is a transactional database that can support thousands of concurrent users, complex traversals, and analytic graph queries. +JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Graph_database) +optimized for storing and querying large graphs with billions of vertices and edges +distributed across a multi-machine cluster. JanusGraph is a transactional database that +can support thousands of concurrent users, complex traversals, and analytic graph queries. [![Build Status][travis-shield]][travis-link] [![Maven][maven-shield]][maven-link] @@ -18,7 +21,8 @@ JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Gr ## Learn More -The [project homepage](http://janusgraph.org) contains more information on JanusGraph and provides links to documentation, getting-started guides and release downloads. +The [project homepage](http://janusgraph.org) contains more information on JanusGraph and +provides links to documentation, getting-started guides and release downloads. ## Community diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile new file mode 100644 index 0000000000..dbdab98488 --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile @@ -0,0 +1,13 @@ +FROM openjdk:8 + +ENV PATH "$JAVA_HOME/bin:$PATH" +RUN apt-get update -y && apt-get install -y zip +WORKDIR /var + +ARG server_zip +ARG server_base +ADD ${server_zip} /var +RUN unzip -q ${server_base}.zip && ln -s ./${server_base} ./janusgraph +WORKDIR /var/janusgraph +EXPOSE 8182 +CMD ["./bin/gremlin-server.sh", "./conf/gremlin-server/gremlin-server-berkeleyje.yaml"] diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml index 16f77b8880..d7f9827af4 100644 --- a/janusgraph-dist/pom.xml +++ b/janusgraph-dist/pom.xml @@ -39,6 +39,7 @@ ${project.basedir}/.. ${it.skip} + 0.4.13 @@ -622,6 +623,33 @@ + + janusgraph-docker + + + + + com.spotify + docker-maven-plugin + ${docker.maven.version} + + janusgraph-dist-hadoop-2 + + target/janusgraph-${project.version}-hadoop2.zip + janusgraph-${project.version}-hadoop2 + + true + janusgraph/server + + ${project.version} + + + + + + + + janusgraph-release diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml new file mode 100644 index 0000000000..984f74e07a --- /dev/null +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml @@ -0,0 +1,40 @@ +host: localhost +port: 8182 +scriptEvaluationTimeout: 30000 +channelizer: org.apache.tinkerpop.gremlin.server.channel.WebSocketChannelizer +graphs: { + graph: conf/gremlin-server/janusgraph-berkeleyje-es-server.properties} +plugins: + - janusgraph.imports +scriptEngines: { + gremlin-groovy: { + imports: [java.lang.Math], + staticImports: [java.lang.Math.PI], + scripts: [scripts/empty-sample.groovy]}} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: {ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV2d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} +processors: + - { className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor, config: { sessionTimeout: 28800000 }} + - { className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor, config: { cacheExpirationTime: 600000, cacheMaxSize: 1000 }} +metrics: { + consoleReporter: {enabled: true, interval: 180000}, + csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv}, + jmxReporter: {enabled: true}, + slf4jReporter: {enabled: true, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000}} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false} diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties new file mode 100644 index 0000000000..cd810e92ef --- /dev/null +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties @@ -0,0 +1,14 @@ +# JanusGraph configuration sample: BerkeleyDB JE and embedded Elasticsearch +# +# This file opens a BDB JE instance in the directory +# db/berkeley. It also starts a local Elasticsearch +# service inside the same JVM running JanusGraph, persisted at +# db/es. + +gremlin.graph=org.janusgraph.core.JanusGraphFactory +storage.backend=berkeleyje +storage.directory=db/berkeley +index.search.backend=elasticsearch +index.search.directory=db/es +index.search.elasticsearch.client-only=false +index.search.elasticsearch.local-mode=true diff --git a/pom.xml b/pom.xml index e8a06bda48..73cbf23b5c 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,7 @@ UTF-8 ${project.build.directory}/janusgraph-test false + 1.4 -Xms256m -Xmx768m -XX:+HeapDumpOnOutOfMemoryError -ea ${test.extra.jvm.opts} -Xms256m -Xmx256m -ea -XX:+HeapDumpOnOutOfMemoryError ${test.extra.jvm.opts} @@ -118,6 +119,7 @@ false true ${basedir} + 2.10.4 1.8 1.8 org.janusgraph.testcategory.MemoryTests,org.janusgraph.testcategory.PerformanceTests,org.janusgraph.testcategory.BrittleTests @@ -383,7 +385,7 @@ maven-javadoc-plugin - 2.9.1 + ${maven.javadoc.version} maven-assembly-plugin @@ -407,7 +409,7 @@ maven-gpg-plugin - 1.4 + ${maven.gpg.version} org.codehaus.mojo @@ -1113,7 +1115,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 + ${maven.javadoc.version} attach-javadocs @@ -1123,7 +1125,7 @@ maven-gpg-plugin - 1.1 + ${maven.gpg.version} sign-artifacts From 4c8c414349bb2d5f68e8b26f641d63b1b79b8908 Mon Sep 17 00:00:00 2001 From: sjudeng Date: Sun, 14 May 2017 21:13:20 -0500 Subject: [PATCH 2/2] Use Docker Compose to manage JanusGraph Gremlin Server and Elasticsearch containers Signed-off-by: sjudeng --- BUILDING.md | 14 ++++++----- .../janusgraph-dist-hadoop-2/Dockerfile | 14 +++++------ .../docker-compose.yml | 24 +++++++++++++++++++ .../es/elasticsearch.yml | 10 ++++++++ .../es/wait-for-es.sh | 18 ++++++++++++++ janusgraph-dist/pom.xml | 3 +-- .../gremlin-server-berkeleyje.yaml | 3 +-- ...janusgraph-berkeleyje-es-server.properties | 4 +--- 8 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml create mode 100644 janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml create mode 100755 janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh diff --git a/BUILDING.md b/BUILDING.md index 1ebe1d9c33..15dace899a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -30,16 +30,18 @@ To build with only the TinkerPop tests\*: mvn clean install -Dtest.skip.tp=false -DskipTests=true ``` -To build and run a docker image with JanusGraph and Gremlin Server, configured -to run the berkeleyje backend with an embedded ElasticSearch instance: +## Building Docker Image for JanusGraph Gremlin Server + +To build and run Docker images with JanusGraph and Gremlin Server, configured +to run the BerkeleyJE backend and Elasticsearch (requires [Docker Compose](https://docs.docker.com/compose/)): ```bash -mvn clean package -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true -cd janusgraph-hbase-parent/janusgraph-hbase-core && mvn install -DskipTests=true && cd ../.. -cd janusgraph-dist && mvn install -Pjanusgraph-docker -DskipTests=true docker:build -docker run -d -p 8182:8182 --name janusgraph janusgraph/server:latest +mvn clean install -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true && mvn docker:build -Pjanusgraph-docker -pl janusgraph-dist +docker-compose -f janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml up ``` +Note the above `docker-compose` call launches containers in the foreground and is convenient for monitoring logs but add "-d" to instead run in the background. + To connect to the server in the same container on the console: ```bash diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile index dbdab98488..e9ae9f3a52 100644 --- a/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile @@ -1,13 +1,11 @@ FROM openjdk:8 -ENV PATH "$JAVA_HOME/bin:$PATH" -RUN apt-get update -y && apt-get install -y zip -WORKDIR /var - ARG server_zip -ARG server_base ADD ${server_zip} /var -RUN unzip -q ${server_base}.zip && ln -s ./${server_base} ./janusgraph + +RUN apt-get update -y && apt-get install -y zip && \ + server_base=`basename ${server_zip} .zip` && \ + unzip -q /var/${server_base}.zip -d /var && \ + ln -s /var/${server_base} /var/janusgraph + WORKDIR /var/janusgraph -EXPOSE 8182 -CMD ["./bin/gremlin-server.sh", "./conf/gremlin-server/gremlin-server-berkeleyje.yaml"] diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml b/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml new file mode 100644 index 0000000000..6426f4fcfc --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml @@ -0,0 +1,24 @@ +version: '2' +services: + janusgraph: + image: janusgraph/server:latest + container_name: janusgraph + ports: + - "8182:8182" + volumes: + - ./es/wait-for-es.sh:/usr/bin/wait-for-es.sh + depends_on: + - "elasticsearch" + command: ["wait-for-es.sh", "elasticsearch", "./bin/gremlin-server.sh", "./conf/gremlin-server/gremlin-server-berkeleyje.yaml"] + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:5.3.2 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "http.host=0.0.0.0" + - "transport.host=127.0.0.1" + ports: + - "9200" + volumes: + - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml b/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml new file mode 100644 index 0000000000..ce86473e4d --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml @@ -0,0 +1,10 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# set to 1 to allow single node clusters +# Details: https://github.com/elastic/elasticsearch/pull/17288 +discovery.zen.minimum_master_nodes: 1 + +xpack.security.enabled : false + diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh b/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh new file mode 100755 index 0000000000..ed610de1f6 --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +host="$1" + +shift +cmd="$@" + +until $(curl --output /dev/null --silent --head --fail http://$host:9200); do + printf '.' + sleep 5 +done + +>&2 echo "Elasticsearch is up" + +exec $cmd + diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml index d7f9827af4..eb955cd3d1 100644 --- a/janusgraph-dist/pom.xml +++ b/janusgraph-dist/pom.xml @@ -633,10 +633,9 @@ docker-maven-plugin ${docker.maven.version} - janusgraph-dist-hadoop-2 + ${project.basedir}/janusgraph-dist-hadoop-2 target/janusgraph-${project.version}-hadoop2.zip - janusgraph-${project.version}-hadoop2 true janusgraph/server diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml index 984f74e07a..c251412dae 100644 --- a/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml @@ -36,5 +36,4 @@ maxAccumulationBufferComponents: 1024 resultIterationBatchSize: 64 writeBufferLowWaterMark: 32768 writeBufferHighWaterMark: 65536 -ssl: { - enabled: false} + diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties index cd810e92ef..146902ff5d 100644 --- a/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties @@ -9,6 +9,4 @@ gremlin.graph=org.janusgraph.core.JanusGraphFactory storage.backend=berkeleyje storage.directory=db/berkeley index.search.backend=elasticsearch -index.search.directory=db/es -index.search.elasticsearch.client-only=false -index.search.elasticsearch.local-mode=true +index.search.hostname=elasticsearch