diff --git a/.gitignore b/.gitignore index c3a1f7c21b..ae14e15839 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ target/ /output/ /scripts/ /elasticsearch/ +/docker/*.zip # When executing tests in alphabetical order, Maven generates temporary # files with names like this: # diff --git a/AUTHORS.txt b/AUTHORS.txt index 9b6573ad95..5fbb36b051 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -11,6 +11,7 @@ # Please keep the list sorted. +Amazon Aurelius DataStax Dylan Bethune-Waddell diff --git a/BUILDING.md b/BUILDING.md index c6b9f91755..15dace899a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -30,6 +30,37 @@ To build with only the TinkerPop tests\*: mvn clean install -Dtest.skip.tp=false -DskipTests=true ``` +## Building Docker Image for JanusGraph Gremlin Server + +To build and run Docker images with JanusGraph and Gremlin Server, configured +to run the BerkeleyJE backend and Elasticsearch (requires [Docker Compose](https://docs.docker.com/compose/)): + +```bash +mvn clean install -Pjanusgraph-release -Dgpg.skip=true -DskipTests=true && mvn docker:build -Pjanusgraph-docker -pl janusgraph-dist +docker-compose -f janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml up +``` + +Note the above `docker-compose` call launches containers in the foreground and is convenient for monitoring logs but add "-d" to instead run in the background. + +To connect to the server in the same container on the console: + +```bash +docker exec -i -t janusgraph /var/janusgraph/bin/gremlin.sh +``` + +Then you can interact with the graph on the console through the `:remote` interface: + +```groovy +gremlin> :remote connect tinkerpop.server conf/remote.yaml +==>Configured localhost/127.0.0.1:8182 +gremlin> :remote console +==>All scripts will now be sent to Gremlin Server - [localhost/127.0.0.1:8182] - type ':remote console' to return to local mode +gremlin> GraphOfTheGodsFactory.load(graph) +==>null +gremlin> g = graph.traversal() +==>graphtraversalsource[standardjanusgraph[berkeleyje:db/berkeley], standard] +``` + ## Building on Eclipse IDE Note that this has only been tested on Eclipse Neon.2 Release (4.6.2) with m2e (1.7.0.20160603-1933) and m2e-wtp (1.3.1.20160831-1005) plugin. diff --git a/README.md b/README.md index 4a29419388..4dc2fa212c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ ![JanusGraph logo](janusgraph.png) -JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Graph_database) optimized for storing and querying large graphs with billions of vertices and edges distributed across a multi-machine cluster. JanusGraph is a transactional database that can support thousands of concurrent users, complex traversals, and analytic graph queries. +JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Graph_database) +optimized for storing and querying large graphs with billions of vertices and edges +distributed across a multi-machine cluster. JanusGraph is a transactional database that +can support thousands of concurrent users, complex traversals, and analytic graph queries. [![Build Status][travis-shield]][travis-link] [![Maven][maven-shield]][maven-link] @@ -18,7 +21,8 @@ JanusGraph is a highly scalable [graph database](http://en.wikipedia.org/wiki/Gr ## Learn More -The [project homepage](http://janusgraph.org) contains more information on JanusGraph and provides links to documentation, getting-started guides and release downloads. +The [project homepage](http://janusgraph.org) contains more information on JanusGraph and +provides links to documentation, getting-started guides and release downloads. ## Community diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile new file mode 100644 index 0000000000..e9ae9f3a52 --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/Dockerfile @@ -0,0 +1,11 @@ +FROM openjdk:8 + +ARG server_zip +ADD ${server_zip} /var + +RUN apt-get update -y && apt-get install -y zip && \ + server_base=`basename ${server_zip} .zip` && \ + unzip -q /var/${server_base}.zip -d /var && \ + ln -s /var/${server_base} /var/janusgraph + +WORKDIR /var/janusgraph diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml b/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml new file mode 100644 index 0000000000..6426f4fcfc --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/docker-compose.yml @@ -0,0 +1,24 @@ +version: '2' +services: + janusgraph: + image: janusgraph/server:latest + container_name: janusgraph + ports: + - "8182:8182" + volumes: + - ./es/wait-for-es.sh:/usr/bin/wait-for-es.sh + depends_on: + - "elasticsearch" + command: ["wait-for-es.sh", "elasticsearch", "./bin/gremlin-server.sh", "./conf/gremlin-server/gremlin-server-berkeleyje.yaml"] + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:5.3.2 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "http.host=0.0.0.0" + - "transport.host=127.0.0.1" + ports: + - "9200" + volumes: + - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml b/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml new file mode 100644 index 0000000000..ce86473e4d --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/es/elasticsearch.yml @@ -0,0 +1,10 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# set to 1 to allow single node clusters +# Details: https://github.com/elastic/elasticsearch/pull/17288 +discovery.zen.minimum_master_nodes: 1 + +xpack.security.enabled : false + diff --git a/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh b/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh new file mode 100755 index 0000000000..ed610de1f6 --- /dev/null +++ b/janusgraph-dist/janusgraph-dist-hadoop-2/es/wait-for-es.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +host="$1" + +shift +cmd="$@" + +until $(curl --output /dev/null --silent --head --fail http://$host:9200); do + printf '.' + sleep 5 +done + +>&2 echo "Elasticsearch is up" + +exec $cmd + diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml index 16f77b8880..eb955cd3d1 100644 --- a/janusgraph-dist/pom.xml +++ b/janusgraph-dist/pom.xml @@ -39,6 +39,7 @@ ${project.basedir}/.. ${it.skip} + 0.4.13 @@ -622,6 +623,32 @@ + + janusgraph-docker + + + + + com.spotify + docker-maven-plugin + ${docker.maven.version} + + ${project.basedir}/janusgraph-dist-hadoop-2 + + target/janusgraph-${project.version}-hadoop2.zip + + true + janusgraph/server + + ${project.version} + + + + + + + + janusgraph-release diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml new file mode 100644 index 0000000000..c251412dae --- /dev/null +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/gremlin-server-berkeleyje.yaml @@ -0,0 +1,39 @@ +host: localhost +port: 8182 +scriptEvaluationTimeout: 30000 +channelizer: org.apache.tinkerpop.gremlin.server.channel.WebSocketChannelizer +graphs: { + graph: conf/gremlin-server/janusgraph-berkeleyje-es-server.properties} +plugins: + - janusgraph.imports +scriptEngines: { + gremlin-groovy: { + imports: [java.lang.Math], + staticImports: [java.lang.Math.PI], + scripts: [scripts/empty-sample.groovy]}} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0, config: {ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerGremlinV2d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, config: { ioRegistries: [org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry] }} +processors: + - { className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor, config: { sessionTimeout: 28800000 }} + - { className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor, config: { cacheExpirationTime: 600000, cacheMaxSize: 1000 }} +metrics: { + consoleReporter: {enabled: true, interval: 180000}, + csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv}, + jmxReporter: {enabled: true}, + slf4jReporter: {enabled: true, interval: 180000}, + gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST}, + graphiteReporter: {enabled: false, interval: 180000}} +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 + diff --git a/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties new file mode 100644 index 0000000000..146902ff5d --- /dev/null +++ b/janusgraph-dist/src/assembly/static/conf/gremlin-server/janusgraph-berkeleyje-es-server.properties @@ -0,0 +1,12 @@ +# JanusGraph configuration sample: BerkeleyDB JE and embedded Elasticsearch +# +# This file opens a BDB JE instance in the directory +# db/berkeley. It also starts a local Elasticsearch +# service inside the same JVM running JanusGraph, persisted at +# db/es. + +gremlin.graph=org.janusgraph.core.JanusGraphFactory +storage.backend=berkeleyje +storage.directory=db/berkeley +index.search.backend=elasticsearch +index.search.hostname=elasticsearch diff --git a/pom.xml b/pom.xml index e8a06bda48..73cbf23b5c 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,7 @@ UTF-8 ${project.build.directory}/janusgraph-test false + 1.4 -Xms256m -Xmx768m -XX:+HeapDumpOnOutOfMemoryError -ea ${test.extra.jvm.opts} -Xms256m -Xmx256m -ea -XX:+HeapDumpOnOutOfMemoryError ${test.extra.jvm.opts} @@ -118,6 +119,7 @@ false true ${basedir} + 2.10.4 1.8 1.8 org.janusgraph.testcategory.MemoryTests,org.janusgraph.testcategory.PerformanceTests,org.janusgraph.testcategory.BrittleTests @@ -383,7 +385,7 @@ maven-javadoc-plugin - 2.9.1 + ${maven.javadoc.version} maven-assembly-plugin @@ -407,7 +409,7 @@ maven-gpg-plugin - 1.4 + ${maven.gpg.version} org.codehaus.mojo @@ -1113,7 +1115,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.7 + ${maven.javadoc.version} attach-javadocs @@ -1123,7 +1125,7 @@ maven-gpg-plugin - 1.1 + ${maven.gpg.version} sign-artifacts