docker-compose.yml to launch proxy-api and elasticsearch

.env

@@ -0,0 +1,4 @@
+ELASTICSEARCH_PERSISTENCE_DIR=./data
+ELASTICSEARCH_LOG_DIR=./log
+PROXY_API_LOGGER=on
+PROXY_API_PORT=4001

.gitignore

@@ -0,0 +1,2 @@
+data
+log

README.md

@@ -1,2 +1,26 @@
 # ddbj-ld-proxy
 Proxy API for DDBJ search ElasticSearch cluster
+
+## Prerequisites
+
+require root permission
+
+```
+$ sysctl -w vm.max_map_count=262144
+```
+
+```
+$ cat /etc/sysctl.conf
+vm.max_map_count=262144
+```
+
+## Migration
+
+```
+$ tar xf /path/to/ddbjsearch_elasticsearch.tar --directory .
+$ ls ./data
+elasticsearch
+$ mkdir ./data/elasticsearch2
+$ chmod a+w -R ./data
+$ docker-compose up
+```

docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3'
+services:
+  elasticsearch01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    environment:
+      node.name: elasticsearch01
+      discovery.seed_hosts: elasticsearch02
+      cluster.initial_master_nodes: elasticsearch01, elasticsearch02
+      ES_JAVA_OPTS: "-Xms16g -Xmx16g"
+    volumes:
+      - ${PWD}/docker/elasticsearch:/usr/share/elasticsearch/config
+      - ${PWD}/docker/elasticsearch/entrypoint.sh:/usr/local/bin/entrypoint.sh
+      - ${ELASTICSEARCH_PERSISTENCE_DIR}/elasticsearch:/usr/share/elasticsearch/data
+    user: ${UID:-1000}:${GID:-0}
+    command: ["bash", "/usr/local/bin/entrypoint.sh"]
+    depends_on:
+      - elasticsearch02
+
+  elasticsearch02:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
+    ports:
+      - "9201:9200"
+      - "9301:9300"
+    environment:
+      node.name: elasticsearch02
+      discovery.seed_hosts: elasticsearch01
+      cluster.initial_master_nodes: elasticsearch01, elasticsearch02
+      ES_JAVA_OPTS: "-Xms16g -Xmx16g"
+    volumes:
+      - ${PWD}/docker/elasticsearch:/usr/share/elasticsearch/config
+      - ${PWD}/docker/elasticsearch/entrypoint.sh:/usr/local/bin/entrypoint.sh
+      - ${ELASTICSEARCH_PERSISTENCE_DIR}/elasticsearch2:/usr/share/elasticsearch/data
+    user: ${UID:-1000}:${GID:-0}
+    command: ["bash", "/usr/local/bin/entrypoint.sh"]
+
+  proxy-api:
+    build: ./docker/proxy-api
+    ports:
+      - "4001:4001"
+    command: npm start
+    environment:
+      ELASTICSEARCH_HOST: "http://elasticsearch01:9200"
+      LOGGER: $PROXY_API_LOGGER
+      PORT: $PROXY_API_PORT
+    depends_on:
+      - elasticsearch01

docker/elasticsearch/elasticsearch.yml

@@ -0,0 +1,16 @@
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# set to 1 to allow single node clusters
+# Details: https://github.com/elastic/elasticsearch/pull/17288
+#discovery.type: single-node
+discovery.zen.minimum_master_nodes: 1
+
+
+http.cors.enabled: true
+http.cors.allow-origin: "*"
+
+http.max_content_length: 2147483647b
+
+xpack.security.enabled: false

docker/elasticsearch/entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+echo "Starting Elasticsearch..."
+bash /usr/local/bin/docker-entrypoint.sh &
+echo "Elasticsearch started."
+
+while :
+do
+  health="$(curl -fsSL "localhost:9200/_cat/health?h=status")"
+  if [[ "$health" == "green" ]] || [[ "$health" == "yellow" ]]; then
+    break
+  fi
+  sleep 1
+done
+
+echo "Start to input initial data."
+
+# JGAのマッピング
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/jga_study_mapping -d @/usr/share/elasticsearch/config/templates/jga-study-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/jga_dataset_mapping -d @/usr/share/elasticsearch/config/templates/jga-dataset-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/jga_policy_mapping -d @/usr/share/elasticsearch/config/templates/jga-policy-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/jga_dac_mapping -d @/usr/share/elasticsearch/config/templates/jga-dac-mapping.json > /dev/null 2>&1
+
+# BIOPROJECTのマッピング
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/bioproject_mapping -d @/usr/share/elasticsearch/config/templates/bioproject-mapping.json > /dev/null 2>&1
+
+# BIOSAMPLEのマッピング
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/biosample_mapping -d @/usr/share/elasticsearch/config/templates/biosample-mapping.json > /dev/null 2>&1
+
+# DRAのマッピング
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_submission_mapping -d @/usr/share/elasticsearch/config/templates/sra-submission-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_experiment_mapping -d @/usr/share/elasticsearch/config/templates/sra-experiment-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_analysis_mapping -d @/usr/share/elasticsearch/config/templates/sra-analysis-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_run_mapping -d @/usr/share/elasticsearch/config/templates/sra-run-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_study_mapping -d @/usr/share/elasticsearch/config/templates/sra-study-mapping.json > /dev/null 2>&1
+curl -H "Content-Type:application/json" -XPUT localhost:9200/_index_template/sra_sample_mapping -d @/usr/share/elasticsearch/config/templates/sra-sample-mapping.json > /dev/null 2>&1
+
+echo "Finish to input initial data."
+tail -f /dev/null

docker/elasticsearch/jvm.options

@@ -0,0 +1,82 @@
+################################################################
+##
+## JVM configuration
+##
+################################################################
+##
+## WARNING: DO NOT EDIT THIS FILE. If you want to override the
+## JVM options in this file, or set any additional options, you
+## should create one or more files in the jvm.options.d
+## directory containing your adjustments.
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/jvm-options.html
+## for more information.
+##
+################################################################
+
+
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## The heap size is automatically configured by Elasticsearch
+## based on the available memory in your system and the roles
+## each node is configured to fulfill. If specifying heap is
+## required, it should be done through a file in jvm.options.d,
+## which should be named with .options suffix, and the min and
+## max should be set to the same value. For example, to set the
+## heap to 4 GB, create a new file in the jvm.options.d
+## directory containing these lines:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/@project.minor.version@/heap-size.html
+## for more information
+##
+################################################################
+
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below here are considered expert settings. Do
+## not adjust them unless you understand what you are doing. Do
+## not edit them in this file; instead, create a new file in the
+## jvm.options.d directory containing your adjustments.
+##
+################################################################
+
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 8-13:-XX:-UseConcMarkSweepGC
+# 8-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails; heap dumps
+# are created in the working directory of the JVM unless an alternative path is
+# specified
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+# @heap.dump.path@
+
+# specify an alternative path for JVM fatal error logs
+# @error.file@
+
+## GC logging
+# -Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,pid,tags:filecount=32,filesize=64m

docker/elasticsearch/log4j2.properties

@@ -0,0 +1,9 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console