diff --git a/.env b/.env index 411605e..dbbac02 100644 --- a/.env +++ b/.env @@ -1,20 +1,12 @@ -# Password for the 'elastic' user (at least 6 characters) -ELASTIC_PASSWORD=ElasticRocks - -# Password for the 'kibana_system' user (at least 6 characters) -KIBANA_PASSWORD=KibanaRocks +# Password for the 'admin' user (at least 6 characters) +# OPENSEARCH_PASSWORD=OpenSearchRocks +OPENSEARCH_PASSWORD=admin # Set the cluster name -CLUSTER_NAME=elasticsearch-cluster - -# Set to 'basic' or 'trial' to automatically start the 30-day trial -LICENSE=basic - -# Port to expose Elasticsearch HTTP API to the host -ES_PORT=9200 +CLUSTER_NAME=opensearch-cluster -# Port to expose Kibana to the host -KIBANA_PORT=5601 +# Port to expose OpenSearch HTTP API to the host +OS_PORT=9200 # Increase or decrease based on the available host memory (in bytes) MEM_LIMIT=1073741824 diff --git a/.gitignore b/.gitignore index 2fc4ea8..9dcd42e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ transformed_data.json icecat-products-w_price-19k-20201127.tar.gz icecat-products-w_price-19k-20201127.json *.bu -rre/target *.DS_Store +opensearch/data diff --git a/README.md b/README.md index b31d2d5..2f31cb5 100644 --- a/README.md +++ b/README.md @@ -16,31 +16,23 @@ Chorus makes deploying powerful ecommerce search easier by shifting the **buy vs 3. ***Sharing Knowledge is a Must!*** It isn't enough to just have conference talks. We need code and data samples in order to share knowledge about improving ecommerce search. Chorus is a public environment that you can use to profit from the community and also to share your next great idea! -This is the project that enables Chorus to use Elasticsearch as the search engine. For those interested in the Solr version of this stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus) +This is the project that enables Chorus to use OpenSearch as the search engine. For those interested in the Solr version of this stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus) Want to stay up-to-date with the community? Visit https://querqy.org/ to learn more, and join the [E-Commerce Search Slack](https://ecom-search.slack.com/) group for tips, tricks, help and news on what's new in the Chorus ecosystem. ## News -* 23 February 2023: [Revolutionizing E-commere Search with Vectors](https://opensourceconnections.com/blog/2023/03/15/revolutionizing-e-commerce-search-with-vectors/) - Vector search is added to Chorus. -* 23 March 2022: [Chorus, now also for Elasticsearch!](https://opensourceconnections.com/blog/2022/03/23/chorus-now-also-for-elasticsearch/) - Chorus is now available for Elasticsearch. -* 17th June 2021: [Encores? - Going beyond matching and ranking of search results](https://www.slideshare.net/o19s/encores) - Chorus is used at BerlinBuzzwords. -* 15th November 2020: [Chorus Workshop Series Announced](https://plainschwarz.com/ps-salon/) - Learn from the creators of the components of Chorus via six workshops. -* 17th October 2020: [Chorus featured at ApacheCon @Home](https://www.youtube.com/watch?v=NGtmSbOoFjA) - René and Eric give a talk at ApacheCon on Chorus. -* 10th June 2020: [Chorus Announced at BerlinBuzzwords](https://2020.berlinbuzzwords.de/session/towards-open-source-tool-stack-e-commerce-search) - First release of Chorus shared with the world at a workshop. -* April 2020: [Paul Maria Bartusch](https://twitter.com/paulbartusch), [René Kriegler](https://twitter.com/renekrie), [Johannes Peter](https://github.com/JohannesDaniel) & [Eric Pugh](https://twitter.com/dep4b) brainstorm challenges with search teams adopting technologies like Querqy and come up with the Chorus idea. +* September 2023: First release of Chorus for OpenSearch for OpenSearchCon. # What Runs Where -* The UI (Reactivesearch) runs at http://localhost:4000 | http://chorus-es-edition.dev.o19s.com:4000 -* Elasticsearch runs at http://localhost:9200 | http://chorus-es-edition.dev.o19s.com:9200 -* Kibana runs at http://localhost:5601 | http://chorus-es-edition.dev.o19s.com:5601 -* SMUI runs at http://localhost:9000 | http://chorus-es-edition.dev.o19s.com:9000 -* Quepid runs at http://localhost:3000 | http://chorus-es-edition.dev.o19s.com:3000 -* The embedding service runs at http://localhost:8000 | http://chorus-es-edition.dev.o19s.com:8000 -* Keycloak runs at http://keycloak:9080 | http://chorus-es-edition.dev.o19s.com:9080 -* Prometheus runs at http://localhost:9090 | http://chorus-es-edition.dev.o19s.com:9090 -* Grafana runs at http://localhost:9091 | http://chorus-es-edition.dev.o19s.com:9091 +* The UI (Reactivesearch) runs at http://localhost:4000 | http://chorus-os-edition.dev.o19s.com:4000 +* OpenSearch runs at http://localhost:9200 | http://chorus-os-edition.dev.o19s.com:9200 +* OpenSearch Dashboard runs at http://localhost:5601 | http://chorus-os-edition.dev.o19s.com:5601 +* SMUI runs at http://localhost:9000 | http://chorus-os-edition.dev.o19s.com:9000 +* Quepid runs at http://localhost:3000 | http://chorus-os-edition.dev.o19s.com:3000 +* The embedding service runs at http://localhost:8000 | http://chorus-os-edition.dev.o19s.com:8000 +* Keycloak runs at http://keycloak:9080 | http://chorus-os-edition.dev.o19s.com:9080 Working with macOS? Pop open all the tuning related web pages with one terminal command: > open http://localhost:4000 http://localhost:9200 http://localhost:5601 http://localhost:9000 http://localhost:3000 @@ -53,7 +45,7 @@ If you are impatient, we provide a quick start script, `./quickstart.sh` that se After setting up Chorus you can check out [Kata 1: Lets Optimize a Query](katas/001_optimize_a_query.md) for an introduction to the world of active search management. -[More Katas can be found in the Solr version of Chorus](https://github.com/querqy/chorus#structured-learning-using-chorus) and many can be transferred to this Elasticsearch based stack. Some are also covered in a video series called [Meet Pete](https://opensourceconnections.com/blog/2020/07/07/meet-pete-the-e-commerce-search-product-manager/). Feel free to open PRs to add Katas you find useful or open issues if you want to see specific Katas included. Every contribution is welcome! +[More Katas can be found in the Solr version of Chorus](https://github.com/querqy/chorus#structured-learning-using-chorus) and many can be transferred to this OpenSearch based stack. Some are also covered in a video series called [Meet Pete](https://opensourceconnections.com/blog/2020/07/07/meet-pete-the-e-commerce-search-product-manager/). Feel free to open PRs to add Katas you find useful or open issues if you want to see specific Katas included. Every contribution is welcome! # Useful Commands for Chorus @@ -63,7 +55,7 @@ To start your environment, but still run each command to set up the integrations docker-compose up --build -d ``` -The quickstart script will launch Elasticsearch, download and index the sample product data for the _ecommerce_ index: +The quickstart script will launch OpenSearch, download and index the sample product data for the _ecommerce_ index: ``` ./quickstart.sh @@ -97,7 +89,7 @@ If you want to see the logs of just one component of the Chorus stack, use: ``` docker-compose ps # list out the names of the components -docker-compose logs -tf elasticsearch # tail elasticsearch only +docker-compose logs -tf opensearch # tail OpenSearch only ``` To stop all containers, you can run: @@ -140,7 +132,6 @@ The version of the Icecat product data that Chorus [provides](https://querqy.org # Known Issues -1. SMUI stands for \'search management UI\'. It is designed to work with Solr. We provided some scripts for basic functionality with Elasticsearch but there are still limitations. You get a feeling what's currently possible by going through [Kata 1: Optimize a Query](katas/001_optimize_a_query.md). -2. RRE: The technical integration is able to run queries and get the correct results from Elasticsearch, but apparently it computes the search metrics incorrectly. +1. SMUI stands for \'search management UI\'. It is designed to work with Solr. We provided some scripts for basic functionality with OpenSearch but there are still limitations. You get a feeling what's currently possible by going through [Kata 1: Optimize a Query](katas/001_optimize_a_query.md). -Of course, contributions are very welcome to improve Chorus - The Elasticsearch Edition! +Of course, contributions are very welcome to improve Chorus - The OpenSearch Edition! diff --git a/docker-compose.yml b/docker-compose.yml index 027176d..991f703 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,21 +2,16 @@ version: '3.7' services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:8.5.3 + image: opensearchproject/opensearch:2.10.0 user: "0" command: > bash -c ' - if [ x${ELASTIC_PASSWORD} == x ]; then - echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; - exit 1; - elif [ x${KIBANA_PASSWORD} == x ]; then - echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + if [ x${OPENSEARCH_PASSWORD} == x ]; then + echo "Set the OPENSEARCH_PASSWORD environment variable in the .env file"; exit 1; fi; - echo "Waiting for Elasticsearch availability"; - until curl -u 'elastic:${ELASTIC_PASSWORD}' -s elasticsearch:9200/_cluster/health | grep "yellow"; do sleep 30; done; - echo "Setting kibana_system password"; - until curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "Waiting for OpenSearch availability"; + until curl -u 'admin:${OPENSEARCH_PASSWORD}' -s opensearch:9200/_cluster/health | grep "yellow"; do sleep 30; done; echo "All done!"; ' @@ -32,24 +27,27 @@ services: - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - mysql_data:/var/lib/mysql - elasticsearch: + opensearch-node1: depends_on: - setup - container_name: elasticsearch - build: ./elasticsearch/. + container_name: opensearch-node1 + build: ./opensearch/. environment: - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "discovery.type=single-node" - - xpack.security.enabled=true # override what is in elasticsearch.yml + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 - bootstrap.memory_lock=true - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} # default username is elastic + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} # default username is admin + - DISABLE_SECURITY_PLUGIN=true ulimits: memlock: soft: -1 hard: -1 volumes: - - ./elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - es_data:/usr/share/elasticsearch/data + - ./opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - os_data1:/usr/share/opensearch/data ports: - "9200:9200" healthcheck: @@ -58,16 +56,38 @@ services: timeout: 10s retries: 50 - kibana: - container_name: kibana - image: docker.elastic.co/kibana/kibana:8.5.3 + opensearch-node2: + depends_on: + - setup + container_name: opensearch-node2 + build: ./opensearch/. environment: - - SERVERNAME=kibana - - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 - - ELASTICSEARCH_USERNAME=kibana_system - - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} # default username is admin + - DISABLE_SECURITY_PLUGIN=true + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - ./opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - os_data2:/usr/share/opensearch/data + + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2.10.0 + container_name: opensearch-dashboards ports: - - 5601:5601 + - 5601:5601 # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + environment: + - 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200","http://opensearch-node2:9200"]' + - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" # disables security dashboards plugin in OpenSearch Dashboards chorus-ui: container_name: chorus_ui @@ -114,106 +134,93 @@ services: ports: - 6379:6379 - smui: - container_name: smui - build: ./smui - ports: - - 9000:9000 - init: true - environment: - - SMUI_HEADLINE=Chorus SMUI - - SMUI_DB_URL=jdbc:mysql://mysql:3306/smui?autoReconnect=true&useSSL=false&allowPublicKeyRetrieval=true - - SMUI_DB_USER=root - - SMUI_DB_PASSWORD=${SMUI_DB_PASSWORD} - - SMUI_DEPLOY_PRELIVE_SOLR_HOST=elasticsearch:9200 - - SMUI_TOGGLE_UI_WITH_SOLR_FIELDS=false - - SMUI_2SOLR_SOLR_HOST=elasticsearch:9200 - - SMUI_TOGGLE_DEPL_PRELIVE=true - - SMUI_TOGGLE_SPELLING=true - - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT=true - - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT_PATH=/smui/conf/smui2es.sh - - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT=/configs/ecommerce/rules.txt - - SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT=/configs/ecommerce/replace-rules.txt - - SMUI_TOGGLE_RULE_ID_LOGGING=true - - SMUI_TOGGLE_EVENTHISTORY=true - - SMUI_RULE_TAGGING_ACTIVE=true - - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json - depends_on: - - mysql +# smui: +# container_name: smui +# build: ./smui +# ports: +# - 9000:9000 +# init: true +# environment: +# - SMUI_HEADLINE=Chorus SMUI +# - SMUI_DB_URL=jdbc:mysql://mysql:3306/smui?autoReconnect=true&useSSL=false&allowPublicKeyRetrieval=true +# - SMUI_DB_USER=root +# - SMUI_DB_PASSWORD=${SMUI_DB_PASSWORD} +# - SMUI_DEPLOY_PRELIVE_SOLR_HOST=opensearch:9200 +# - SMUI_TOGGLE_UI_WITH_SOLR_FIELDS=false +# - SMUI_2SOLR_SOLR_HOST=opensearch:9200 +# - SMUI_TOGGLE_DEPL_PRELIVE=true +# - SMUI_TOGGLE_SPELLING=true +# - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT=true +# - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT_PATH=/smui/conf/smui2es.sh +# - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT=/configs/ecommerce/rules.txt +# - SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT=/configs/ecommerce/replace-rules.txt +# - SMUI_TOGGLE_RULE_ID_LOGGING=true +# - SMUI_TOGGLE_EVENTHISTORY=true +# - SMUI_RULE_TAGGING_ACTIVE=true +# - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json +# depends_on: +# - mysql - prometheus: - image: prom/prometheus:v2.32.1 - container_name: prometheus - restart: always - ports: - - 9090:9090 - command: - - '--config.file=/etc/prometheus/prometheus.yml' - healthcheck: - test: [ "CMD", "wget", "--tries=1", "--spider", "http://localhost:9090" ] - interval: 5s - timeout: 10s - volumes: - - ./prometheus/:/etc/prometheus/ - - grafana: - image: grafana/grafana:7.5.17 - container_name: grafana - restart: always - ports: - - 9091:3000 - volumes: - - ./grafana/provisioning/:/etc/grafana/provisioning/ - environment: - - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} - - GF_USERS_ALLOW_SIGN_UP=true - healthcheck: - test: [ "CMD", "wget", "--tries=1", "--spider", "http://localhost:3000" ] - interval: 5s - timeout: 10s - depends_on: - - prometheus +# prometheus: +# image: prom/prometheus:v2.32.1 +# container_name: prometheus +# restart: always +# ports: +# - 9090:9090 +# command: +# - '--config.file=/etc/prometheus/prometheus.yml' +# healthcheck: +# test: [ "CMD", "wget", "--tries=1", "--spider", "http://localhost:9090" ] +# interval: 5s +# timeout: 10s +# volumes: +# - ./prometheus/:/etc/prometheus/ - rre: - container_name: rre - build: - context: ./rre - dockerfile: Dockerfile - volumes: - - ./rre:/rre - ports: - - "7979:8080" - command: "java -jar rre-server-1.1.jar" - depends_on: - - elasticsearch +# grafana: +# image: grafana/grafana:7.5.17 +# container_name: grafana +# restart: always +# ports: +# - 9091:3000 +# volumes: +# - ./grafana/provisioning/:/etc/grafana/provisioning/ +# environment: +# - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} +# - GF_USERS_ALLOW_SIGN_UP=true +# healthcheck: +# test: [ "CMD", "wget", "--tries=1", "--spider", "http://localhost:3000" ] +# interval: 5s +# timeout: 10s +# depends_on: +# - prometheus - embeddings: - container_name: embeddings - build: - context: ./embeddings - dockerfile: Dockerfile - ports: - - "8000:8000" - environment: - - OMP_NUM_THREADS=1 - deploy: - resources: - limits: - memory: 4G - reservations: - memory: 512M +# embeddings: +# container_name: embeddings +# build: +# context: ./embeddings +# dockerfile: Dockerfile +# ports: +# - "8000:8000" +# environment: +# - OMP_NUM_THREADS=1 +# deploy: +# resources: +# limits: +# memory: 4G +# reservations: +# memory: 512M - elasticsearch-exporter: - image: quay.io/prometheuscommunity/elasticsearch-exporter:latest - container_name: elasticsearch-exporter - command: - - '--es.uri=http://elasticsearch:9200' - environment: - - ES_USERNAME=elastic - - ES_PASSWORD=${ELASTIC_PASSWORD} - restart: always - ports: - - "9114:9114" +# opensearch-exporter: +# image: quay.io/prometheuscommunity/elasticsearch-exporter:latest +# container_name: elasticsearch-exporter +# command: +# - '--es.uri=http://elasticsearch:9200' +# environment: +# - ES_USERNAME=elastic +# - ES_PASSWORD=${ELASTIC_PASSWORD} +# restart: always +# ports: +# - "9114:9114" keycloak: image: quay.io/keycloak/keycloak:21.0.2 @@ -235,4 +242,5 @@ services: volumes: mysql_data: {} - es_data: {} + os_data1: {} + os_data2: {} diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile deleted file mode 100644 index 67aa321..0000000 --- a/elasticsearch/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM docker.elastic.co/elasticsearch/elasticsearch:8.5.3 -COPY querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip /usr/share/elasticsearch/querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip -WORKDIR /usr/share/elasticsearch - -# Use this once the plugin is released -#RUN ./bin/elasticsearch-plugin install --batch \ -# "https://repo1.maven.org/maven2/org/querqy/querqy-elasticsearch/1.6.es852.0/querqy-elasticsearch-1.6.es852.0.zip" - -RUN ./bin/elasticsearch-plugin install --batch file:///usr/share/elasticsearch/querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip -COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/ diff --git a/elasticsearch/elasticsearch.yml b/elasticsearch/elasticsearch.yml deleted file mode 100644 index cfb18d0..0000000 --- a/elasticsearch/elasticsearch.yml +++ /dev/null @@ -1,15 +0,0 @@ -#xpack.security.enabled: true -network.host: 0.0.0.0 -cluster.name: "elasticsearch-cluster" -http.cors: - enabled: true - allow-origin: "*" - allow-credentials: true - allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE - allow-headers: "X-Requested-With, Content-Type, Content-Length, if-modified-since, Authorization" - -# Anonymous user can read all -xpack.security.authc: - anonymous: - roles: anonymous_user - authz_exception: true \ No newline at end of file diff --git a/elasticsearch/querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip b/elasticsearch/querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip deleted file mode 100644 index aaf3e72..0000000 Binary files a/elasticsearch/querqy-elasticsearch-1.8.es853.0-SNAPSHOT.zip and /dev/null differ diff --git a/elasticsearch/wait-for-es.sh b/elasticsearch/wait-for-es.sh deleted file mode 100755 index 1a689e4..0000000 --- a/elasticsearch/wait-for-es.sh +++ /dev/null @@ -1,3 +0,0 @@ -DOT='\033[0;37m.\033[0m' -while [[ "$(curl -u 'elastic:ElasticRocks' -s localhost:9200/_cluster/health | jq '."status"')" != "\"green\"" ]]; do printf ${DOT}; sleep 5; done -echo "" \ No newline at end of file diff --git a/grafana/provisioning/dashboards/elasticsearch_rev1.json b/grafana/provisioning/dashboards/opensearch_rev1.json similarity index 92% rename from grafana/provisioning/dashboards/elasticsearch_rev1.json rename to grafana/provisioning/dashboards/opensearch_rev1.json index 91c92b8..e770aab 100644 --- a/grafana/provisioning/dashboards/elasticsearch_rev1.json +++ b/grafana/provisioning/dashboards/opensearch_rev1.json @@ -48,7 +48,7 @@ } ] }, - "description": "Elasticsearch detailed dashboard with instance picker", + "description": "OpenSearch detailed dashboard with instance picker", "editable": true, "gnetId": 4377, "graphTooltip": 1, @@ -120,7 +120,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(elasticsearch_cluster_health_status{cluster=~\"$cluster.*\", instance=~\"$instance.*\"})", + "expr": "sum(opensearch_cluster_health_status{cluster=~\"$cluster.*\", instance=~\"$instance.*\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -207,7 +207,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"})", + "expr": "sum(opensearch_cluster_health_number_of_nodes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"})", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -290,7 +290,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -374,7 +374,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -471,7 +471,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_active_primary_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -552,7 +552,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_active_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_active_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -633,7 +633,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_initializing_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -714,7 +714,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_relocating_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_relocating_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -795,7 +795,7 @@ "tableColumn": "", "targets": [ { - "expr": "elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -869,7 +869,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_process_cpu_percent{cluster=~\"$cluster.*\",es_master_node=\"true\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_process_cpu_percent{cluster=~\"$cluster.*\",es_master_node=\"true\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "instant": false, "interval": "", @@ -880,7 +880,7 @@ "step": 10 }, { - "expr": "elasticsearch_process_cpu_percent{cluster=~\"$cluster.*\",es_data_node=\"true\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_process_cpu_percent{cluster=~\"$cluster.*\",es_data_node=\"true\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -969,7 +969,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_jvm_memory_used_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -979,7 +979,7 @@ "step": 10 }, { - "expr": "elasticsearch_jvm_memory_committed_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_jvm_memory_committed_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - committed: {{area}}", @@ -987,7 +987,7 @@ "step": 10 }, { - "expr": "elasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_jvm_memory_max_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - max: {{area}}", @@ -1073,7 +1073,7 @@ "steppedLine": false, "targets": [ { - "expr": "1-(elasticsearch_filesystem_data_available_bytes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}/elasticsearch_filesystem_data_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"})", + "expr": "1-(opensearch_filesystem_data_available_bytes{cluster=~\"$cluster.*\", instance=~\"$instance.*\"}/opensearch_filesystem_data_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"})", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1183,7 +1183,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(elasticsearch_transport_tx_size_bytes_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_transport_tx_size_bytes_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} -sent", @@ -1191,7 +1191,7 @@ "step": 10 }, { - "expr": "irate(elasticsearch_transport_rx_size_bytes_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_transport_rx_size_bytes_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} -received", @@ -1289,7 +1289,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_indices_docs{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_indices_docs{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1377,7 +1377,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_indexing_index_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1465,7 +1465,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_docs_deleted{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_docs_deleted{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1553,7 +1553,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_merges_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_merges_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1655,7 +1655,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_indexing_index_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1665,7 +1665,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_search_query_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_search_query_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - query", @@ -1673,7 +1673,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_search_fetch_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_search_fetch_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - fetch", @@ -1681,7 +1681,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_merges_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_merges_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - merges", @@ -1689,7 +1689,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_refresh_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_refresh_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - refresh", @@ -1697,7 +1697,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_flush_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_flush_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - flush", @@ -1785,7 +1785,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_indexing_index_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1795,7 +1795,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_search_query_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_search_query_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - query", @@ -1803,7 +1803,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_search_fetch_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_search_fetch_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - fetch", @@ -1811,7 +1811,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_merges_total_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_merges_total_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - merges", @@ -1819,7 +1819,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_refresh_total_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_refresh_total_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - refresh", @@ -1827,7 +1827,7 @@ "step": 4 }, { - "expr": "irate(elasticsearch_indices_flush_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_indices_flush_time_ms_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ name }} - flush", @@ -1925,7 +1925,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_search_query_time_seconds{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval]) ", + "expr": "rate(opensearch_indices_search_query_time_seconds{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval]) ", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2013,7 +2013,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_indexing_index_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2101,7 +2101,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_merges_total_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_merges_total_time_seconds_total{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2201,7 +2201,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_indices_fielddata_memory_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_indices_fielddata_memory_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2289,7 +2289,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_fielddata_evictions{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_fielddata_evictions{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2377,7 +2377,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_indices_query_cache_memory_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_indices_query_cache_memory_size_bytes{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2465,7 +2465,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_indices_query_cache_evictions{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_indices_query_cache_evictions{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2562,7 +2562,7 @@ "steppedLine": false, "targets": [ { - "expr": " irate(elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster.*\",name=~\"$node.*\"}[$interval])", + "expr": " irate(opensearch_thread_pool_rejected_count{cluster=~\"$cluster.*\",name=~\"$node.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} - {{ type }}", @@ -2644,7 +2644,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_thread_pool_active_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} - {{ type }}", @@ -2727,7 +2727,7 @@ "steppedLine": false, "targets": [ { - "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", + "expr": "opensearch_thread_pool_active_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} - {{ type }}", @@ -2809,7 +2809,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(elasticsearch_thread_pool_completed_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "irate(opensearch_thread_pool_completed_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} - {{ type }}", @@ -2906,7 +2906,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_jvm_gc_collection_seconds_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_jvm_gc_collection_seconds_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2994,7 +2994,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(elasticsearch_jvm_gc_collection_seconds_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", + "expr": "rate(opensearch_jvm_gc_collection_seconds_count{cluster=~\"$cluster.*\",name=~\"$node.*\", instance=~\"$instance.*\"}[$interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3054,7 +3054,7 @@ "schemaVersion": 14, "style": "dark", "tags": [ - "elasticsearch", + "opensearch", "App" ], "templating": { @@ -3141,7 +3141,7 @@ "multi": false, "name": "cluster", "options": [], - "query": "label_values(elasticsearch_cluster_health_status,cluster)", + "query": "label_values(opensearch_cluster_health_status,cluster)", "refresh": 1, "regex": "", "sort": 1, @@ -3161,7 +3161,7 @@ "multi": true, "name": "node", "options": [], - "query": "label_values(elasticsearch_process_cpu_percent,name)", + "query": "label_values(opensearch_process_cpu_percent,name)", "refresh": 1, "regex": "", "sort": 1, @@ -3181,7 +3181,7 @@ "multi": false, "name": "instance", "options": [], - "query": "label_values(elasticsearch_cluster_health_up, instance)", + "query": "label_values(opensearch_cluster_health_up, instance)", "refresh": 1, "regex": "", "sort": 0, @@ -3223,6 +3223,6 @@ ] }, "timezone": "browser", - "title": "Elasticsearch", + "title": "OpenSearch", "version": 3 -} \ No newline at end of file +} diff --git a/index-vectors.sh b/index-vectors.sh index 1985d66..5db148c 100755 --- a/index-vectors.sh +++ b/index-vectors.sh @@ -3,7 +3,7 @@ set -eo pipefail # The directory where we locally buffer data files before uploading to Elasticsearch -DATA_DIR="./elasticsearch/data" +DATA_DIR="./opensearch/data" # The URL prefix we're downloading from REMOTE_URL="https://o19s-public-datasets.s3.amazonaws.com/chorus/product-vectors-2023-03-23" @@ -34,9 +34,13 @@ for ((i=1; i<=NUM_FILES;i++)); fi fi - echo -e "${MAJOR}Converting JSON to NDJSON${RESET}" LOCAL_NDJSON_FILE="${LOCAL_JSON_FILE}.nd" - cat "$LOCAL_JSON_FILE" | jq -c '.[]' > $LOCAL_NDJSON_FILE + if [ -f "$LOCAL_NDJSON_FILE" ]; then + echo -e "${MAJOR}Skipping conversion from JSON to NDJSON.${RESET}" + else + echo -e "${MAJOR}Converting JSON to NDJSON${RESET}" + cat "$LOCAL_JSON_FILE" | jq -c '.[]' > $LOCAL_NDJSON_FILE + fi echo -e "${MAJOR}Splitting ${LOCAL_NDJSON_FILE} to a maximum of ${BULK_MAX} rows.${RESET}" split -l${BULK_MAX} $LOCAL_NDJSON_FILE "${DATA_DIR}/${PRODUCT_VECTORS_FILE}.splitted.nd." @@ -46,6 +50,6 @@ for ((i=1; i<=NUM_FILES;i++)); echo -e "${MAJOR}Converting ${f} to ES bulk format.${RESET}" awk '{print "{\"index\": {}}\n" $0}' ${f} > "${f}.bulk" echo -e "${MAJOR}Populating products from ${f}.bulk.${RESET}" - curl -sS -o /dev/null -u 'elastic:ElasticRocks' -X POST "localhost:9200/ecommerce/_bulk?pretty" --data-binary @"${f}.bulk" -H 'Content-type:application/x-ndjson '; + curl -sS -o /dev/null -u 'admin:admin' -X POST "localhost:9200/ecommerce/_bulk?pretty" --data-binary @"${f}.bulk" -H 'Content-type:application/x-ndjson '; done; done; diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile new file mode 100644 index 0000000..6bff233 --- /dev/null +++ b/opensearch/Dockerfile @@ -0,0 +1,16 @@ +FROM opensearchproject/opensearch:2.10.0 +#COPY opensearch-querqy-1.0.os2.3.0-SNAPSHOT.zip /usr/share/opensearch/opensearch-querqy-1.0.os2.3.0-SNAPSHOT.zip +WORKDIR /usr/share/opensearch + +# Use this once the plugin is released +#RUN ./bin/opensearch-plugin install --batch \ +# "https://repo1.maven.org/maven2/org/querqy/opensearch-querqy/1.0.os2.3.0/opensearch-querqy-1.0.os2.3.0.zip" + +#RUN ./bin/opensearch-plugin install --batch file:///usr/share/opensearch/opensearch-querqy-1.0.os2.3.0-SNAPSHOT.zip + +COPY --chown=opensearch:opensearch opensearch.yml /usr/share/opensearch/config/ +# COPY --chown=opensearch:opensearch internal_users.yml /usr/share/opensearch/config/opensearch-security/internal_users.yml + +# The following plugins are installed by default on 2.9, and default conf requires > 1 shard +# An alternative would be to override their configuration during image creation +RUN ./bin/opensearch-plugin remove opensearch-security-analytics diff --git a/opensearch/custom-opensearch.yml b/opensearch/custom-opensearch.yml new file mode 100644 index 0000000..9a066f7 --- /dev/null +++ b/opensearch/custom-opensearch.yml @@ -0,0 +1,6 @@ +- http.host:0.0.0.0 +- http.port:9200 +- http.cors.allow-origin:* +- http.cors.enabled:true +- http.cors.allow-headers:X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization +- http.cors.allow-credentials:true diff --git a/opensearch/internal_users.yml b/opensearch/internal_users.yml new file mode 100644 index 0000000..9163faf --- /dev/null +++ b/opensearch/internal_users.yml @@ -0,0 +1,15 @@ +# This is the internal user database +# The hash value is a bcrypt hash and can be generated with plugins/opensearch-security/tools/hash.sh + +_meta: + type: "internalusers" + config_version: 2 + +# Define your internal users here + +admin: + hash: "$2y$12$y.lmwluwtMN9S4ktIQPfIu578/7AL2hgdgapQfcQBNjy2kzx4GrH6" + reserved: true + backend_roles: + - "admin" + description: "Admin user" diff --git a/opensearch/opensearch-querqy-1.0.os2.3.0-SNAPSHOT.zip b/opensearch/opensearch-querqy-1.0.os2.3.0-SNAPSHOT.zip new file mode 100644 index 0000000..e69de29 diff --git a/opensearch/opensearch.yml b/opensearch/opensearch.yml new file mode 100644 index 0000000..1b4d95b --- /dev/null +++ b/opensearch/opensearch.yml @@ -0,0 +1,40 @@ +network.host: 0.0.0.0 +cluster.name: "opensearch-cluster" + +http.host: 0.0.0.0 +http.port: 9200 +http.cors.allow-origin: "*" +http.cors.enabled: true +http.cors.allow-headers: X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization +http.cors.allow-credentials: true + + +######## Start OpenSearch Security Demo Configuration ######## +# WARNING: revise all the lines below before you go into production +# plugins.security.ssl.transport.pemcert_filepath: node1.pem +# plugins.security.ssl.transport.pemkey_filepath: node1-key.pem +# plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +# plugins.security.ssl.transport.enforce_hostname_verification: false +# plugins.security.ssl.http.enabled: true +# plugins.security.ssl.http.pemcert_filepath: node1.pem +# plugins.security.ssl.http.pemkey_filepath: node1-key.pem +# plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem +# plugins.security.allow_unsafe_democertificates: true +# plugins.security.allow_default_init_securityindex: true + +# plugins.security.authcz.admin_dn: +# - 'CN=A,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' +# plugins.security.nodes_dn: +# - 'CN=node1.dns.a-record,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' +# - 'CN=node2.dns.a-record,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' + +# plugins.security.audit.type: internal_opensearch +# plugins.security.enable_snapshot_restore_privilege: true +# plugins.security.check_snapshot_restore_write_privileges: true +# plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +# plugins.security.system_indices.enabled: true +# plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] +# node.max_local_storage_nodes: 3 +######## End OpenSearch Security Demo Configuration ######## + +plugins.ml_commons.only_run_on_ml_node: false \ No newline at end of file diff --git a/opensearch/register-model.sh b/opensearch/register-model.sh new file mode 100755 index 0000000..8d0be9a --- /dev/null +++ b/opensearch/register-model.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Define the API endpoint for uploading the model +UPLOAD_API_URL="http://localhost:9200/_plugins/_ml/models/_upload" + +# Define the JSON data for uploading the model +UPLOAD_JSON_DATA=' +{ + "name": "huggingface/sentence-transformers/all-MiniLM-L6-v2", + "version": "1.0.1", + "model_format": "TORCH_SCRIPT" +} +' + +# Define authentication credentials +USERNAME="admin" +PASSWORD="admin" + +# Send the POST request to upload the model and capture the response +UPLOAD_RESPONSE=$(curl -s -u "$USERNAME:$PASSWORD" -X POST "$UPLOAD_API_URL" -H 'Content-Type: application/json' -d "$UPLOAD_JSON_DATA") + +# Check if the upload request was successful +if [ $? -ne 0 ]; then + echo "Error: Model upload request failed." + exit 1 +fi + +# Extract task_id from the upload response +UPLOAD_TASK_ID=$(echo "$UPLOAD_RESPONSE" | jq -r '.task_id') + +# Check if the upload task_id is "null" or empty +if [ -z "$UPLOAD_TASK_ID" ] || [ "$UPLOAD_TASK_ID" = "null" ]; then + echo "Error: Upload task_id not found or is null in the response. $UPLOAD_RESPONSE" + exit 1 +fi + +echo "Model upload request successful. Uploaded task_id: $UPLOAD_TASK_ID" + +# Define the API endpoint for checking the status of the upload task +UPLOAD_STATUS_API_URL="http://localhost:9200/_plugins/_ml/tasks/$UPLOAD_TASK_ID" + +# Use a while loop to wait for the upload task "state" to become "COMPLETED" +while true; do + # Send the GET request to check the upload task status using curl in silent mode + UPLOAD_STATUS_RESPONSE=$(curl -s -u "$USERNAME:$PASSWORD" -X GET "$UPLOAD_STATUS_API_URL") + + # Check if the request for upload task status was successful + if [ $? -ne 0 ]; then + echo "Error in checking upload task status. Response:" + echo "$UPLOAD_STATUS_RESPONSE" + exit 1 + fi + + # Extract the upload task "state" from the response + UPLOAD_STATE=$(echo "$UPLOAD_STATUS_RESPONSE" | jq -r '.state') + + # Check if the upload task "state" is "COMPLETED" + if [ "$UPLOAD_STATE" = "COMPLETED" ]; then + MODEL_ID=$(echo "$UPLOAD_STATUS_RESPONSE" | jq -r '.model_id') + echo "Model upload completed successfully. Model ID: $MODEL_ID" + break + fi + if [ "$UPLOAD_STATE" = "FAILED" ]; then + REASON=$(echo "$UPLOAD_STATUS_RESPONSE" | jq -r '.error') + echo "Model upload failed: $REASON" + exit 1 + fi + + # Sleep for a while before checking the status again + sleep 1 +done + +# Define the API endpoint for deploying the model +DEPLOY_API_URL="http://localhost:9200/_plugins/_ml/models/$MODEL_ID/_deploy" + +# Send the POST request to deploy the model using curl in silent mode and capture the response +DEPLOY_RESPONSE=$(curl -s -u "$USERNAME:$PASSWORD" -X POST "$DEPLOY_API_URL") + +# Check if the deploy request was successful +if [ $? -ne 0 ]; then + echo "Error: Model deployment request failed." + exit 1 +fi + +# Extract task_id from the deploy response and store it in a variable +DEPLOY_TASK_ID=$(echo "$DEPLOY_RESPONSE" | jq -r '.task_id') + +# Check if the deploy task_id is "null" or empty +if [ -z "$DEPLOY_TASK_ID" ] || [ "$DEPLOY_TASK_ID" = "null" ]; then + echo "Error: Deploy task_id not found or is null in the response." + exit 1 +fi + +echo "Model deployment request successful. Deployed task_id: $DEPLOY_TASK_ID" + +# Define the API endpoint for checking the status of the deploy task +DEPLOY_STATUS_API_URL="http://localhost:9200/_plugins/_ml/tasks/$DEPLOY_TASK_ID" + +# Use a while loop to wait for the deploy task "state" to become "COMPLETED" +while true; do + # Send the GET request to check the deploy task status using curl in silent mode + DEPLOY_STATUS_RESPONSE=$(curl -s -u "$USERNAME:$PASSWORD" -X GET "$DEPLOY_STATUS_API_URL") + + # Check if the request for deploy task status was successful + if [ $? -ne 0 ]; then + echo "Error in checking deploy task status. Response:" + echo "$DEPLOY_STATUS_RESPONSE" + exit 1 + fi + + # Extract the deploy task "state" from the response + DEPLOY_STATE=$(echo "$DEPLOY_STATUS_RESPONSE" | jq -r '.state') + + # Check if the deploy task "state" is "COMPLETED" + if [ "$DEPLOY_STATE" = "COMPLETED" ]; then + echo "Model deployment completed successfully." + break + fi + + # Sleep for a while before checking the status again + sleep 1 +done + +DEPLOY_MODEL_ID=$(echo "$DEPLOY_STATUS_RESPONSE" | jq -r '.model_id') +echo "Deployed model ID: $DEPLOY_MODEL_ID" + +QUERY_TEMPLATE=' +POST _scripts/ecom_neural_search +{ + "script": { + "lang": "mustache", + "source": { + "query": { + "bool": { + "must": [ + { + "neural": { + "product_vector": { + "query_text": "{{query_text}}", + "model_id": "'$MODEL_ID'", + "k": 10 + } + } + } + ] + } + } + }, + "params": { + "query_text": "notebook" + } + } +}' + +SCRIPT_URL="http://localhost:9200/_scripts/ecom_neural_search" + +UPLOAD_RESPONSE=$(curl -s -u "$USERNAME:$PASSWORD" -X POST "$SCRIPT_URL" -H 'Content-Type: application/json' -d "$QUERY_TEMPLATE") + +# Check if the upload request was successful +if [ $? -ne 0 ]; then + echo "Error: Query template request failed." + exit 1 +fi + +echo "Query template ecom_neural_search created." diff --git a/elasticsearch/roles.yml b/opensearch/roles.yml similarity index 100% rename from elasticsearch/roles.yml rename to opensearch/roles.yml diff --git a/elasticsearch/schema.json b/opensearch/schema.json similarity index 70% rename from elasticsearch/schema.json rename to opensearch/schema.json index 42d420f..5ddf545 100644 --- a/elasticsearch/schema.json +++ b/opensearch/schema.json @@ -3,7 +3,9 @@ "index": { "number_of_shards": 1, "number_of_replicas": 0, - "mapping.total_fields.limit": 10000 + "mapping.total_fields.limit": 10000, + "knn": true, + "knn.algo_param.ef_search": 100 }, "analysis": { "filter": { @@ -89,16 +91,30 @@ "fielddata": true }, "product_vector": { - "type": "dense_vector", - "dims": 384, - "index": true, - "similarity": "cosine" + "type": "knn_vector", + "dimension": 384, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } }, "product_image_vector": { - "type": "dense_vector", - "dims": 768, - "index": true, - "similarity": "cosine" + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } }, "filter_brand": { "type": "keyword" @@ -112,8 +128,17 @@ "vector_attributes_768": { "match": "*_768", "mapping": { - "type": "dense_vector", - "dims": 768 + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } } } }, @@ -121,8 +146,17 @@ "vector_attributes_384": { "match": "*_384", "mapping": { - "type": "dense_vector", - "dims": 384 + "type": "knn_vector", + "dimension": 384, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } } } }, diff --git a/opensearch/wait-for-os.sh b/opensearch/wait-for-os.sh new file mode 100755 index 0000000..5e7a929 --- /dev/null +++ b/opensearch/wait-for-os.sh @@ -0,0 +1,3 @@ +DOT='\033[0;37m.\033[0m' +while [[ "$(curl -u 'admin:admin' -s localhost:9200/_cluster/health | jq '."status"')" != "\"green\"" ]]; do printf ${DOT}; sleep 5; done +echo "" diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index f485ca7..eea34c9 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -28,6 +28,6 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] - - job_name: 'elasticsearch' + - job_name: 'opensearch' static_configs: - - targets: ['elasticsearch-exporter:9114'] \ No newline at end of file + - targets: ['opensearch-exporter:9114'] diff --git a/quickstart.sh b/quickstart.sh index 52b2de9..2e67b64 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -16,10 +16,12 @@ if ! [ -x "$(command -v curl)" ]; then echo "${ERROR}Error: curl is not installed.${RESET}" >&2 exit 1 fi -if ! [ -x "$(command -v docker-compose)" ]; then - echo "${ERROR}Error: docker-compose is not installed.${RESET}" >&2 - exit 1 -fi +# todo: replace this with smart check on "docker-compose" vs "docker compose" +COMPOSE_CMD='docker compose' +# if ! [ -x "$(command -v docker-compose)" ]; then +# echo "${ERROR}Error: docker-compose is not installed.${RESET}" >&2 +# exit 1 +# fi if ! [ -x "$(command -v jq)" ]; then echo "${ERROR}Error: jq is not installed.${RESET}" >&2 exit 1 @@ -40,29 +42,30 @@ while [ ! $# -eq 0 ] do case "$1" in --help | -h) + echo -e "Use the option --with-vector-search | -vector to include Vector services in Chorus." echo -e "Use the option --with-offline-lab | -lab to include Quepid and RRE services in Chorus." - echo -e "Use the option --with-observability | -obs to include Grafana, Prometheus, and Elasticsearch Exporter services in Chorus." + echo -e "Use the option --with-observability | -obs to include Grafana, Prometheus, and OpenSearch Exporter services in Chorus." echo -e "Use the option --shutdown | -s to shutdown and remove the Docker containers and data." echo -e "Use the option --stop to stop the Docker containers." - echo -e "Use the option --online-deployment | -online to update configuration to run on chorus-es-edition.dev.o19s.com environment." + echo -e "Use the option --online-deployment | -online to update configuration to run on chorus-os-edition.dev.o19s.com environment." exit ;; - --with-observability | -obs) - observability=true - echo -e "${MAJOR}Running Chorus with observability services enabled\n${RESET}" - ;; +# --with-observability | -obs) +# observability=true +# echo -e "${MAJOR}Running Chorus with observability services enabled\n${RESET}" +# ;; --with-offline-lab | -lab) offline_lab=true echo -e "${MAJOR}Running Chorus with offline lab environment enabled\n${RESET}" ;; --online-deployment | -online) local_deploy=false - echo -e "${MAJOR}Configuring Chorus for chorus-es-edition.dev.o19s.com environment${RESET}" + echo -e "${MAJOR}Configuring Chorus for chorus-os-edition.dev.o19s.com environment${RESET}" ;; - --with-vector-search | -vector) - vector_search=true - echo -e "${MAJOR}Configuring Chorus with vector search services enabled${RESET}" - ;; + --with-vector-search | -vector) + vector_search=true + echo -e "${MAJOR}Configuring Chorus with vector search services enabled${RESET}" + ;; --shutdown | -s) shutdown=true echo -e "${MAJOR}Shutting down Chorus\n${RESET}" @@ -75,21 +78,22 @@ do shift done -services="elasticsearch kibana chorus-ui smui" -if $observability; then - services="${services} grafana elasticsearch-exporter" -fi +services="opensearch-node1 opensearch-node2 opensearch-dashboards chorus-ui" +#services="opensearch opensearch-dashboards chorus-ui smui" +#if $observability; then +# services="${services} grafana opensearch-exporter" +#fi if $offline_lab; then - services="${services} quepid rre keycloak" + services="${services} quepid keycloak" fi if ! $local_deploy; then echo -e "${MAJOR}Updating configuration files for online deploy${RESET}" - sed -i.bu 's/localhost:3000/chorus-es-edition.dev.o19s.com:3000/g' ./keycloak/realm-config/chorus-realm.json - sed -i.bu 's/keycloak:9080/chorus-es-edition.dev.o19s.com:9080/g' ./keycloak/wait-for-keycloak.sh - sed -i.bu 's/keycloak:9080/chorus-es-edition.dev.o19s.com:9080/g' ./docker-compose.yml - sed -i.bu 's/localhost:9200/chorus-es-edition.dev.o19s.com:9200/g' ./reactivesearch/src/App.js + sed -i.bu 's/localhost:3000/chorus-os-edition.dev.o19s.com:3000/g' ./keycloak/realm-config/chorus-realm.json + sed -i.bu 's/keycloak:9080/chorus-os-edition.dev.o19s.com:9080/g' ./keycloak/wait-for-keycloak.sh + sed -i.bu 's/keycloak:9080/chorus-os-edition.dev.o19s.com:9080/g' ./docker-compose.yml + sed -i.bu 's/localhost:9200/chorus-os-edition.dev.o19s.com:9200/g' ./reactivesearch/src/App.js fi if $vector_search; then @@ -98,50 +102,51 @@ if $vector_search; then if (( $docker_memory_allocated < 10737418240 )); then docker_memory_allocated_in_gb=$((docker_memory_allocated/1024/1024/1024)) - log_red "You have only ${docker_memory_allocated_in_gb} GB memory allocated to Docker, and you need at least 10GB for vectors demo." + echo -e "${ERROR}You have only ${docker_memory_allocated_in_gb} GB memory allocated to Docker, and you need at least 10GB for vectors demo.${RESET}" fi fi if $stop; then - services="${services} grafana elasticsearch-exporter quepid rre keycloak" - docker-compose stop ${services} + services="${services} grafana quepid keycloak" +# services="${services} grafana opensearch-exporter quepid rre keycloak" + ${COMPOSE_CMD} stop ${services} exit fi if $shutdown; then - docker-compose down -v + ${COMPOSE_CMD} down -v exit fi -docker-compose up -d --build ${services} - -echo -e "${MAJOR}Waiting for Elasticsearch to start up and be online.${RESET}" -./elasticsearch/wait-for-es.sh # Wait for Elasticsearch to be online - -echo -e "${MAJOR}Setting up an admin user to explore Elasticsearch and a role for anonymous access to run RRE offline tests.\n${RESET}" -curl -u 'elastic:ElasticRocks' -X POST "localhost:9200/_security/user/chorus_admin?pretty" -H 'Content-Type: application/json' -d' -{ - "password" : "password", - "roles" : ["superuser"] -} -' - -curl -u 'elastic:ElasticRocks' -X POST "localhost:9200/_security/role/anonymous_user" -H 'Content-Type: application/json' -d' -{ - "run_as": [ ], - "cluster": [ ], - "indices": [ - { - "names": [ "ecommerce" ], - "privileges": [ "read" ] - } - ] -} -' +${COMPOSE_CMD} up -d --build ${services} + +echo -e "${MAJOR}Waiting for OpenSearch to start up and be online.${RESET}" +./opensearch/wait-for-os.sh # Wait for OpenSearch to be online + +# echo -e "${MAJOR}Setting up an admin user to explore OpenSearch and a role for anonymous access to run RRE offline tests.\n${RESET}" +# curl -u 'admin:admin' -X POST "localhost:9200/_security/user/chorus_admin?pretty" -H 'Content-Type: application/json' -d' +# { +# "password" : "password", +# "roles" : ["superuser"] +# } +# ' + +# curl -u 'admin:admin' -X POST "localhost:9200/_security/role/anonymous_user" -H 'Content-Type: application/json' -d' +# { +# "run_as": [ ], +# "cluster": [ ], +# "indices": [ +# { +# "names": [ "ecommerce" ], +# "privileges": [ "read" ] +# } +# ] +# } +# ' echo -e "${MAJOR}Creating ecommerce index, defining its mapping & settings\n${RESET}" -curl -u 'elastic:ElasticRocks' -s -X PUT "localhost:9200/ecommerce/" -H 'Content-Type: application/json' --data-binary @./elasticsearch/schema.json +curl -u 'admin:admin' -s -X PUT "localhost:9200/ecommerce/" -H 'Content-Type: application/json' --data-binary @./opensearch/schema.json if $vector_search; then # Populating product data for vector search @@ -176,88 +181,88 @@ else done fi echo -e "${MAJOR}Indexing the sample product data, please wait...\n${RESET}" - curl -u 'elastic:ElasticRocks' -s -X POST "localhost:9200/ecommerce/_bulk?pretty" -H 'Content-Type: application/json' --data-binary @transformed_data.json + curl -u 'admin:admin' -s -X POST "localhost:9200/ecommerce/_bulk?pretty" -H 'Content-Type: application/json' --data-binary @transformed_data.json fi -echo -e "${MAJOR}Creating default (empty) query rewriters\n${RESET}" -# Query rewriters are configured empty, without any rules. This ensures that no errors occur when the different -# relevance algorithms are used in the frontend before any rules are set. -# For configuring the rules SMUI will be set up and used. -curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/common_rules' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "class": "querqy.elasticsearch.rewriter.SimpleCommonRulesRewriterFactory", - "config": { - "rules": "" - } -}' - -curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/common_rules_prelive' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "class": "querqy.elasticsearch.rewriter.SimpleCommonRulesRewriterFactory", - "config": { - "rules": "" - } -}' - -curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "class": "querqy.elasticsearch.rewriter.ReplaceRewriterFactory", - "config": { - "rules": "" - } -}' - -curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace_prelive' \ ---header 'Content-Type: application/json' \ ---data-raw '{ - "class": "querqy.elasticsearch.rewriter.ReplaceRewriterFactory", - "config": { - "rules": "" - } -}' - -if $vector_search; then - echo "Creating embedding rewriters." - - curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/embtxt' \ - --header 'Content-Type: application/json' \ - --data-raw '{ - "class": "querqy.elasticsearch.rewriter.EmbeddingsRewriterFactory", - "config": { - "model" : { - "class": "querqy.elasticsearch.rewriter.ChorusEmbeddingModel", - "url": "http://embeddings:8000/minilm/text/", - "normalize": false, - "cache" : "embeddings" - } - } - }' - - curl -u 'elastic:ElasticRocks' -s --request PUT 'http://localhost:9200/_querqy/rewriter/embimg' \ - --header 'Content-Type: application/json' \ - --data-raw '{ - "class": "querqy.elasticsearch.rewriter.EmbeddingsRewriterFactory", - "config": { - "model" : { - "class": "querqy.elasticsearch.rewriter.ChorusEmbeddingModel", - "url": "http://embeddings:8000/clip/text/", - "normalize": false, - "cache" : "embeddings" - } - } - }' -fi - -echo -e "${MAJOR}Setting up SMUI\n${RESET}" -while [ $(curl -s http://localhost:9000/api/v1/solr-index | wc -c) -lt 2 ]; do - echo "Waiting 5s for SMUI to be ready..." - sleep 5 -done -curl -X PUT -H "Content-Type: application/json" -d '{"name":"ecommerce", "description":"Chorus Webshop"}' http://localhost:9000/api/v1/solr-index +# echo -e "${MAJOR}Creating default (empty) query rewriters\n${RESET}" +# # Query rewriters are configured empty, without any rules. This ensures that no errors occur when the different +# # relevance algorithms are used in the frontend before any rules are set. +# # For configuring the rules SMUI will be set up and used. +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/common_rules' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.SimpleCommonRulesRewriterFactory", +# "config": { +# "rules": "" +# } +# }' + +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/common_rules_prelive' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.SimpleCommonRulesRewriterFactory", +# "config": { +# "rules": "" +# } +# }' + +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.ReplaceRewriterFactory", +# "config": { +# "rules": "" +# } +# }' + +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace_prelive' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.ReplaceRewriterFactory", +# "config": { +# "rules": "" +# } +# }' + +# if $vector_search; then +# echo "Creating embedding rewriters." + +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/embtxt' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.EmbeddingsRewriterFactory", +# "config": { +# "model" : { +# "class": "querqy.opensearch.rewriter.ChorusEmbeddingModel", +# "url": "http://embeddings:8000/minilm/text/", +# "normalize": false, +# "cache" : "embeddings" +# } +# } +# }' + +# curl -u 'admin:admin' -s --request PUT 'http://localhost:9200/_querqy/rewriter/embimg' \ +# --header 'Content-Type: application/json' \ +# --data-raw '{ +# "class": "querqy.opensearch.rewriter.EmbeddingsRewriterFactory", +# "config": { +# "model" : { +# "class": "querqy.opensearch.rewriter.ChorusEmbeddingModel", +# "url": "http://embeddings:8000/clip/text/", +# "normalize": false, +# "cache" : "embeddings" +# } +# } +# }' +# fi + +#echo -e "${MAJOR}Setting up SMUI\n${RESET}" +#while [ $(curl -s http://localhost:9000/api/v1/solr-index | wc -c) -lt 2 ]; do +# echo "Waiting 5s for SMUI to be ready..." +# sleep 5 +#done +#curl -X PUT -H "Content-Type: application/json" -d '{"name":"ecommerce", "description":"Chorus Webshop"}' http://localhost:9000/api/v1/solr-index if $offline_lab; then if $local_deploy; then @@ -268,12 +273,8 @@ if $offline_lab; then echo -e "${MAJOR}Setting up Quepid${RESET}" - docker-compose run --rm quepid bin/rake db:setup - docker-compose run quepid thor user:create -a admin@choruselectronics.com "Chorus Admin" password - - echo -e "${MAJOR}Setting up RRE\n${RESET}" - docker-compose run rre mvn rre:evaluate - docker-compose run rre mvn rre-report:report + ${COMPOSE_CMD} run --rm quepid bin/rake db:setup + ${COMPOSE_CMD} run quepid thor user:create -a admin@choruselectronics.com "Chorus Admin" password fi if $observability; then @@ -283,10 +284,10 @@ if $observability; then curl -u admin:password -S -X POST -H "Content-Type: application/json" http://localhost:9091/api/users/2/using/1 fi -if $vector_search; then - echo "Setting up Embeddings service" - docker-compose up -d --build embeddings - ./embeddings/wait-for-api.sh -fi +#if $vector_search; then +# echo "Setting up Embeddings service" +# ${COMPOSE_CMD} up -d --build embeddings +# ./embeddings/wait-for-api.sh +#fi -echo -e "${MAJOR}Welcome to Chorus ES Edition!${RESET}" +echo -e "${MAJOR}Welcome to Chorus OS Edition!${RESET}" diff --git a/reactivesearch/src/App.js b/reactivesearch/src/App.js index 42bb0bb..4293fa8 100644 --- a/reactivesearch/src/App.js +++ b/reactivesearch/src/App.js @@ -17,7 +17,7 @@ class App extends Component {
@@ -84,6 +84,23 @@ class App extends Component { } } } + } else if (algo === "neural_minilm") { + return { + "query": { + "bool": { + "must": [ + { + "neural": { + "product_vector": { + "query_text": value, + "model_id": "IZVRuYoB9pn48M2pKtCX", + "k": 10 + } + } + } + ] + } + } } } else if (algo === "querqy_preview") { return { query: { diff --git a/reactivesearch/src/custom/AlgoPicker.js b/reactivesearch/src/custom/AlgoPicker.js index b847758..56f9ef7 100644 --- a/reactivesearch/src/custom/AlgoPicker.js +++ b/reactivesearch/src/custom/AlgoPicker.js @@ -29,6 +29,7 @@ class AlgoPicker extends Component { )}