From ee7b0789f85578d19a1f8baf336068a28f5eaf38 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Thu, 26 Feb 2026 17:31:48 -0300 Subject: [PATCH 1/5] fix: prevent memory leaks in search query and response lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract scroll_id before the loop in scroll_hits to ensure ES scroll contexts are always cleared, even if yield raises on the first iteration - Break Response → Query circular reference by storing query_definition instead of the full Query object, allowing GC to collect responses - Make Query#reset! public so apps can clear cached responses - Use lightweight query_definition in event payloads instead of retaining the full Query object reference --- lib/esse/search/query.rb | 16 ++++++++-------- lib/esse/search/response.rb | 6 +++--- spec/esse/search/query_spec.rb | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/esse/search/query.rb b/lib/esse/search/query.rb index b9a1988f..f89879ba 100644 --- a/lib/esse/search/query.rb +++ b/lib/esse/search/query.rb @@ -40,7 +40,7 @@ def results def scroll_hits(batch_size: 1_000, scroll: '1m') response = execute_search_query!(size: batch_size, scroll: scroll) - scroll_id = nil + scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] fetched = 0 total = response.total @@ -48,9 +48,9 @@ def scroll_hits(batch_size: 1_000, scroll: '1m') fetched += response.hits.size yield(response.hits) if response.hits.any? break if fetched >= total - scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] break unless scroll_id response = execute_scroll_query(scroll: scroll, scroll_id: scroll_id) + scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] end ensure begin @@ -83,12 +83,16 @@ def search_after_hits(batch_size: 1_000) end end + def reset! + @response = nil + end + private def execute_search_query!(**execution_options) resp, err = nil Esse::Events.instrument('elasticsearch.execute_search_query') do |payload| - payload[:query] = self + payload[:query_definition] = definition begin resp = Response.new(self, transport.search(**definition, **execution_options)) rescue => e @@ -105,7 +109,7 @@ def execute_search_query!(**execution_options) def execute_scroll_query(scroll:, scroll_id:) resp, err = nil Esse::Events.instrument('elasticsearch.execute_search_query') do |payload| - payload[:query] = self + payload[:query_definition] = definition begin resp = Response.new(self, transport.scroll(scroll: scroll, body: { scroll_id: scroll_id })) rescue => e @@ -118,10 +122,6 @@ def execute_scroll_query(scroll:, scroll_id:) resp end - - def reset! - @response = nil - end end end end diff --git a/lib/esse/search/response.rb b/lib/esse/search/response.rb index a006d8ef..a620e7f6 100644 --- a/lib/esse/search/response.rb +++ b/lib/esse/search/response.rb @@ -7,13 +7,13 @@ class Response extend Forwardable def_delegators :hits, :each, :size, :empty? - attr_reader :query, :raw_response, :options + attr_reader :query_definition, :raw_response, :options - # @param [Esse::Search::Query] query The search query + # @param [Esse::Search::Query, Hash] query The search query or its definition hash # @param [Hash] raw_response The raw response from Elasticsearch # @param [Hash] options The options passed to the search def initialize(query, raw_response, **options) - @query = query + @query_definition = query.is_a?(Hash) ? query : query.definition @raw_response = raw_response @options = options end diff --git a/spec/esse/search/query_spec.rb b/spec/esse/search/query_spec.rb index 8af2ddd7..af0133ea 100644 --- a/spec/esse/search/query_spec.rb +++ b/spec/esse/search/query_spec.rb @@ -107,7 +107,7 @@ expect(transport).to receive(:search).with(index: 'events', body: request_body).and_return(body) expect(resp = query.response).to be_an_instance_of(Esse::Search::Response) - expect(resp.query).to eq(query) + expect(resp.query_definition).to eq(query.definition) end end @@ -124,7 +124,7 @@ expect(transport).to receive(:search).with(index: 'events', body: request_body).and_return(response_body) expect(query.response).to be_an_instance_of(Esse::Search::Response) - assert_event 'elasticsearch.execute_search_query', { query: query, response: query.response } + assert_event 'elasticsearch.execute_search_query', { query_definition: query.definition, response: query.response } end end @@ -138,7 +138,7 @@ expect { query.response }.to raise_error(Esse::Transport::BadRequestError) - assert_event 'elasticsearch.execute_search_query', { query: query, error: exception } + assert_event 'elasticsearch.execute_search_query', { query_definition: query.definition, error: exception } end end end From 64497d284bc60cce5033ccb99753cdd0bf38effb Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Wed, 18 Mar 2026 16:08:18 -0300 Subject: [PATCH 2/5] chore: bump gem version to 0.4.0 --- gemfiles/Gemfile.elasticsearch-1.x.lock | 4 ++-- gemfiles/Gemfile.elasticsearch-2.x.lock | 4 ++-- gemfiles/Gemfile.elasticsearch-5.x.lock | 4 ++-- gemfiles/Gemfile.elasticsearch-6.x.lock | 4 ++-- gemfiles/Gemfile.elasticsearch-7.x.lock | 4 ++-- gemfiles/Gemfile.elasticsearch-8.x.lock | 4 ++-- gemfiles/Gemfile.opensearch-1.x.lock | 4 ++-- gemfiles/Gemfile.opensearch-2.x.lock | 4 ++-- lib/esse/version.rb | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/gemfiles/Gemfile.elasticsearch-1.x.lock b/gemfiles/Gemfile.elasticsearch-1.x.lock index e564515d..6ce03736 100644 --- a/gemfiles/Gemfile.elasticsearch-1.x.lock +++ b/gemfiles/Gemfile.elasticsearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -95,7 +95,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-2.x.lock b/gemfiles/Gemfile.elasticsearch-2.x.lock index 97157b25..4f34a3be 100644 --- a/gemfiles/Gemfile.elasticsearch-2.x.lock +++ b/gemfiles/Gemfile.elasticsearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-5.x.lock b/gemfiles/Gemfile.elasticsearch-5.x.lock index 05eb5bb8..4e75c3bb 100644 --- a/gemfiles/Gemfile.elasticsearch-5.x.lock +++ b/gemfiles/Gemfile.elasticsearch-5.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-6.x.lock b/gemfiles/Gemfile.elasticsearch-6.x.lock index 6d0f6684..0c71983a 100644 --- a/gemfiles/Gemfile.elasticsearch-6.x.lock +++ b/gemfiles/Gemfile.elasticsearch-6.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -117,7 +117,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-7.x.lock b/gemfiles/Gemfile.elasticsearch-7.x.lock index 3e02bff6..7bbfe225 100644 --- a/gemfiles/Gemfile.elasticsearch-7.x.lock +++ b/gemfiles/Gemfile.elasticsearch-7.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -100,7 +100,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-8.x.lock b/gemfiles/Gemfile.elasticsearch-8.x.lock index b32833a4..ac0a47f2 100644 --- a/gemfiles/Gemfile.elasticsearch-8.x.lock +++ b/gemfiles/Gemfile.elasticsearch-8.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.opensearch-1.x.lock b/gemfiles/Gemfile.opensearch-1.x.lock index a282670d..80d7db92 100644 --- a/gemfiles/Gemfile.opensearch-1.x.lock +++ b/gemfiles/Gemfile.opensearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.opensearch-2.x.lock b/gemfiles/Gemfile.opensearch-2.x.lock index 98539ae7..77664c30 100644 --- a/gemfiles/Gemfile.opensearch-2.x.lock +++ b/gemfiles/Gemfile.opensearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/lib/esse/version.rb b/lib/esse/version.rb index 8a54a97d..882759d2 100644 --- a/lib/esse/version.rb +++ b/lib/esse/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Esse - VERSION = '0.4.0.rc5' + VERSION = '0.4.0' end From 9dc0d43e3b0699c3d19511993af0c3d7a94264b6 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Wed, 18 Mar 2026 16:28:42 -0300 Subject: [PATCH 3/5] drop compatibility with ruby older than 2.7 --- .github/workflows/build.yml | 15 - .travis.yml | 47 --- Gemfile.lock | 6 +- README.old | 420 ------------------------ bin/setup | 16 +- esse.gemspec | 2 +- gemfiles/Gemfile.elasticsearch-1.x.lock | 2 +- gemfiles/Gemfile.elasticsearch-2.x.lock | 2 +- gemfiles/Gemfile.elasticsearch-5.x.lock | 2 +- gemfiles/Gemfile.elasticsearch-6.x.lock | 2 +- gemfiles/Gemfile.elasticsearch-7.x.lock | 2 +- gemfiles/Gemfile.elasticsearch-8.x.lock | 2 +- gemfiles/Gemfile.opensearch-1.x.lock | 2 +- gemfiles/Gemfile.opensearch-2.x.lock | 2 +- 14 files changed, 26 insertions(+), 496 deletions(-) delete mode 100644 .travis.yml delete mode 100644 README.old diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 056ebc10..d4b71af3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,21 +6,6 @@ jobs: fail-fast: false matrix: include: - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-1.x - stack: elasticsearch:1.7.6 - stack_version: "1.7.6" - allow_failure: true - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-2.x - stack: elasticsearch:2.4.6 - stack_version: "2.4.6" - allow_failure: true - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-5.x - stack: elasticsearch:5.6.16 - stack_version: "5.6.16" - allow_failure: true - ruby: 2.7.7 gemfile: gemfiles/Gemfile.elasticsearch-6.x stack: elasticsearch:6.8.16 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 065aabca..00000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -# dist: trusty -language: ruby -before_install: - - gem --version - - gem update bundler - - gem --version - - bash gemfiles/setup.sh - -jobs: - fast_finish: true - include: - - rvm: 2.6 - env: STACK_VERSION=1.7.6 DOCKER_IMAGE=elasticsearch - gemfile: gemfiles/Gemfile.elasticsearch-1.x - - rvm: 2.6 - env: STACK_VERSION=2.3 DOCKER_IMAGE=elasticsearch - gemfile: gemfiles/Gemfile.elasticsearch-2.x - - rvm: 2.6 - env: STACK_VERSION=5.6.16 - gemfile: gemfiles/Gemfile.elasticsearch-5.x - - rvm: 2.6 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 2.6 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.7 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 2.7 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.7 - env: STACK_VERSION=8.2.1 - gemfile: gemfiles/Gemfile.elasticsearch-8.x - - rvm: 3.0 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 3.0 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.6 - env: STACK_VERSION=1.3.3 DOCKER_IMAGE=opensearchproject/opensearch - gemfile: gemfiles/Gemfile.opensearch-2.x - - rvm: 2.6 - env: STACK_VERSION=2.19.3 DOCKER_IMAGE=opensearchproject/opensearch - gemfile: gemfiles/Gemfile.opensearch-2.x diff --git a/Gemfile.lock b/Gemfile.lock index c17452df..542de95e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -37,7 +37,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -84,7 +84,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/README.old b/README.old deleted file mode 100644 index 82dd2805..00000000 --- a/README.old +++ /dev/null @@ -1,420 +0,0 @@ -![esse-red](https://user-images.githubusercontent.com/18994/186032704-f1c9ce86-a41a-41ae-a224-30f4b382c012.png) - - -# esse - -**This project is under development and may suffer constant structural changes. I don't recommend using it right now** - -Simple and efficient way to organize queries/mapping/indices/tasks based on the official elasticsearch-ruby. - -## Why to use it? -Some facts to use this library: - -### Don't spend time learning our DLS -You don't need to spend time learning our DSL or gem usage to start using it. All you need know is the elasticsearch syntax. You are free to build your queries/mappings/settings using JSON/RubyHash flexibility. And keeping simple any elasticsearch upgrade and its syntax changes. - -### Multiple ElasticSearch Versions -You can use multiple elasticsearch servers with different versions in an elegant way. Take a look at [LINK TO TOPIC](#anchors-id-here) for more details. - -### It's pure Ruby -Yeah!! Nor [activesupport](http://github.com/rails/rails/tree/master/activesupport) dependency and all its monkey patchings. But if you are using rails, suggest install `esse-rails` extension that makes things even easier. Use the [Get started with esse-rails](#anchors-id-here) for more details. - -## Installation - -Add this line to your application's Gemfile: - -```ruby -gem 'esse' -``` - -And then execute: - - $ bundle install - -Or install it yourself as: - - $ gem install esse - -## Usage - -### Configuration - -Example of gem configuration using the `configure` method: -```ruby -Esse.configure do |config| - config.indices_directory = 'app/indices' - config.cluster do |cluster| - cluster.index_prefix = 'illinois' - cluster.settings = { - index: { - number_of_shards: 4, - number_of_replicas: 1, - }, - analysis: { - analyzer: { - esse_index: { - type: "custom", - char_filter: ["ampersand"], - filter: [ - "lowercase", - "asciifolding", - "esse_english_stop", - "esse_index_shingle", - "esse_stemmer" - ], - tokenizer: "standard" - }, - }, - filter: { - esse_index_shingle: { - token_separator: "", - type: "shingle" - }, - esse_stemmer: { - type: "stemmer", - language: "English" - }, - }, - char_filter: { - ampersand: { - type: "mapping", - mappings: ["&=> and "] - } - } - } - } - cluster.mappings = { - dynamic_templates: [ - { - esse_string_template: { - match: "*", - match_mapping_type: "string", - mapping: { - fields: { - analyzed: { - analyzer: "esse_index", - index: true, - type: "text" - } - }, - ignore_above: 1024, - type: "keyword" - } - } - } - ] - } - cluster.client = Elasticsearch::Client.new(host: 'http://localhost:9200') - end -end -``` - -Note that that if the cluster is configured without any identifier, it will be used as the `:default` cluster. - -```ruby -Esse.config.cluster.settings -# => {index: {number_of_shards: 4, number_of_replicas: 1} } - -# or -Esse.config.cluster(:default).settings -# => {index: {number_of_shards: 4, number_of_replicas: 1} } -``` - -You may also configure multiple cluster connections by specifying the identifier like the example below: -```ruby -Esse.configure do |config| - config.indices_directory = 'app/indices' - config.cluster(:il) do |cluster| - cluster.index_prefix = 'illinois' - cluster.settings = { - index: { - number_of_shards: 4, - number_of_replicas: 1, - } - } - cluster.client = Elasticsearch::Client.new(host: 'https://illinois:9200') - end - config.cluster(:fl) do |cluster| - cluster.index_prefix = 'florida' - cluster.settings = { - index: { - number_of_shards: 2, - number_of_replicas: 2, - } - } - cluster.client = Elasticsearch::Client.new(host: 'https://florida:9200') - end -end -``` - -And on the index you can use the `:il` or `:fl` identifier to specify which cluster you want to use. - -```ruby -class Illinois::AccountsIndex < Esse::Index - self.cluster_id = :il - ... -end -class Florida::AccountsIndex < Esse::Index - self.cluster_id = :fl -end -``` - -You can also configure it through the YAML file. -```ruby -Esse.config.load('./config/esse.yml') - -# or -Esse.configure do |config| - config.load('./config/esse.yml') - config.indices_directory = 'override/indices/directory/from/yml' -end -``` - -And the `config/esse.yml` file should look like: - -```yaml -indices_directory: "app/indices" -clusters: - il: - index_prefix: "illinois" - settings: - index: - number_of_shards: 4 - number_of_replicas: 1 - client: - host: "https://illinois:9200" - fl: - index_prefix: "florida" - settings: - index: - number_of_shards: 2 - number_of_replicas: 2 - client: - host: "https://florida:9200" -``` - -## Indices - -### Single type per index - -The mapping of elasticsearch 1.x, 2.x and 5.x allow multiple types. The single-type-per-index behaviour was introduction on es 5.x disabled as default. After es 6.x de single type per index is enabled by default but the explicit mapping is still required by using the `_doc` type. Es 7.x and above the `_doc` was totally removed and the mapping type is no longer required. The `esse` framework will automatically detect the version of elasticsearch and use the correct mapping type definition. But you can enforce the mapping type by using the `mapping_single_type` option in the `index`(Just make sure the server support it and it's configured accordingly). See [this article](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html) for more details. - -```ruby -class MyIndex < Esse::Index - self.mapping_single_type = true # This only control the mapping format. Add the setting `index.mapping.single_type: true` to the index settings in elasticsearch as well if needed. -end -``` - -## CLI - -The `esse` command line tool is available to manage indices and tasks. It's configured to load the configuration from the `Essefile`, `config/esse.rb` or `config/initializers/esse.rb` files(in that order). - -```bash -Commands: - esse --version, -v # Show package version - esse generate SUBCOMMAND ...ARGS # Run generators - esse help [COMMAND] # Describe available commands or one specific command - esse index SUBCOMMAND ...ARGS # Manage indices - esse install # Generate boilerplate configuration files - -Options: - -r, [--require=REQUIRE] # Require config file where the application is defined - -s, [--silent], [--no-silent] # Silent mode - -h, [--help], -?, [--usage] # Show help -``` - -### Generate Index - -Generate an index with the following command: - -```bash -❯ esse generate index <*doc_type> -``` - -List of types are optional. If not specified, the index will be created with definition on Index level with the `"default"` as type. Example: - -```bash -❯ bundle exec esse generate index GeosIndex - create app/indices/geos_index.rb -``` - -or with multiple datasources as document types: - -```bash -❯ bundle exec esse generate index GeosIndex state city - create app/indices/geos_index.rb -``` - -As default, the index will be create d with the collection, serializer, settings, mapping directly to the index class. By you can also specify custom arguments to better organize your code by splitting the it into multiple files. - -```bash -❯ ./exec/esse generate index GeosIndex state city --settings --mappings --documents --collections - create app/indices/geos_index.rb - create app/indices/geos_index/templates/settings.json - create app/indices/geos_index/templates/mappings.json - create app/indices/geos_index/documents/state_document.rb - create app/indices/geos_index/collections/state_collection.rb - create app/indices/geos_index/documents/city_document.rb - create app/indices/geos_index/collections/city_collection.rb -``` - -### Index Commands - -There are several commands to manage indices. The following commands are available: - -```bash -❯ bundle exec esse index [ruby-2.6.9p207] -Commands: - esse index close *INDEX_CLASS # Close an index (keep the data on disk, but deny operations with the index). - esse index create *INDEX_CLASSES # Creates indices for the given classes - esse index delete *INDEX_CLASSES # Deletes indices for the given classes - esse index help [COMMAND] # Describe subcommands or one specific subcommand - esse index import *INDEX_CLASSES --context=key:value # Import documents from the given classes - esse index open *INDEX_CLASS # Open a previously closed index. - esse index reset *INDEX_CLASSES # Performs zero-downtime index resetting. - esse index update_aliases *INDEX_CLASS # Replaces all existing aliases by the given suffix - esse index update_mapping *INDEX_CLASS # Create or update a mapping - esse index update_settings *INDEX_CLASS # Closes the index for read/write operations, updates the index settings, and open it again -``` - - -## Search - -Searching is done through the `search` method on the index class. It returns an instance of `Esse::Search::Query` that can be used to build the query and retrieve the results. - -```ruby -# Searching using Lucene query syntax -> query = GeosIndex.search('*', size: 10, offset: 0) -=> #10, :offset=>0, :q=>"*", :index=>"esse_console_geos"}> - - -# Searching using DSL -> query = GeosIndex.search(body: {query: {match: {name: 'Illinois'}}}, size: 1) -=> #{:query=>{:match=>{:name=>"Illinois"}}}, :size=>1, :index=>"esse_console_geos"}> - -# Retrieve response hits -> query.results -=> [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"IL", "_score"=>2.5433555, "_routing"=>"IL", "_source"=>{"id"=>"IL", "name"=>"Illinois"}}] -> query = GeosIndex.search(body: {query: {match: {"name.analyzed" => 'Illinois'}}}, _source: false) -=> #{:query=>{:match=>{"name.analyzed"=>"Illinois"}}}, :_source=>false, :index=>"esse_console_geos"} -> query.results -=> [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"IL", "_score"=>2.5433555, "_routing"=>"IL"} -# Retrieve response -> query.response -=> # -``` - -Search aggregations can be retrieved using the `aggregations` method from query response. - -```ruby -> query = GeosIndex.search(body: {query: { match_all: {}}, aggregations: { names: { terms: { field: "name", size: 2 }}}}, size: 2) -=> #{:query=>{:match_all=>{}}, :aggregations=>{:names=>{:terms=>{:field=>"name", :size=>2}}}}, :size=>2, :index=>"esse_console_geos"}> -> query.response.aggregations -=> {"names"=>{"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>14, "buckets"=>[{"key"=>"Alabama", "doc_count"=>1}, {"key"=>"Alaska", "doc_count"=>1}]}} - -# or just use the raw response -> query.response.raw_response -=> {"took"=>1, - "timed_out"=>false, - "_shards"=>{"total"=>1, "successful"=>1, "skipped"=>0, "failed"=>0}, - "hits"=> - {"total"=>{"value"=>16, "relation"=>"eq"}, - "max_score"=>1.0, - "hits"=> - [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"AL", "_score"=>1.0, "_routing"=>"AL", "_source"=>{"id"=>"AL", "name"=>"Alabama"}}, - {"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"AK", "_score"=>1.0, "_routing"=>"AK", "_source"=>{"id"=>"AK", "name"=>"Alaska"}}]}, - "aggregations"=>{"names"=>{"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>14, "buckets"=>[{"key"=>"Alabama", "doc_count"=>1}, {"key"=>"Alaska", "doc_count"=>1}]}}} -> query.response.total -=> 16 -``` - -Scroll queries can be performed using the `scroll_hits` method on the query. - -```ruby -> query.scroll_hits(batch_size: 6, scroll: '1m') { |hits| puts hits.size } -6 -6 -4 -=> nil -``` - -Searching across multiple indices can be done using the `search` method on the cluster. - -Using string indices as arguments: - -```ruby -> Esse.config.cluster.search('esse_*', body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_*"}> -> Esse.config.cluster.search('esse_cities_index', 'esse_counties_index', body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_cities_index,esse_counties_index"}> -``` - -Using index classes as arguments: - -```ruby -> Esse.config.cluster.search(CitiesIndex, CountiesIndex, body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_cities_index,esse_counties_index"}> -``` - -## Development - -After checking out the repo, run `bin/setup` to install dependencies. The command will dependencies of all `./gemfiles/Gemfile.*`. - -You can use `docker-compose` to start elasticsearch and opensearch services. - -```bash -❯ docker-compose up -d -``` - -```bash -❯ docker-compose ps -NAME COMMAND SERVICE STATUS PORTS -esse-elasticsearch-v1-1 "/docker-entrypoint.…" elasticsearch-v1 running 9300/tcp, 0.0.0.0:9201->9200/tcp, :::9201->9200/tcp -esse-elasticsearch-v2-1 "/docker-entrypoint.…" elasticsearch-v2 running 9300/tcp, 0.0.0.0:9202->9200/tcp, :::9202->9200/tcp -esse-elasticsearch-v5-1 "/bin/bash bin/es-do…" elasticsearch-v5 running 9300/tcp, 0.0.0.0:9205->9200/tcp, :::9205->9200/tcp -esse-elasticsearch-v6-1 "/usr/local/bin/dock…" elasticsearch-v6 running 9300/tcp, 0.0.0.0:9206->9200/tcp, :::9206->9200/tcp -esse-elasticsearch-v7-1 "/bin/tini -- /usr/l…" elasticsearch-v7 running 9300/tcp, 0.0.0.0:9207->9200/tcp, :::9207->9200/tcp -esse-elasticsearch-v8-1 "/bin/tini -- /usr/l…" elasticsearch-v8 running 9300/tcp, 0.0.0.0:9208->9200/tcp, :::9208->9200/tcp -esse-opensearch-v1-1 "./opensearch-docker…" opensearch-v1 running 9300/tcp, 9600/tcp, 9650/tcp, 0.0.0.0:19201->9200/tcp, :::19201->9200/tcp -esse-opensearch-v2-1 "./opensearch-docker…" opensearch-v2 running 9300/tcp, 9600/tcp, 9650/tcp, 0.0.0.0:19202->9200/tcp, :::19202->9200/tcp -``` -As you can see each elasticsearch or opensearch version is binded to a different port. - -Tests and console uses the `ESSE_URL` along with `STACK_VERSION` and `BUNDLE_GEMFILE` to determine which elasticsearch/opensearch and Gemfile with the official ruby client to use: - -```bash -❯ export ESSE_URL=http://localhost:9201 -❯ export STACK_VERSION=7.13.2 -❯ export BUNDLE_GEMFILE=gemfiles/Gemfile.v7 -``` - -But we have `bin/run` script to simplify this process. It will automatically set the required envionment variables according to the given arguments. - -```bash -❯ bin/run elasticsearch v7 ./bin/console -> ENV.values_at('ESSE_URL', 'STACK_VERSION', 'BUNDLE_GEMFILE') -=> ["http://localhost:9207", "7.13.2", "gemfiles/Gemfile.elasticsearch-7.x"] -> Esse.config.cluster.api.health.fetch('cluster_name') -=> "cluster-elasticsearch-v7" -``` - -And to run the tests: - -```bash -❯ bin/run elasticsearch v7 bundle exec rspec -``` - -If you don't have elasticsearch running and want to ignore integratino tests, you can use the `STUB_STACK=-` environment variable to stup the test suite for a specific elasticsearch version. Note that all examples with `:es_version` meta data will be skipped. - -```bash -❯ STUB_STACK=elasticsearch-7.0.3 bundle exec --gemfile gemfiles/Gemfile.elasticsearch-7.x rspec -``` - -## Contributing - -Bug reports and pull requests are welcome on GitHub at https://github.com/marcosgz/esse. - - -## License - -The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/bin/setup b/bin/setup index 282b9397..7252bf77 100755 --- a/bin/setup +++ b/bin/setup @@ -3,5 +3,17 @@ set -euo pipefail IFS=$'\n\t' set -vx -find gemfiles -type f \( -iname "Gemfile.elasticsearch-*" ! -iname "*.lock" \) -exec bundle install --gemfile {} \; -find gemfiles -type f \( -iname "Gemfile.opensearch-*" ! -iname "*.lock" \) -exec bundle install --gemfile {} \; +RUBY_IMAGE="ruby:2.7" +BUNDLER_VERSION="2.3.26" + +docker run --rm -v "$(pwd)":/app -w /app "$RUBY_IMAGE" \ + bash -c "gem install bundler:$BUNDLER_VERSION --no-document && bundle install" + +for gemfile in gemfiles/Gemfile.elasticsearch-*.x gemfiles/Gemfile.opensearch-*.x; do + [ -f "$gemfile" ] || continue + [[ "$gemfile" == *.lock ]] && continue + docker run --rm -v "$(pwd)":/app -w /app \ + -e BUNDLE_GEMFILE="$gemfile" \ + "$RUBY_IMAGE" \ + bash -c "gem install bundler:$BUNDLER_VERSION --no-document && bundle install" +done diff --git a/esse.gemspec b/esse.gemspec index 564f953a..b0cfd7f6 100644 --- a/esse.gemspec +++ b/esse.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| 'which means you can use it with any Ruby framework or even without any framework at all.' spec.homepage = 'https://github.com/marcosgz/esse' spec.license = 'MIT' - spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0') + spec.required_ruby_version = Gem::Requirement.new('>= 2.7') spec.metadata['homepage_uri'] = spec.homepage spec.metadata['source_code_uri'] = 'https://github.com/marcosgz/esse' diff --git a/gemfiles/Gemfile.elasticsearch-1.x.lock b/gemfiles/Gemfile.elasticsearch-1.x.lock index 6ce03736..ce3971ec 100644 --- a/gemfiles/Gemfile.elasticsearch-1.x.lock +++ b/gemfiles/Gemfile.elasticsearch-1.x.lock @@ -48,7 +48,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.elasticsearch-2.x.lock b/gemfiles/Gemfile.elasticsearch-2.x.lock index 4f34a3be..2f6433a6 100644 --- a/gemfiles/Gemfile.elasticsearch-2.x.lock +++ b/gemfiles/Gemfile.elasticsearch-2.x.lock @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.elasticsearch-5.x.lock b/gemfiles/Gemfile.elasticsearch-5.x.lock index 4e75c3bb..a720c36b 100644 --- a/gemfiles/Gemfile.elasticsearch-5.x.lock +++ b/gemfiles/Gemfile.elasticsearch-5.x.lock @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.elasticsearch-6.x.lock b/gemfiles/Gemfile.elasticsearch-6.x.lock index 0c71983a..d067d326 100644 --- a/gemfiles/Gemfile.elasticsearch-6.x.lock +++ b/gemfiles/Gemfile.elasticsearch-6.x.lock @@ -69,7 +69,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.elasticsearch-7.x.lock b/gemfiles/Gemfile.elasticsearch-7.x.lock index 7bbfe225..19e53d7e 100644 --- a/gemfiles/Gemfile.elasticsearch-7.x.lock +++ b/gemfiles/Gemfile.elasticsearch-7.x.lock @@ -52,7 +52,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.elasticsearch-8.x.lock b/gemfiles/Gemfile.elasticsearch-8.x.lock index ac0a47f2..2c98fd27 100644 --- a/gemfiles/Gemfile.elasticsearch-8.x.lock +++ b/gemfiles/Gemfile.elasticsearch-8.x.lock @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.opensearch-1.x.lock b/gemfiles/Gemfile.opensearch-1.x.lock index 80d7db92..0b1e150b 100644 --- a/gemfiles/Gemfile.opensearch-1.x.lock +++ b/gemfiles/Gemfile.opensearch-1.x.lock @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) diff --git a/gemfiles/Gemfile.opensearch-2.x.lock b/gemfiles/Gemfile.opensearch-2.x.lock index 77664c30..a5381611 100644 --- a/gemfiles/Gemfile.opensearch-2.x.lock +++ b/gemfiles/Gemfile.opensearch-2.x.lock @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) From 46c4e31b02482f11341d473dda41c644d3c6888b Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Wed, 18 Mar 2026 17:11:02 -0300 Subject: [PATCH 4/5] update stack version --- .github/workflows/build.yml | 16 ++++++++-------- bin/run | 4 ++-- docker-compose.yml | 4 ++-- .../opensearch-response/1.x/info.json.erb | 2 +- .../opensearch-response/2.x/info.json.erb | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4b71af3..62b6a275 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,23 +53,23 @@ jobs: allow_failure: false - ruby: 2.7.7 gemfile: gemfiles/Gemfile.opensearch-1.x - stack: opensearch:1.3.3 - stack_version: "1.3.3" + stack: opensearch:1.3.20 + stack_version: "1.3.20" allow_failure: true - ruby: 2.7.7 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.19.3 - stack_version: "2.19.3" + stack: opensearch:2.19.5 + stack_version: "2.19.5" allow_failure: false - ruby: 3.2.4 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.1.0 - stack_version: "2.1.0" + stack: opensearch:2.19.5 + stack_version: "2.19.5" allow_failure: false - ruby: 3.3.3 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.1.0 - stack_version: "2.1.0" + stack: opensearch:2.19.5 + stack_version: "2.19.5" allow_failure: false runs-on: ubuntu-latest env: diff --git a/bin/run b/bin/run index 213cdd2c..3b82d733 100755 --- a/bin/run +++ b/bin/run @@ -61,14 +61,14 @@ case "${service_name}-${service_version}" in # docker-compose up -d elasticsearch-v8 ;; opensearch-v1) - export STACK_VERSION=1.3.3 + export STACK_VERSION=1.3.20 export DOCKER_OS_V1=${STACK_VERSION} export ESSE_URL=http://localhost:19201 export BUNDLE_GEMFILE=gemfiles/Gemfile.opensearch-1.x # docker-compose up -d opensearch-v1 ;; opensearch-v2) - export STACK_VERSION=2.19.3 + export STACK_VERSION=2.19.5 export DOCKER_OS_V2=${STACK_VERSION} export ESSE_URL=http://localhost:19202 export BUNDLE_GEMFILE=gemfiles/Gemfile.opensearch-2.x diff --git a/docker-compose.yml b/docker-compose.yml index c65a5b94..c09be557 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,7 @@ services: networks: - esse opensearch-v1: - image: opensearchproject/opensearch:${DOCKER_OS_V1:-1.3.3} + image: opensearchproject/opensearch:${DOCKER_OS_V1:-1.3.20} environment: - cluster.name=cluster-opensearch-v1 - cluster.routing.allocation.disk.threshold_enabled=false @@ -143,7 +143,7 @@ services: networks: - esse opensearch-v2: - image: opensearchproject/opensearch:${DOCKER_OS_V2:-2.19.3} + image: opensearchproject/opensearch:${DOCKER_OS_V2:-2.19.5} environment: - cluster.name=cluster-opensearch-v2 - cluster.routing.allocation.disk.threshold_enabled=false diff --git a/spec/fixtures/opensearch-response/1.x/info.json.erb b/spec/fixtures/opensearch-response/1.x/info.json.erb index 0ad725c9..42ed36a0 100644 --- a/spec/fixtures/opensearch-response/1.x/info.json.erb +++ b/spec/fixtures/opensearch-response/1.x/info.json.erb @@ -4,7 +4,7 @@ "cluster_uuid" : "CRJEd89rQ66qWxfoebXPKA", "version" : { "distribution" : "opensearch", - "number" : "<%= assigns.fetch(:version__number, "1.3.3") %>", + "number" : "<%= assigns.fetch(:version__number, "1.3.20") %>", "build_type" : "tar", "build_hash" : "6d63fc707f6010c5980ec5cb51324856caf1f03d", "build_date" : "2022-06-07T15:24:06.598626Z", diff --git a/spec/fixtures/opensearch-response/2.x/info.json.erb b/spec/fixtures/opensearch-response/2.x/info.json.erb index 07153e38..19a780aa 100644 --- a/spec/fixtures/opensearch-response/2.x/info.json.erb +++ b/spec/fixtures/opensearch-response/2.x/info.json.erb @@ -4,7 +4,7 @@ "cluster_uuid" : "FCKGemptSwWpD0JXhLEX-g", "version" : { "distribution" : "opensearch", - "number" : "<%= assigns.fetch(:version__number, "2.19.3") %>", + "number" : "<%= assigns.fetch(:version__number, "2.19.5") %>", "build_type" : "tar", "build_hash" : "6462a546240f6d7a158519499729bce12dc1058b", "build_date" : "2022-06-15T08:47:42.243126494Z", From d648f42122684ac93b6c41147203d379a9e78c59 Mon Sep 17 00:00:00 2001 From: "Marcos G. Zimmermann" Date: Wed, 18 Mar 2026 17:12:32 -0300 Subject: [PATCH 5/5] feat: include new versions to the ci --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62b6a275..80906962 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,11 @@ jobs: stack: opensearch:2.19.5 stack_version: "2.19.5" allow_failure: false + - ruby: 3.4.8 + gemfile: gemfiles/Gemfile.opensearch-2.x + stack: opensearch:3.5.0 + stack_version: "3.5.0" + allow_failure: false runs-on: ubuntu-latest env: BUNDLE_GEMFILE: ${{ matrix.gemfile }}