diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 056ebc10..80906962 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,21 +6,6 @@ jobs: fail-fast: false matrix: include: - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-1.x - stack: elasticsearch:1.7.6 - stack_version: "1.7.6" - allow_failure: true - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-2.x - stack: elasticsearch:2.4.6 - stack_version: "2.4.6" - allow_failure: true - - ruby: 2.6.10 - gemfile: gemfiles/Gemfile.elasticsearch-5.x - stack: elasticsearch:5.6.16 - stack_version: "5.6.16" - allow_failure: true - ruby: 2.7.7 gemfile: gemfiles/Gemfile.elasticsearch-6.x stack: elasticsearch:6.8.16 @@ -68,23 +53,28 @@ jobs: allow_failure: false - ruby: 2.7.7 gemfile: gemfiles/Gemfile.opensearch-1.x - stack: opensearch:1.3.3 - stack_version: "1.3.3" + stack: opensearch:1.3.20 + stack_version: "1.3.20" allow_failure: true - ruby: 2.7.7 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.19.3 - stack_version: "2.19.3" + stack: opensearch:2.19.5 + stack_version: "2.19.5" allow_failure: false - ruby: 3.2.4 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.1.0 - stack_version: "2.1.0" + stack: opensearch:2.19.5 + stack_version: "2.19.5" allow_failure: false - ruby: 3.3.3 gemfile: gemfiles/Gemfile.opensearch-2.x - stack: opensearch:2.1.0 - stack_version: "2.1.0" + stack: opensearch:2.19.5 + stack_version: "2.19.5" + allow_failure: false + - ruby: 3.4.8 + gemfile: gemfiles/Gemfile.opensearch-2.x + stack: opensearch:3.5.0 + stack_version: "3.5.0" allow_failure: false runs-on: ubuntu-latest env: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 065aabca..00000000 --- a/.travis.yml +++ /dev/null @@ -1,47 +0,0 @@ -# dist: trusty -language: ruby -before_install: - - gem --version - - gem update bundler - - gem --version - - bash gemfiles/setup.sh - -jobs: - fast_finish: true - include: - - rvm: 2.6 - env: STACK_VERSION=1.7.6 DOCKER_IMAGE=elasticsearch - gemfile: gemfiles/Gemfile.elasticsearch-1.x - - rvm: 2.6 - env: STACK_VERSION=2.3 DOCKER_IMAGE=elasticsearch - gemfile: gemfiles/Gemfile.elasticsearch-2.x - - rvm: 2.6 - env: STACK_VERSION=5.6.16 - gemfile: gemfiles/Gemfile.elasticsearch-5.x - - rvm: 2.6 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 2.6 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.7 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 2.7 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.7 - env: STACK_VERSION=8.2.1 - gemfile: gemfiles/Gemfile.elasticsearch-8.x - - rvm: 3.0 - env: STACK_VERSION=6.8.16 - gemfile: gemfiles/Gemfile.elasticsearch-6.x - - rvm: 3.0 - env: STACK_VERSION=7.13.2 - gemfile: gemfiles/Gemfile.elasticsearch-7.x - - rvm: 2.6 - env: STACK_VERSION=1.3.3 DOCKER_IMAGE=opensearchproject/opensearch - gemfile: gemfiles/Gemfile.opensearch-2.x - - rvm: 2.6 - env: STACK_VERSION=2.19.3 DOCKER_IMAGE=opensearchproject/opensearch - gemfile: gemfiles/Gemfile.opensearch-2.x diff --git a/Gemfile.lock b/Gemfile.lock index c17452df..542de95e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -37,7 +37,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -84,7 +84,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/README.old b/README.old deleted file mode 100644 index 82dd2805..00000000 --- a/README.old +++ /dev/null @@ -1,420 +0,0 @@ -![esse-red](https://user-images.githubusercontent.com/18994/186032704-f1c9ce86-a41a-41ae-a224-30f4b382c012.png) - - -# esse - -**This project is under development and may suffer constant structural changes. I don't recommend using it right now** - -Simple and efficient way to organize queries/mapping/indices/tasks based on the official elasticsearch-ruby. - -## Why to use it? -Some facts to use this library: - -### Don't spend time learning our DLS -You don't need to spend time learning our DSL or gem usage to start using it. All you need know is the elasticsearch syntax. You are free to build your queries/mappings/settings using JSON/RubyHash flexibility. And keeping simple any elasticsearch upgrade and its syntax changes. - -### Multiple ElasticSearch Versions -You can use multiple elasticsearch servers with different versions in an elegant way. Take a look at [LINK TO TOPIC](#anchors-id-here) for more details. - -### It's pure Ruby -Yeah!! Nor [activesupport](http://github.com/rails/rails/tree/master/activesupport) dependency and all its monkey patchings. But if you are using rails, suggest install `esse-rails` extension that makes things even easier. Use the [Get started with esse-rails](#anchors-id-here) for more details. - -## Installation - -Add this line to your application's Gemfile: - -```ruby -gem 'esse' -``` - -And then execute: - - $ bundle install - -Or install it yourself as: - - $ gem install esse - -## Usage - -### Configuration - -Example of gem configuration using the `configure` method: -```ruby -Esse.configure do |config| - config.indices_directory = 'app/indices' - config.cluster do |cluster| - cluster.index_prefix = 'illinois' - cluster.settings = { - index: { - number_of_shards: 4, - number_of_replicas: 1, - }, - analysis: { - analyzer: { - esse_index: { - type: "custom", - char_filter: ["ampersand"], - filter: [ - "lowercase", - "asciifolding", - "esse_english_stop", - "esse_index_shingle", - "esse_stemmer" - ], - tokenizer: "standard" - }, - }, - filter: { - esse_index_shingle: { - token_separator: "", - type: "shingle" - }, - esse_stemmer: { - type: "stemmer", - language: "English" - }, - }, - char_filter: { - ampersand: { - type: "mapping", - mappings: ["&=> and "] - } - } - } - } - cluster.mappings = { - dynamic_templates: [ - { - esse_string_template: { - match: "*", - match_mapping_type: "string", - mapping: { - fields: { - analyzed: { - analyzer: "esse_index", - index: true, - type: "text" - } - }, - ignore_above: 1024, - type: "keyword" - } - } - } - ] - } - cluster.client = Elasticsearch::Client.new(host: 'http://localhost:9200') - end -end -``` - -Note that that if the cluster is configured without any identifier, it will be used as the `:default` cluster. - -```ruby -Esse.config.cluster.settings -# => {index: {number_of_shards: 4, number_of_replicas: 1} } - -# or -Esse.config.cluster(:default).settings -# => {index: {number_of_shards: 4, number_of_replicas: 1} } -``` - -You may also configure multiple cluster connections by specifying the identifier like the example below: -```ruby -Esse.configure do |config| - config.indices_directory = 'app/indices' - config.cluster(:il) do |cluster| - cluster.index_prefix = 'illinois' - cluster.settings = { - index: { - number_of_shards: 4, - number_of_replicas: 1, - } - } - cluster.client = Elasticsearch::Client.new(host: 'https://illinois:9200') - end - config.cluster(:fl) do |cluster| - cluster.index_prefix = 'florida' - cluster.settings = { - index: { - number_of_shards: 2, - number_of_replicas: 2, - } - } - cluster.client = Elasticsearch::Client.new(host: 'https://florida:9200') - end -end -``` - -And on the index you can use the `:il` or `:fl` identifier to specify which cluster you want to use. - -```ruby -class Illinois::AccountsIndex < Esse::Index - self.cluster_id = :il - ... -end -class Florida::AccountsIndex < Esse::Index - self.cluster_id = :fl -end -``` - -You can also configure it through the YAML file. -```ruby -Esse.config.load('./config/esse.yml') - -# or -Esse.configure do |config| - config.load('./config/esse.yml') - config.indices_directory = 'override/indices/directory/from/yml' -end -``` - -And the `config/esse.yml` file should look like: - -```yaml -indices_directory: "app/indices" -clusters: - il: - index_prefix: "illinois" - settings: - index: - number_of_shards: 4 - number_of_replicas: 1 - client: - host: "https://illinois:9200" - fl: - index_prefix: "florida" - settings: - index: - number_of_shards: 2 - number_of_replicas: 2 - client: - host: "https://florida:9200" -``` - -## Indices - -### Single type per index - -The mapping of elasticsearch 1.x, 2.x and 5.x allow multiple types. The single-type-per-index behaviour was introduction on es 5.x disabled as default. After es 6.x de single type per index is enabled by default but the explicit mapping is still required by using the `_doc` type. Es 7.x and above the `_doc` was totally removed and the mapping type is no longer required. The `esse` framework will automatically detect the version of elasticsearch and use the correct mapping type definition. But you can enforce the mapping type by using the `mapping_single_type` option in the `index`(Just make sure the server support it and it's configured accordingly). See [this article](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html) for more details. - -```ruby -class MyIndex < Esse::Index - self.mapping_single_type = true # This only control the mapping format. Add the setting `index.mapping.single_type: true` to the index settings in elasticsearch as well if needed. -end -``` - -## CLI - -The `esse` command line tool is available to manage indices and tasks. It's configured to load the configuration from the `Essefile`, `config/esse.rb` or `config/initializers/esse.rb` files(in that order). - -```bash -Commands: - esse --version, -v # Show package version - esse generate SUBCOMMAND ...ARGS # Run generators - esse help [COMMAND] # Describe available commands or one specific command - esse index SUBCOMMAND ...ARGS # Manage indices - esse install # Generate boilerplate configuration files - -Options: - -r, [--require=REQUIRE] # Require config file where the application is defined - -s, [--silent], [--no-silent] # Silent mode - -h, [--help], -?, [--usage] # Show help -``` - -### Generate Index - -Generate an index with the following command: - -```bash -❯ esse generate index <*doc_type> -``` - -List of types are optional. If not specified, the index will be created with definition on Index level with the `"default"` as type. Example: - -```bash -❯ bundle exec esse generate index GeosIndex - create app/indices/geos_index.rb -``` - -or with multiple datasources as document types: - -```bash -❯ bundle exec esse generate index GeosIndex state city - create app/indices/geos_index.rb -``` - -As default, the index will be create d with the collection, serializer, settings, mapping directly to the index class. By you can also specify custom arguments to better organize your code by splitting the it into multiple files. - -```bash -❯ ./exec/esse generate index GeosIndex state city --settings --mappings --documents --collections - create app/indices/geos_index.rb - create app/indices/geos_index/templates/settings.json - create app/indices/geos_index/templates/mappings.json - create app/indices/geos_index/documents/state_document.rb - create app/indices/geos_index/collections/state_collection.rb - create app/indices/geos_index/documents/city_document.rb - create app/indices/geos_index/collections/city_collection.rb -``` - -### Index Commands - -There are several commands to manage indices. The following commands are available: - -```bash -❯ bundle exec esse index [ruby-2.6.9p207] -Commands: - esse index close *INDEX_CLASS # Close an index (keep the data on disk, but deny operations with the index). - esse index create *INDEX_CLASSES # Creates indices for the given classes - esse index delete *INDEX_CLASSES # Deletes indices for the given classes - esse index help [COMMAND] # Describe subcommands or one specific subcommand - esse index import *INDEX_CLASSES --context=key:value # Import documents from the given classes - esse index open *INDEX_CLASS # Open a previously closed index. - esse index reset *INDEX_CLASSES # Performs zero-downtime index resetting. - esse index update_aliases *INDEX_CLASS # Replaces all existing aliases by the given suffix - esse index update_mapping *INDEX_CLASS # Create or update a mapping - esse index update_settings *INDEX_CLASS # Closes the index for read/write operations, updates the index settings, and open it again -``` - - -## Search - -Searching is done through the `search` method on the index class. It returns an instance of `Esse::Search::Query` that can be used to build the query and retrieve the results. - -```ruby -# Searching using Lucene query syntax -> query = GeosIndex.search('*', size: 10, offset: 0) -=> #10, :offset=>0, :q=>"*", :index=>"esse_console_geos"}> - - -# Searching using DSL -> query = GeosIndex.search(body: {query: {match: {name: 'Illinois'}}}, size: 1) -=> #{:query=>{:match=>{:name=>"Illinois"}}}, :size=>1, :index=>"esse_console_geos"}> - -# Retrieve response hits -> query.results -=> [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"IL", "_score"=>2.5433555, "_routing"=>"IL", "_source"=>{"id"=>"IL", "name"=>"Illinois"}}] -> query = GeosIndex.search(body: {query: {match: {"name.analyzed" => 'Illinois'}}}, _source: false) -=> #{:query=>{:match=>{"name.analyzed"=>"Illinois"}}}, :_source=>false, :index=>"esse_console_geos"} -> query.results -=> [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"IL", "_score"=>2.5433555, "_routing"=>"IL"} -# Retrieve response -> query.response -=> # -``` - -Search aggregations can be retrieved using the `aggregations` method from query response. - -```ruby -> query = GeosIndex.search(body: {query: { match_all: {}}, aggregations: { names: { terms: { field: "name", size: 2 }}}}, size: 2) -=> #{:query=>{:match_all=>{}}, :aggregations=>{:names=>{:terms=>{:field=>"name", :size=>2}}}}, :size=>2, :index=>"esse_console_geos"}> -> query.response.aggregations -=> {"names"=>{"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>14, "buckets"=>[{"key"=>"Alabama", "doc_count"=>1}, {"key"=>"Alaska", "doc_count"=>1}]}} - -# or just use the raw response -> query.response.raw_response -=> {"took"=>1, - "timed_out"=>false, - "_shards"=>{"total"=>1, "successful"=>1, "skipped"=>0, "failed"=>0}, - "hits"=> - {"total"=>{"value"=>16, "relation"=>"eq"}, - "max_score"=>1.0, - "hits"=> - [{"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"AL", "_score"=>1.0, "_routing"=>"AL", "_source"=>{"id"=>"AL", "name"=>"Alabama"}}, - {"_index"=>"esse_console_geos_v1", "_type"=>"_doc", "_id"=>"AK", "_score"=>1.0, "_routing"=>"AK", "_source"=>{"id"=>"AK", "name"=>"Alaska"}}]}, - "aggregations"=>{"names"=>{"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>14, "buckets"=>[{"key"=>"Alabama", "doc_count"=>1}, {"key"=>"Alaska", "doc_count"=>1}]}}} -> query.response.total -=> 16 -``` - -Scroll queries can be performed using the `scroll_hits` method on the query. - -```ruby -> query.scroll_hits(batch_size: 6, scroll: '1m') { |hits| puts hits.size } -6 -6 -4 -=> nil -``` - -Searching across multiple indices can be done using the `search` method on the cluster. - -Using string indices as arguments: - -```ruby -> Esse.config.cluster.search('esse_*', body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_*"}> -> Esse.config.cluster.search('esse_cities_index', 'esse_counties_index', body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_cities_index,esse_counties_index"}> -``` - -Using index classes as arguments: - -```ruby -> Esse.config.cluster.search(CitiesIndex, CountiesIndex, body: {query: {match_all:{}}}) -=> #{:query=>{:match_all=>{}}}, :index=>"esse_cities_index,esse_counties_index"}> -``` - -## Development - -After checking out the repo, run `bin/setup` to install dependencies. The command will dependencies of all `./gemfiles/Gemfile.*`. - -You can use `docker-compose` to start elasticsearch and opensearch services. - -```bash -❯ docker-compose up -d -``` - -```bash -❯ docker-compose ps -NAME COMMAND SERVICE STATUS PORTS -esse-elasticsearch-v1-1 "/docker-entrypoint.…" elasticsearch-v1 running 9300/tcp, 0.0.0.0:9201->9200/tcp, :::9201->9200/tcp -esse-elasticsearch-v2-1 "/docker-entrypoint.…" elasticsearch-v2 running 9300/tcp, 0.0.0.0:9202->9200/tcp, :::9202->9200/tcp -esse-elasticsearch-v5-1 "/bin/bash bin/es-do…" elasticsearch-v5 running 9300/tcp, 0.0.0.0:9205->9200/tcp, :::9205->9200/tcp -esse-elasticsearch-v6-1 "/usr/local/bin/dock…" elasticsearch-v6 running 9300/tcp, 0.0.0.0:9206->9200/tcp, :::9206->9200/tcp -esse-elasticsearch-v7-1 "/bin/tini -- /usr/l…" elasticsearch-v7 running 9300/tcp, 0.0.0.0:9207->9200/tcp, :::9207->9200/tcp -esse-elasticsearch-v8-1 "/bin/tini -- /usr/l…" elasticsearch-v8 running 9300/tcp, 0.0.0.0:9208->9200/tcp, :::9208->9200/tcp -esse-opensearch-v1-1 "./opensearch-docker…" opensearch-v1 running 9300/tcp, 9600/tcp, 9650/tcp, 0.0.0.0:19201->9200/tcp, :::19201->9200/tcp -esse-opensearch-v2-1 "./opensearch-docker…" opensearch-v2 running 9300/tcp, 9600/tcp, 9650/tcp, 0.0.0.0:19202->9200/tcp, :::19202->9200/tcp -``` -As you can see each elasticsearch or opensearch version is binded to a different port. - -Tests and console uses the `ESSE_URL` along with `STACK_VERSION` and `BUNDLE_GEMFILE` to determine which elasticsearch/opensearch and Gemfile with the official ruby client to use: - -```bash -❯ export ESSE_URL=http://localhost:9201 -❯ export STACK_VERSION=7.13.2 -❯ export BUNDLE_GEMFILE=gemfiles/Gemfile.v7 -``` - -But we have `bin/run` script to simplify this process. It will automatically set the required envionment variables according to the given arguments. - -```bash -❯ bin/run elasticsearch v7 ./bin/console -> ENV.values_at('ESSE_URL', 'STACK_VERSION', 'BUNDLE_GEMFILE') -=> ["http://localhost:9207", "7.13.2", "gemfiles/Gemfile.elasticsearch-7.x"] -> Esse.config.cluster.api.health.fetch('cluster_name') -=> "cluster-elasticsearch-v7" -``` - -And to run the tests: - -```bash -❯ bin/run elasticsearch v7 bundle exec rspec -``` - -If you don't have elasticsearch running and want to ignore integratino tests, you can use the `STUB_STACK=-` environment variable to stup the test suite for a specific elasticsearch version. Note that all examples with `:es_version` meta data will be skipped. - -```bash -❯ STUB_STACK=elasticsearch-7.0.3 bundle exec --gemfile gemfiles/Gemfile.elasticsearch-7.x rspec -``` - -## Contributing - -Bug reports and pull requests are welcome on GitHub at https://github.com/marcosgz/esse. - - -## License - -The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/bin/run b/bin/run index 213cdd2c..3b82d733 100755 --- a/bin/run +++ b/bin/run @@ -61,14 +61,14 @@ case "${service_name}-${service_version}" in # docker-compose up -d elasticsearch-v8 ;; opensearch-v1) - export STACK_VERSION=1.3.3 + export STACK_VERSION=1.3.20 export DOCKER_OS_V1=${STACK_VERSION} export ESSE_URL=http://localhost:19201 export BUNDLE_GEMFILE=gemfiles/Gemfile.opensearch-1.x # docker-compose up -d opensearch-v1 ;; opensearch-v2) - export STACK_VERSION=2.19.3 + export STACK_VERSION=2.19.5 export DOCKER_OS_V2=${STACK_VERSION} export ESSE_URL=http://localhost:19202 export BUNDLE_GEMFILE=gemfiles/Gemfile.opensearch-2.x diff --git a/bin/setup b/bin/setup index 282b9397..7252bf77 100755 --- a/bin/setup +++ b/bin/setup @@ -3,5 +3,17 @@ set -euo pipefail IFS=$'\n\t' set -vx -find gemfiles -type f \( -iname "Gemfile.elasticsearch-*" ! -iname "*.lock" \) -exec bundle install --gemfile {} \; -find gemfiles -type f \( -iname "Gemfile.opensearch-*" ! -iname "*.lock" \) -exec bundle install --gemfile {} \; +RUBY_IMAGE="ruby:2.7" +BUNDLER_VERSION="2.3.26" + +docker run --rm -v "$(pwd)":/app -w /app "$RUBY_IMAGE" \ + bash -c "gem install bundler:$BUNDLER_VERSION --no-document && bundle install" + +for gemfile in gemfiles/Gemfile.elasticsearch-*.x gemfiles/Gemfile.opensearch-*.x; do + [ -f "$gemfile" ] || continue + [[ "$gemfile" == *.lock ]] && continue + docker run --rm -v "$(pwd)":/app -w /app \ + -e BUNDLE_GEMFILE="$gemfile" \ + "$RUBY_IMAGE" \ + bash -c "gem install bundler:$BUNDLER_VERSION --no-document && bundle install" +done diff --git a/docker-compose.yml b/docker-compose.yml index c65a5b94..c09be557 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,7 @@ services: networks: - esse opensearch-v1: - image: opensearchproject/opensearch:${DOCKER_OS_V1:-1.3.3} + image: opensearchproject/opensearch:${DOCKER_OS_V1:-1.3.20} environment: - cluster.name=cluster-opensearch-v1 - cluster.routing.allocation.disk.threshold_enabled=false @@ -143,7 +143,7 @@ services: networks: - esse opensearch-v2: - image: opensearchproject/opensearch:${DOCKER_OS_V2:-2.19.3} + image: opensearchproject/opensearch:${DOCKER_OS_V2:-2.19.5} environment: - cluster.name=cluster-opensearch-v2 - cluster.routing.allocation.disk.threshold_enabled=false diff --git a/esse.gemspec b/esse.gemspec index 564f953a..b0cfd7f6 100644 --- a/esse.gemspec +++ b/esse.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| 'which means you can use it with any Ruby framework or even without any framework at all.' spec.homepage = 'https://github.com/marcosgz/esse' spec.license = 'MIT' - spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0') + spec.required_ruby_version = Gem::Requirement.new('>= 2.7') spec.metadata['homepage_uri'] = spec.homepage spec.metadata['source_code_uri'] = 'https://github.com/marcosgz/esse' diff --git a/gemfiles/Gemfile.elasticsearch-1.x.lock b/gemfiles/Gemfile.elasticsearch-1.x.lock index e564515d..ce3971ec 100644 --- a/gemfiles/Gemfile.elasticsearch-1.x.lock +++ b/gemfiles/Gemfile.elasticsearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -48,7 +48,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -95,7 +95,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-2.x.lock b/gemfiles/Gemfile.elasticsearch-2.x.lock index 97157b25..2f6433a6 100644 --- a/gemfiles/Gemfile.elasticsearch-2.x.lock +++ b/gemfiles/Gemfile.elasticsearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-5.x.lock b/gemfiles/Gemfile.elasticsearch-5.x.lock index 05eb5bb8..a720c36b 100644 --- a/gemfiles/Gemfile.elasticsearch-5.x.lock +++ b/gemfiles/Gemfile.elasticsearch-5.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-6.x.lock b/gemfiles/Gemfile.elasticsearch-6.x.lock index 6d0f6684..d067d326 100644 --- a/gemfiles/Gemfile.elasticsearch-6.x.lock +++ b/gemfiles/Gemfile.elasticsearch-6.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -69,7 +69,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -117,7 +117,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-7.x.lock b/gemfiles/Gemfile.elasticsearch-7.x.lock index 3e02bff6..19e53d7e 100644 --- a/gemfiles/Gemfile.elasticsearch-7.x.lock +++ b/gemfiles/Gemfile.elasticsearch-7.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -52,7 +52,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -100,7 +100,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.elasticsearch-8.x.lock b/gemfiles/Gemfile.elasticsearch-8.x.lock index b32833a4..2c98fd27 100644 --- a/gemfiles/Gemfile.elasticsearch-8.x.lock +++ b/gemfiles/Gemfile.elasticsearch-8.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.opensearch-1.x.lock b/gemfiles/Gemfile.opensearch-1.x.lock index a282670d..0b1e150b 100644 --- a/gemfiles/Gemfile.opensearch-1.x.lock +++ b/gemfiles/Gemfile.opensearch-1.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/gemfiles/Gemfile.opensearch-2.x.lock b/gemfiles/Gemfile.opensearch-2.x.lock index 98539ae7..a5381611 100644 --- a/gemfiles/Gemfile.opensearch-2.x.lock +++ b/gemfiles/Gemfile.opensearch-2.x.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - esse (0.4.0.rc5) + esse (0.4.0) multi_json thor (>= 0.19) @@ -51,7 +51,7 @@ GEM rainbow (3.1.1) rake (12.3.3) regexp_parser (2.9.2) - rexml (3.4.1) + rexml (3.4.4) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -99,7 +99,7 @@ GEM standard-performance (1.0.1) lint_roller (~> 1.0) rubocop-performance (~> 1.16.0) - thor (1.4.0) + thor (1.5.0) unicode-display_width (2.5.0) webmock (3.23.1) addressable (>= 2.8.0) diff --git a/lib/esse/search/query.rb b/lib/esse/search/query.rb index 70f0380c..f89879ba 100644 --- a/lib/esse/search/query.rb +++ b/lib/esse/search/query.rb @@ -40,7 +40,7 @@ def results def scroll_hits(batch_size: 1_000, scroll: '1m') response = execute_search_query!(size: batch_size, scroll: scroll) - scroll_id = nil + scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] fetched = 0 total = response.total @@ -48,9 +48,9 @@ def scroll_hits(batch_size: 1_000, scroll: '1m') fetched += response.hits.size yield(response.hits) if response.hits.any? break if fetched >= total - scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] break unless scroll_id response = execute_scroll_query(scroll: scroll, scroll_id: scroll_id) + scroll_id = response.raw_response['scroll_id'] || response.raw_response['_scroll_id'] end ensure begin @@ -83,17 +83,15 @@ def search_after_hits(batch_size: 1_000) end end + def reset! + @response = nil + end + private def execute_search_query!(**execution_options) resp, err = nil Esse::Events.instrument('elasticsearch.execute_search_query') do |payload| - # Store only the definition hash instead of the full Query object. - # The Query holds a reference to transport (connection pool) and - # storing it in the event payload causes it to be copied into a new - # Event object for each listener (via @payload.merge). With 24+ - # esse-rails subscribers, this means 24+ copies of the Query reference - # per search. The definition hash is lightweight and already exists. payload[:query_definition] = definition begin resp = Response.new(self, transport.search(**definition, **execution_options)) @@ -124,10 +122,6 @@ def execute_scroll_query(scroll:, scroll_id:) resp end - - def reset! - @response = nil - end end end end diff --git a/lib/esse/search/response.rb b/lib/esse/search/response.rb index a006d8ef..a620e7f6 100644 --- a/lib/esse/search/response.rb +++ b/lib/esse/search/response.rb @@ -7,13 +7,13 @@ class Response extend Forwardable def_delegators :hits, :each, :size, :empty? - attr_reader :query, :raw_response, :options + attr_reader :query_definition, :raw_response, :options - # @param [Esse::Search::Query] query The search query + # @param [Esse::Search::Query, Hash] query The search query or its definition hash # @param [Hash] raw_response The raw response from Elasticsearch # @param [Hash] options The options passed to the search def initialize(query, raw_response, **options) - @query = query + @query_definition = query.is_a?(Hash) ? query : query.definition @raw_response = raw_response @options = options end diff --git a/lib/esse/version.rb b/lib/esse/version.rb index 8a54a97d..882759d2 100644 --- a/lib/esse/version.rb +++ b/lib/esse/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Esse - VERSION = '0.4.0.rc5' + VERSION = '0.4.0' end diff --git a/spec/esse/search/query_spec.rb b/spec/esse/search/query_spec.rb index 8af2ddd7..af0133ea 100644 --- a/spec/esse/search/query_spec.rb +++ b/spec/esse/search/query_spec.rb @@ -107,7 +107,7 @@ expect(transport).to receive(:search).with(index: 'events', body: request_body).and_return(body) expect(resp = query.response).to be_an_instance_of(Esse::Search::Response) - expect(resp.query).to eq(query) + expect(resp.query_definition).to eq(query.definition) end end @@ -124,7 +124,7 @@ expect(transport).to receive(:search).with(index: 'events', body: request_body).and_return(response_body) expect(query.response).to be_an_instance_of(Esse::Search::Response) - assert_event 'elasticsearch.execute_search_query', { query: query, response: query.response } + assert_event 'elasticsearch.execute_search_query', { query_definition: query.definition, response: query.response } end end @@ -138,7 +138,7 @@ expect { query.response }.to raise_error(Esse::Transport::BadRequestError) - assert_event 'elasticsearch.execute_search_query', { query: query, error: exception } + assert_event 'elasticsearch.execute_search_query', { query_definition: query.definition, error: exception } end end end diff --git a/spec/fixtures/opensearch-response/1.x/info.json.erb b/spec/fixtures/opensearch-response/1.x/info.json.erb index 0ad725c9..42ed36a0 100644 --- a/spec/fixtures/opensearch-response/1.x/info.json.erb +++ b/spec/fixtures/opensearch-response/1.x/info.json.erb @@ -4,7 +4,7 @@ "cluster_uuid" : "CRJEd89rQ66qWxfoebXPKA", "version" : { "distribution" : "opensearch", - "number" : "<%= assigns.fetch(:version__number, "1.3.3") %>", + "number" : "<%= assigns.fetch(:version__number, "1.3.20") %>", "build_type" : "tar", "build_hash" : "6d63fc707f6010c5980ec5cb51324856caf1f03d", "build_date" : "2022-06-07T15:24:06.598626Z", diff --git a/spec/fixtures/opensearch-response/2.x/info.json.erb b/spec/fixtures/opensearch-response/2.x/info.json.erb index 07153e38..19a780aa 100644 --- a/spec/fixtures/opensearch-response/2.x/info.json.erb +++ b/spec/fixtures/opensearch-response/2.x/info.json.erb @@ -4,7 +4,7 @@ "cluster_uuid" : "FCKGemptSwWpD0JXhLEX-g", "version" : { "distribution" : "opensearch", - "number" : "<%= assigns.fetch(:version__number, "2.19.3") %>", + "number" : "<%= assigns.fetch(:version__number, "2.19.5") %>", "build_type" : "tar", "build_hash" : "6462a546240f6d7a158519499729bce12dc1058b", "build_date" : "2022-06-15T08:47:42.243126494Z",