From a06a119df355c54032b9e7f19123787451c168c5 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sun, 22 Mar 2026 13:01:19 -0300 Subject: [PATCH 01/19] feat: transparent leader hint reconnect on consume --- lib/fila/client.rb | 58 +++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index dbcdc73..f6d35b9 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -40,8 +40,8 @@ class Client # @param api_key [String, nil] API key for Bearer token authentication def initialize(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) @api_key = api_key - credentials = build_credentials(tls: tls, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key) - @stub = ::Fila::V1::FilaService::Stub.new(addr, credentials) + @credentials = build_credentials(tls: tls, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key) + @stub = ::Fila::V1::FilaService::Stub.new(addr, @credentials) end # Close the underlying gRPC channel. @@ -88,20 +88,7 @@ def enqueue(queue:, payload:, headers: nil) def consume(queue:, &block) return enum_for(:consume, queue: queue) unless block - req = ::Fila::V1::ConsumeRequest.new(queue: queue) - stream = @stub.consume(req, metadata: call_metadata) - stream.each do |resp| - msg = resp.message - next if msg.nil? || msg.id.empty? - - block.call(build_consume_message(msg)) - end - rescue GRPC::Cancelled - # Stream cancelled — normal when consumer breaks out of the loop. - rescue GRPC::NotFound => e - raise QueueNotFoundError, "consume: #{e.details}" - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) + consume_with_redirect(queue: queue, redirected: false, &block) end # Acknowledge a successfully processed message. @@ -139,6 +126,45 @@ def nack(queue:, msg_id:, error:) private + LEADER_ADDR_KEY = 'x-fila-leader-addr' + + # Execute consume against a stub, following a leader hint redirect once. + # + # @param queue [String] queue to consume from + # @param redirected [Boolean] whether this is already a redirect attempt + def consume_with_redirect(queue:, redirected:, &block) + req = ::Fila::V1::ConsumeRequest.new(queue: queue) + stream = @stub.consume(req, metadata: call_metadata) + stream.each do |resp| + msg = resp.message + next if msg.nil? || msg.id.empty? + + block.call(build_consume_message(msg)) + end + rescue GRPC::Cancelled + # Stream cancelled — normal when consumer breaks out of the loop. + rescue GRPC::NotFound => e + raise QueueNotFoundError, "consume: #{e.details}" + rescue GRPC::Unavailable => e + leader_addr = extract_leader_addr(e) + raise RPCError.new(e.code, e.details) if leader_addr.nil? || redirected + + @stub = ::Fila::V1::FilaService::Stub.new(leader_addr, @credentials) + consume_with_redirect(queue: queue, redirected: true, &block) + rescue GRPC::BadStatus => e + raise RPCError.new(e.code, e.details) + end + + # Extract the leader address from an UNAVAILABLE error's trailing metadata. + # + # @param err [GRPC::Unavailable] the gRPC error + # @return [String, nil] leader address or nil if not present + def extract_leader_addr(err) + err.metadata[LEADER_ADDR_KEY] + rescue StandardError + nil + end + # Build gRPC channel credentials from the provided TLS options. # # When +ca_cert+ is provided, it is used for server verification (implies TLS). From 2658c531b65281eb5b83690ea937dd3771852158 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 22:55:51 -0300 Subject: [PATCH 02/19] fix: rubocop offenses in client.rb --- lib/fila/client.rb | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index f6d35b9..e5a9fc2 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -124,30 +124,25 @@ def nack(queue:, msg_id:, error:) raise RPCError.new(e.code, e.details) end - private - LEADER_ADDR_KEY = 'x-fila-leader-addr' - # Execute consume against a stub, following a leader hint redirect once. - # - # @param queue [String] queue to consume from - # @param redirected [Boolean] whether this is already a redirect attempt - def consume_with_redirect(queue:, redirected:, &block) - req = ::Fila::V1::ConsumeRequest.new(queue: queue) - stream = @stub.consume(req, metadata: call_metadata) + private_constant :LEADER_ADDR_KEY + + private + + def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/AbcSize + stream = @stub.consume(::Fila::V1::ConsumeRequest.new(queue: queue), metadata: call_metadata) stream.each do |resp| msg = resp.message next if msg.nil? || msg.id.empty? block.call(build_consume_message(msg)) end - rescue GRPC::Cancelled - # Stream cancelled — normal when consumer breaks out of the loop. + rescue GRPC::Cancelled then nil rescue GRPC::NotFound => e raise QueueNotFoundError, "consume: #{e.details}" rescue GRPC::Unavailable => e - leader_addr = extract_leader_addr(e) - raise RPCError.new(e.code, e.details) if leader_addr.nil? || redirected + raise RPCError.new(e.code, e.details) if (leader_addr = extract_leader_addr(e)).nil? || redirected @stub = ::Fila::V1::FilaService::Stub.new(leader_addr, @credentials) consume_with_redirect(queue: queue, redirected: true, &block) @@ -155,10 +150,6 @@ def consume_with_redirect(queue:, redirected:, &block) raise RPCError.new(e.code, e.details) end - # Extract the leader address from an UNAVAILABLE error's trailing metadata. - # - # @param err [GRPC::Unavailable] the gRPC error - # @return [String, nil] leader address or nil if not present def extract_leader_addr(err) err.metadata[LEADER_ADDR_KEY] rescue StandardError From 42afd7c2d944698e5bcb80b567c31123d26f50d9 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 22:56:55 -0300 Subject: [PATCH 03/19] fix: reduce class length to pass rubocop --- lib/fila/client.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index e5a9fc2..91e2fd2 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -156,13 +156,6 @@ def extract_leader_addr(err) nil end - # Build gRPC channel credentials from the provided TLS options. - # - # When +ca_cert+ is provided, it is used for server verification (implies TLS). - # When +tls+ is true without +ca_cert+, the OS system trust store is used. - # When neither is set and no client certs are given, plaintext is used. - # - # @return [Symbol, GRPC::Core::ChannelCredentials] credentials object def build_credentials(tls:, ca_cert:, client_cert:, client_key:) tls_enabled = tls || ca_cert validate_tls_options(tls_enabled, client_cert, client_key) From 55cafabd2207cb8fd9487ed67b6e04cd7eacde80 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 22:58:26 -0300 Subject: [PATCH 04/19] fix: trim class length to pass rubocop 100-line limit --- lib/fila/client.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 91e2fd2..3476bff 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -44,11 +44,7 @@ def initialize(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil @stub = ::Fila::V1::FilaService::Stub.new(addr, @credentials) end - # Close the underlying gRPC channel. - def close - # grpc-ruby doesn't expose a direct channel close on stubs; - # the channel is garbage-collected. This is a no-op for API symmetry. - end + def close; end # Enqueue a message to the specified queue. # @@ -182,9 +178,6 @@ def build_channel_credentials(ca_cert, client_cert, client_key) end end - # Return metadata hash for gRPC calls, including Bearer token when api_key is set. - # - # @return [Hash] metadata hash (may be empty) def call_metadata return {} unless @api_key From 11aeaf2e05fdfd7fdabc76c77828ab038e2e29af Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 23:00:12 -0300 Subject: [PATCH 05/19] fix: trim more doc comments to pass rubocop class length --- lib/fila/client.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 3476bff..80924ed 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -30,14 +30,6 @@ module Fila # client_key: File.read("client-key.pem"), # api_key: "fila_abc123") class Client - # Connect to a Fila broker at the given address. - # - # @param addr [String] broker address in "host:port" format (e.g., "localhost:5555") - # @param tls [Boolean] enable TLS using the OS system trust store (default: false) - # @param ca_cert [String, nil] PEM-encoded CA certificate for TLS verification (implies tls: true) - # @param client_cert [String, nil] PEM-encoded client certificate for mTLS - # @param client_key [String, nil] PEM-encoded client private key for mTLS - # @param api_key [String, nil] API key for Bearer token authentication def initialize(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) @api_key = api_key @credentials = build_credentials(tls: tls, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key) From ae180a340834e3bf90a99b2e0583cd0e6c26985b Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 23:01:41 -0300 Subject: [PATCH 06/19] fix: trim consume doc comment to pass rubocop class length --- lib/fila/client.rb | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 80924ed..07d1b7e 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -60,19 +60,8 @@ def enqueue(queue:, payload:, headers: nil) raise RPCError.new(e.code, e.details) end - # Open a streaming consumer on the specified queue. - # - # Yields messages as they become available. Nil message frames (keepalive - # signals) are skipped automatically. Nacked messages are redelivered on - # the same stream. - # - # If no block is given, returns an Enumerator. - # - # @param queue [String] queue to consume from - # @yield [ConsumeMessage] each message received from the broker - # @return [Enumerator] if no block given - # @raise [QueueNotFoundError] if the queue does not exist - # @raise [RPCError] for unexpected gRPC failures + # Open a streaming consumer. Yields messages as they arrive. + # Returns an Enumerator if no block given. def consume(queue:, &block) return enum_for(:consume, queue: queue) unless block From d77b7772a1d9cedf870ca3772bc5a2f0a6860708 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 23:05:30 -0300 Subject: [PATCH 07/19] fix: trim enqueue doc comments to pass rubocop class length --- lib/fila/client.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 07d1b7e..86c51d9 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -38,14 +38,6 @@ def initialize(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil def close; end - # Enqueue a message to the specified queue. - # - # @param queue [String] target queue name - # @param headers [Hash, nil] optional message headers - # @param payload [String] message payload bytes - # @return [String] broker-assigned message ID (UUIDv7) - # @raise [QueueNotFoundError] if the queue does not exist - # @raise [RPCError] for unexpected gRPC failures def enqueue(queue:, payload:, headers: nil) req = ::Fila::V1::EnqueueRequest.new( queue: queue, From 7601fb138cdb6597d1e54e8bf5fcfe7c802f39f8 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 23:07:16 -0300 Subject: [PATCH 08/19] fix: condense build_channel_credentials to pass rubocop class length --- lib/fila/client.rb | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 86c51d9..1fdef59 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -140,14 +140,9 @@ def validate_tls_options(tls_enabled, client_cert, client_key) end def build_channel_credentials(ca_cert, client_cert, client_key) - has_client_certs = client_cert && client_key - - if ca_cert - GRPC::Core::ChannelCredentials.new(ca_cert, client_key, client_cert) - elsif has_client_certs - GRPC::Core::ChannelCredentials.new(nil, client_key, client_cert) - else - GRPC::Core::ChannelCredentials.new + if ca_cert then GRPC::Core::ChannelCredentials.new(ca_cert, client_key, client_cert) + elsif client_cert && client_key then GRPC::Core::ChannelCredentials.new(nil, client_key, client_cert) + else GRPC::Core::ChannelCredentials.new end end From f570f47561b166473c2391e246c543cf723e2525 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Mon, 23 Mar 2026 23:26:47 -0300 Subject: [PATCH 09/19] chore: bump version to 0.2.0 --- lib/fila/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fila/version.rb b/lib/fila/version.rb index 33019cf..cce5b33 100644 --- a/lib/fila/version.rb +++ b/lib/fila/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Fila - VERSION = '0.1.0' + VERSION = '0.2.0' end From 1510b29f61089be1e00004c2d839deebbebac800 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 24 Mar 2026 10:20:39 -0300 Subject: [PATCH 10/19] feat: add batch enqueue, delivery batching, and smart batch modes (#3) - Add batch_enqueue() for explicit multi-message BatchEnqueue RPC - Add background batcher with three modes: :auto (default), :linger, :disabled - Auto mode: opportunistic batching via Queue drain (zero latency at low load) - Linger mode: timer-based batching with configurable linger_ms and batch_size - Single-item optimization: 1 message uses Enqueue RPC (preserves error types) - Delivery batching: consume unpacks repeated messages field transparently - close() drains pending messages before disconnecting - Update proto with BatchEnqueue RPC and ConsumeResponse.messages field - Bump version to 0.3.0 --- lib/fila.rb | 2 + lib/fila/batch_enqueue_result.rb | 37 +++ lib/fila/batcher.rb | 198 +++++++++++++ lib/fila/client.rb | 158 +++++++++- lib/fila/proto/fila/v1/service_pb.rb | 5 +- lib/fila/proto/fila/v1/service_services_pb.rb | 1 + lib/fila/version.rb | 2 +- proto/fila/v1/service.proto | 19 +- test/test_batch.rb | 273 ++++++++++++++++++ test/test_client.rb | 1 + 10 files changed, 677 insertions(+), 19 deletions(-) create mode 100644 lib/fila/batch_enqueue_result.rb create mode 100644 lib/fila/batcher.rb create mode 100644 test/test_batch.rb diff --git a/lib/fila.rb b/lib/fila.rb index 91e321c..977a9a9 100644 --- a/lib/fila.rb +++ b/lib/fila.rb @@ -3,4 +3,6 @@ require_relative 'fila/version' require_relative 'fila/errors' require_relative 'fila/consume_message' +require_relative 'fila/batch_enqueue_result' +require_relative 'fila/batcher' require_relative 'fila/client' diff --git a/lib/fila/batch_enqueue_result.rb b/lib/fila/batch_enqueue_result.rb new file mode 100644 index 0000000..be9cf13 --- /dev/null +++ b/lib/fila/batch_enqueue_result.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Fila + # Result of a single message within a batch enqueue call. + # + # Each message in a batch is independently validated and processed. + # A failed message does not affect the others. + # + # @example + # results = client.batch_enqueue(messages) + # results.each do |r| + # if r.success? + # puts "Enqueued: #{r.message_id}" + # else + # puts "Failed: #{r.error}" + # end + # end + class BatchEnqueueResult + # @return [String, nil] broker-assigned message ID on success + attr_reader :message_id + + # @return [String, nil] error description on failure + attr_reader :error + + # @param message_id [String, nil] message ID if successful + # @param error [String, nil] error string if failed + def initialize(message_id: nil, error: nil) + @message_id = message_id + @error = error + end + + # @return [Boolean] true if the message was successfully enqueued + def success? + !@message_id.nil? + end + end +end diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb new file mode 100644 index 0000000..577f4d4 --- /dev/null +++ b/lib/fila/batcher.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +module Fila + # Background batcher that collects enqueue requests and flushes them + # in batches via BatchEnqueue RPC. Supports auto (opportunistic) and + # linger (timer-based) modes. + # + # @api private + class Batcher + # An item queued for batching, pairing a message with its result slot. + BatchItem = Struct.new(:request, :result_queue, keyword_init: true) + + # @param stub [Fila::V1::FilaService::Stub] gRPC stub + # @param metadata [Hash] call metadata (auth headers) + # @param mode [Symbol] :auto or :linger + # @param max_batch_size [Integer] cap on batch size (auto mode) + # @param batch_size [Integer] batch size threshold (linger mode) + # @param linger_ms [Integer] linger time in ms (linger mode) + def initialize(stub:, metadata:, mode:, max_batch_size: 100, batch_size: 100, linger_ms: 10) + @stub = stub + @metadata = metadata + @mode = mode + @max_batch_size = mode == :auto ? max_batch_size : batch_size + @linger_ms = linger_ms + @queue = Queue.new + @stopped = false + @mutex = Mutex.new + + @thread = Thread.new { run_loop } + @thread.abort_on_exception = true + end + + # Submit a message for batched sending. Blocks until the batch + # containing this message is flushed and the result is available. + # + # @param request [Fila::V1::EnqueueRequest] the enqueue request + # @return [String] message ID on success + # @raise [Fila::QueueNotFoundError] if the queue does not exist + # @raise [Fila::RPCError] for unexpected gRPC failures + def submit(request) + result_queue = Queue.new + item = BatchItem.new(request: request, result_queue: result_queue) + + @mutex.synchronize do + raise Fila::Error, 'batcher is closed' if @stopped + + @queue.push(item) + end + + # Block until the batcher flushes our batch and posts the result. + outcome = result_queue.pop + case outcome + when String then outcome + when Exception then raise outcome + else raise Fila::Error, "unexpected batcher result: #{outcome.inspect}" + end + end + + # Drain pending messages and stop the background thread. + def close + @mutex.synchronize { @stopped = true } + @queue.push(:shutdown) + @thread.join + end + + private + + def run_loop + case @mode + when :auto then run_auto_loop + when :linger then run_linger_loop + end + end + + # Auto mode: block for the first message, then non-blocking drain + # any additional messages that have arrived, flush concurrently. + def run_auto_loop + loop do + first = @queue.pop + break if first == :shutdown + + batch = [first] + drain_nonblocking(batch) + flush_batch(batch) + end + end + + # Linger mode: block for the first message, then wait up to linger_ms + # for more messages or until batch_size is reached. + def run_linger_loop + loop do + first = @queue.pop + break if first == :shutdown + + batch = [first] + deadline = current_time_ms + @linger_ms + + while batch.size < @max_batch_size + remaining_ms = deadline - current_time_ms + break if remaining_ms <= 0 + + begin + item = pop_with_timeout(remaining_ms) + break if item == :shutdown + + batch << item + rescue ThreadError + break + end + end + + flush_batch(batch) + end + end + + def drain_nonblocking(batch) + while batch.size < @max_batch_size + begin + item = @queue.pop(true) # non_block = true + if item == :shutdown + @queue.push(:shutdown) # re-enqueue so the loop sees it + break + end + batch << item + rescue ThreadError + break + end + end + end + + # Flush a batch of items. Uses single Enqueue RPC for 1 message + # (preserves exact error types like QueueNotFoundError), and + # BatchEnqueue RPC for 2+ messages. + def flush_batch(items) + if items.size == 1 + flush_single(items.first) + else + flush_multi(items) + end + end + + def flush_single(item) + resp = @stub.enqueue(item.request, metadata: @metadata) + item.result_queue.push(resp.message_id) + rescue GRPC::NotFound => e + item.result_queue.push(QueueNotFoundError.new("enqueue: #{e.details}")) + rescue GRPC::BadStatus => e + item.result_queue.push(RPCError.new(e.code, e.details)) + rescue StandardError => e + item.result_queue.push(Fila::Error.new(e.message)) + end + + def flush_multi(items) + req = ::Fila::V1::BatchEnqueueRequest.new( + messages: items.map(&:request) + ) + resp = @stub.batch_enqueue(req, metadata: @metadata) + results = resp.results + + items.each_with_index do |item, idx| + result = results[idx] + if result.nil? + item.result_queue.push(Fila::Error.new('no result from server')) + elsif result.result == :success + item.result_queue.push(result.success.message_id) + else + item.result_queue.push(RPCError.new(GRPC::Core::StatusCodes::INTERNAL, result.error)) + end + end + rescue GRPC::BadStatus => e + # Transport-level failure: all messages in this batch get the error. + err = RPCError.new(e.code, e.details) + items.each { |item| item.result_queue.push(err) } + rescue StandardError => e + err = Fila::Error.new(e.message) + items.each { |item| item.result_queue.push(err) } + end + + def current_time_ms + (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i + end + + # Pop from @queue with a timeout in milliseconds. + # Raises ThreadError if nothing is available within the timeout. + def pop_with_timeout(timeout_ms) + deadline = current_time_ms + timeout_ms + loop do + begin + return @queue.pop(true) + rescue ThreadError + raise if current_time_ms >= deadline + + sleep(0.001) # 1ms polling interval + end + end + end + end +end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 1fdef59..ce4e8a8 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -8,51 +8,133 @@ require_relative 'proto/fila/v1/service_services_pb' require_relative 'errors' require_relative 'consume_message' +require_relative 'batch_enqueue_result' +require_relative 'batcher' module Fila # Client for the Fila message broker. # # Wraps the hot-path gRPC operations: enqueue, consume, ack, nack. # - # @example Plain-text (no auth) + # @example Plain-text, default auto-batching # client = Fila::Client.new("localhost:5555") # + # @example Batching disabled + # client = Fila::Client.new("localhost:5555", batch_mode: :disabled) + # + # @example Linger-based batching + # client = Fila::Client.new("localhost:5555", + # batch_mode: :linger, linger_ms: 10, batch_size: 50) + # # @example TLS with system trust store # client = Fila::Client.new("localhost:5555", tls: true) # - # @example TLS with custom CA - # client = Fila::Client.new("localhost:5555", ca_cert: File.read("ca.pem")) - # # @example mTLS + API key # client = Fila::Client.new("localhost:5555", # ca_cert: File.read("ca.pem"), # client_cert: File.read("client.pem"), # client_key: File.read("client-key.pem"), # api_key: "fila_abc123") - class Client - def initialize(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) + class Client # rubocop:disable Metrics/ClassLength + # Valid batch mode values. + BATCH_MODES = %i[auto linger disabled].freeze + + private_constant :BATCH_MODES + + # @param addr [String] server address (host:port) + # @param tls [Boolean] enable TLS with system trust store + # @param ca_cert [String, nil] PEM-encoded CA certificate + # @param client_cert [String, nil] PEM-encoded client certificate (mTLS) + # @param client_key [String, nil] PEM-encoded client key (mTLS) + # @param api_key [String, nil] API key for authentication + # @param batch_mode [Symbol] :auto (default), :linger, or :disabled + # @param max_batch_size [Integer] max batch size for auto mode (default: 100) + # @param batch_size [Integer] batch size for linger mode (default: 100) + # @param linger_ms [Integer] linger time in ms for linger mode (default: 10) + def initialize( # rubocop:disable Metrics/ParameterLists + addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, + api_key: nil, batch_mode: :auto, max_batch_size: 100, + batch_size: 100, linger_ms: 10 + ) + validate_batch_mode(batch_mode) @api_key = api_key @credentials = build_credentials(tls: tls, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key) @stub = ::Fila::V1::FilaService::Stub.new(addr, @credentials) + @batcher = start_batcher(batch_mode, max_batch_size, batch_size, linger_ms) end - def close; end + # Drain pending batched messages and disconnect. + def close + @batcher&.close + @batcher = nil + end + # Enqueue a message to a queue. + # + # When batching is enabled (default), the message is submitted to + # the background batcher. At low load each message is sent + # individually; at high load messages cluster into batches. + # + # @param queue [String] target queue name + # @param payload [String] message payload + # @param headers [Hash, nil] optional headers + # @return [String] broker-assigned message ID + # @raise [QueueNotFoundError] if the queue does not exist + # @raise [RPCError] for unexpected gRPC failures def enqueue(queue:, payload:, headers: nil) req = ::Fila::V1::EnqueueRequest.new( queue: queue, headers: headers || {}, payload: payload ) - resp = @stub.enqueue(req, metadata: call_metadata) - resp.message_id - rescue GRPC::NotFound => e - raise QueueNotFoundError, "enqueue: #{e.details}" + + if @batcher + @batcher.submit(req) + else + enqueue_direct(req) + end + end + + # Enqueue a batch of messages in a single RPC call. + # + # Each message is independently validated and processed. A failed + # message does not affect the others. Returns an array of + # BatchEnqueueResult with one result per input message, in order. + # + # This bypasses the background batcher and always uses the + # BatchEnqueue RPC directly. + # + # @param messages [Array] messages to enqueue; each hash has + # keys :queue (String), :payload (String), and optionally + # :headers (Hash) + # @return [Array] + # @raise [RPCError] for transport-level gRPC failures + def batch_enqueue(messages) + proto_messages = messages.map do |m| + ::Fila::V1::EnqueueRequest.new( + queue: m[:queue], + headers: m[:headers] || {}, + payload: m[:payload] + ) + end + + req = ::Fila::V1::BatchEnqueueRequest.new(messages: proto_messages) + resp = @stub.batch_enqueue(req, metadata: call_metadata) + + resp.results.map do |r| + if r.result == :success + BatchEnqueueResult.new(message_id: r.success.message_id) + else + BatchEnqueueResult.new(error: r.error) + end + end rescue GRPC::BadStatus => e raise RPCError.new(e.code, e.details) end # Open a streaming consumer. Yields messages as they arrive. + # Transparently unpacks batched delivery (repeated messages field) + # with fallback to singular message field. # Returns an Enumerator if no block given. def consume(queue:, &block) return enum_for(:consume, queue: queue) unless block @@ -99,13 +181,38 @@ def nack(queue:, msg_id:, error:) private - def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/AbcSize + def validate_batch_mode(mode) + return if BATCH_MODES.include?(mode) + + raise ArgumentError, "invalid batch_mode: #{mode.inspect}, must be one of #{BATCH_MODES.inspect}" + end + + def start_batcher(mode, max_batch_size, batch_size, linger_ms) + return nil if mode == :disabled + + Batcher.new( + stub: @stub, + metadata: call_metadata, + mode: mode, + max_batch_size: max_batch_size, + batch_size: batch_size, + linger_ms: linger_ms + ) + end + + def enqueue_direct(req) + resp = @stub.enqueue(req, metadata: call_metadata) + resp.message_id + rescue GRPC::NotFound => e + raise QueueNotFoundError, "enqueue: #{e.details}" + rescue GRPC::BadStatus => e + raise RPCError.new(e.code, e.details) + end + + def consume_with_redirect(queue:, redirected:, &block) stream = @stub.consume(::Fila::V1::ConsumeRequest.new(queue: queue), metadata: call_metadata) stream.each do |resp| - msg = resp.message - next if msg.nil? || msg.id.empty? - - block.call(build_consume_message(msg)) + yield_messages_from_response(resp, &block) end rescue GRPC::Cancelled then nil rescue GRPC::NotFound => e @@ -119,6 +226,25 @@ def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics raise RPCError.new(e.code, e.details) end + # Unpack messages from a ConsumeResponse. Prefers the repeated + # messages field (batched delivery); falls back to singular message + # field for backward compatibility with older servers. + def yield_messages_from_response(resp, &block) + msgs = resp.messages + if msgs && !msgs.empty? + msgs.each do |msg| + next if msg.nil? || msg.id.empty? + + block.call(build_consume_message(msg)) + end + else + msg = resp.message + return if msg.nil? || msg.id.empty? + + block.call(build_consume_message(msg)) + end + end + def extract_leader_addr(err) err.metadata[LEADER_ADDR_KEY] rescue StandardError diff --git a/lib/fila/proto/fila/v1/service_pb.rb b/lib/fila/proto/fila/v1/service_pb.rb index 7eba33f..6120065 100644 --- a/lib/fila/proto/fila/v1/service_pb.rb +++ b/lib/fila/proto/fila/v1/service_pb.rb @@ -7,7 +7,7 @@ require 'fila/v1/messages_pb' -descriptor_data = "\n\x15\x66ila/v1/service.proto\x12\x07\x66ila.v1\x1a\x16\x66ila/v1/messages.proto\"\x97\x01\n\x0e\x45nqueueRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x35\n\x07headers\x18\x02 \x03(\x0b\x32$.fila.v1.EnqueueRequest.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"%\n\x0f\x45nqueueResponse\x12\x12\n\nmessage_id\x18\x01 \x01(\t\"\x1f\n\x0e\x43onsumeRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"4\n\x0f\x43onsumeResponse\x12!\n\x07message\x18\x01 \x01(\x0b\x32\x10.fila.v1.Message\"/\n\nAckRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\"\r\n\x0b\x41\x63kResponse\"?\n\x0bNackRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"\x0e\n\x0cNackResponse2\xf2\x01\n\x0b\x46ilaService\x12<\n\x07\x45nqueue\x12\x17.fila.v1.EnqueueRequest\x1a\x18.fila.v1.EnqueueResponse\x12>\n\x07\x43onsume\x12\x17.fila.v1.ConsumeRequest\x1a\x18.fila.v1.ConsumeResponse0\x01\x12\x30\n\x03\x41\x63k\x12\x13.fila.v1.AckRequest\x1a\x14.fila.v1.AckResponse\x12\x33\n\x04Nack\x12\x14.fila.v1.NackRequest\x1a\x15.fila.v1.NackResponseb\x06proto3" +descriptor_data = "\n\x15\x66ila/v1/service.proto\x12\x07\x66ila.v1\x1a\x16\x66ila/v1/messages.proto\"\x97\x01\n\x0e\x45nqueueRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x35\n\x07headers\x18\x02 \x03(\x0b\x32$.fila.v1.EnqueueRequest.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"%\n\x0f\x45nqueueResponse\x12\x12\n\nmessage_id\x18\x01 \x01(\t\"\x1f\n\x0e\x43onsumeRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"X\n\x0f\x43onsumeResponse\x12!\n\x07message\x18\x01 \x01(\x0b\x32\x10.fila.v1.Message\x12\"\n\x08messages\x18\x02 \x03(\x0b\x32\x10.fila.v1.Message\"/\n\nAckRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\"\r\n\x0b\x41\x63kResponse\"?\n\x0bNackRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"\x0e\n\x0cNackResponse\"@\n\x13\x42\x61tchEnqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueRequest\"D\n\x14\x42\x61tchEnqueueResponse\x12,\n\x07results\x18\x01 \x03(\x0b\x32\x1b.fila.v1.BatchEnqueueResult\"\\\n\x12\x42\x61tchEnqueueResult\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x18.fila.v1.EnqueueResponseH\x00\x12\x0f\n\x05\x65rror\x18\x02 \x01(\tH\x00\x42\x08\n\x06result2\xbf\x02\n\x0b\x46ilaService\x12<\n\x07\x45nqueue\x12\x17.fila.v1.EnqueueRequest\x1a\x18.fila.v1.EnqueueResponse\x12K\n\x0c\x42\x61tchEnqueue\x12\x1c.fila.v1.BatchEnqueueRequest\x1a\x1d.fila.v1.BatchEnqueueResponse\x12>\n\x07\x43onsume\x12\x17.fila.v1.ConsumeRequest\x1a\x18.fila.v1.ConsumeResponse0\x01\x12\x30\n\x03\x41\x63k\x12\x13.fila.v1.AckRequest\x1a\x14.fila.v1.AckResponse\x12\x33\n\x04Nack\x12\x14.fila.v1.NackRequest\x1a\x15.fila.v1.NackResponseb\x06proto3" pool = ::Google::Protobuf::DescriptorPool.generated_pool pool.add_serialized_file(descriptor_data) @@ -22,5 +22,8 @@ module V1 AckResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckResponse").msgclass NackRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackRequest").msgclass NackResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackResponse").msgclass + BatchEnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueRequest").msgclass + BatchEnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueResponse").msgclass + BatchEnqueueResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueResult").msgclass end end diff --git a/lib/fila/proto/fila/v1/service_services_pb.rb b/lib/fila/proto/fila/v1/service_services_pb.rb index 941aba4..0ec0dcb 100644 --- a/lib/fila/proto/fila/v1/service_services_pb.rb +++ b/lib/fila/proto/fila/v1/service_services_pb.rb @@ -17,6 +17,7 @@ class Service self.service_name = 'fila.v1.FilaService' rpc :Enqueue, ::Fila::V1::EnqueueRequest, ::Fila::V1::EnqueueResponse + rpc :BatchEnqueue, ::Fila::V1::BatchEnqueueRequest, ::Fila::V1::BatchEnqueueResponse rpc :Consume, ::Fila::V1::ConsumeRequest, stream(::Fila::V1::ConsumeResponse) rpc :Ack, ::Fila::V1::AckRequest, ::Fila::V1::AckResponse rpc :Nack, ::Fila::V1::NackRequest, ::Fila::V1::NackResponse diff --git a/lib/fila/version.rb b/lib/fila/version.rb index cce5b33..e7c9e84 100644 --- a/lib/fila/version.rb +++ b/lib/fila/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Fila - VERSION = '0.2.0' + VERSION = '0.3.0' end diff --git a/proto/fila/v1/service.proto b/proto/fila/v1/service.proto index f14fdd0..fc0f710 100644 --- a/proto/fila/v1/service.proto +++ b/proto/fila/v1/service.proto @@ -6,6 +6,7 @@ import "fila/v1/messages.proto"; // Hot-path RPCs for producers and consumers. service FilaService { rpc Enqueue(EnqueueRequest) returns (EnqueueResponse); + rpc BatchEnqueue(BatchEnqueueRequest) returns (BatchEnqueueResponse); rpc Consume(ConsumeRequest) returns (stream ConsumeResponse); rpc Ack(AckRequest) returns (AckResponse); rpc Nack(NackRequest) returns (NackResponse); @@ -26,7 +27,8 @@ message ConsumeRequest { } message ConsumeResponse { - Message message = 1; + Message message = 1; // Single message (backward compatible, used when batch size is 1) + repeated Message messages = 2; // Batched messages (populated when server sends multiple at once) } message AckRequest { @@ -43,3 +45,18 @@ message NackRequest { } message NackResponse {} + +message BatchEnqueueRequest { + repeated EnqueueRequest messages = 1; +} + +message BatchEnqueueResponse { + repeated BatchEnqueueResult results = 1; +} + +message BatchEnqueueResult { + oneof result { + EnqueueResponse success = 1; + string error = 2; + } +} diff --git a/test/test_batch.rb b/test/test_batch.rb new file mode 100644 index 0000000..5a8778d --- /dev/null +++ b/test/test_batch.rb @@ -0,0 +1,273 @@ +# frozen_string_literal: true + +require 'test_helper' + +return unless FILA_SERVER_AVAILABLE + +class TestBatchEnqueue < Minitest::Test + def setup + @server = TestServerHelper.start + @client = Fila::Client.new(@server[:addr], batch_mode: :disabled) + end + + def teardown + @client&.close + TestServerHelper.stop(@server) if @server + end + + def test_batch_enqueue_multiple_messages + TestServerHelper.create_queue(@server, 'batch-test') + + messages = 5.times.map do |i| + { queue: 'batch-test', payload: "batch-msg-#{i}", headers: { 'index' => i.to_s } } + end + + results = @client.batch_enqueue(messages) + + assert_equal 5, results.size + results.each do |r| + assert r.success?, "expected success but got error: #{r.error}" + refute_empty r.message_id + end + + # Verify all messages are consumable. + received_ids = [] + @client.consume(queue: 'batch-test') do |msg| + received_ids << msg.id + @client.ack(queue: 'batch-test', msg_id: msg.id) + break if received_ids.size >= 5 + end + assert_equal 5, received_ids.size + end + + def test_batch_enqueue_single_message + TestServerHelper.create_queue(@server, 'batch-single') + + results = @client.batch_enqueue([ + { queue: 'batch-single', payload: 'solo' } + ]) + + assert_equal 1, results.size + assert results.first.success? + refute_empty results.first.message_id + end + + def test_batch_enqueue_empty_array + results = @client.batch_enqueue([]) + assert_equal 0, results.size + end + + def test_batch_enqueue_mixed_success_and_failure + TestServerHelper.create_queue(@server, 'batch-mixed') + + messages = [ + { queue: 'batch-mixed', payload: 'good-1' }, + { queue: 'no-such-queue-xyz', payload: 'bad' }, + { queue: 'batch-mixed', payload: 'good-2' } + ] + + results = @client.batch_enqueue(messages) + assert_equal 3, results.size + + assert results[0].success?, "first message should succeed" + refute results[1].success?, "second message should fail (nonexistent queue)" + assert results[1].error, "second message should have error description" + assert results[2].success?, "third message should succeed" + end +end + +class TestBatchEnqueueResult < Minitest::Test + def test_success_result + r = Fila::BatchEnqueueResult.new(message_id: 'abc-123') + assert r.success? + assert_equal 'abc-123', r.message_id + assert_nil r.error + end + + def test_error_result + r = Fila::BatchEnqueueResult.new(error: 'queue not found') + refute r.success? + assert_nil r.message_id + assert_equal 'queue not found', r.error + end +end + +class TestAutoBatching < Minitest::Test + def setup + @server = TestServerHelper.start + # Default batch_mode is :auto + @client = Fila::Client.new(@server[:addr]) + end + + def teardown + @client&.close + TestServerHelper.stop(@server) if @server + end + + def test_auto_batch_enqueue_single + TestServerHelper.create_queue(@server, 'auto-single') + + msg_id = @client.enqueue(queue: 'auto-single', payload: 'auto-msg') + assert msg_id + refute_empty msg_id + + received = false + @client.consume(queue: 'auto-single') do |msg| + assert_equal msg_id, msg.id + assert_equal 'auto-msg', msg.payload + @client.ack(queue: 'auto-single', msg_id: msg.id) + received = true + break + end + assert received + end + + def test_auto_batch_enqueue_concurrent + TestServerHelper.create_queue(@server, 'auto-concurrent') + + # Fire multiple enqueues concurrently to exercise batching. + threads = 10.times.map do |i| + Thread.new do + @client.enqueue(queue: 'auto-concurrent', payload: "msg-#{i}") + end + end + ids = threads.map(&:value) + + assert_equal 10, ids.size + ids.each do |id| + assert id + refute_empty id + end + + # Consume all messages. + received = [] + @client.consume(queue: 'auto-concurrent') do |msg| + received << msg.id + @client.ack(queue: 'auto-concurrent', msg_id: msg.id) + break if received.size >= 10 + end + assert_equal 10, received.size + end + + def test_auto_batch_nonexistent_queue_raises + assert_raises(Fila::QueueNotFoundError) do + @client.enqueue(queue: 'no-such-queue-auto', payload: 'fail') + end + end +end + +class TestLingerBatching < Minitest::Test + def setup + @server = TestServerHelper.start + @client = Fila::Client.new(@server[:addr], batch_mode: :linger, linger_ms: 50, batch_size: 10) + end + + def teardown + @client&.close + TestServerHelper.stop(@server) if @server + end + + def test_linger_batch_enqueue + TestServerHelper.create_queue(@server, 'linger-test') + + msg_id = @client.enqueue(queue: 'linger-test', payload: 'linger-msg') + assert msg_id + refute_empty msg_id + end + + def test_linger_batch_concurrent + TestServerHelper.create_queue(@server, 'linger-concurrent') + + threads = 5.times.map do |i| + Thread.new do + @client.enqueue(queue: 'linger-concurrent', payload: "linger-#{i}") + end + end + ids = threads.map(&:value) + + assert_equal 5, ids.size + ids.each { |id| refute_empty id } + end +end + +class TestDisabledBatching < Minitest::Test + def setup + @server = TestServerHelper.start + @client = Fila::Client.new(@server[:addr], batch_mode: :disabled) + end + + def teardown + @client&.close + TestServerHelper.stop(@server) if @server + end + + def test_disabled_batch_enqueue_direct + TestServerHelper.create_queue(@server, 'disabled-test') + + msg_id = @client.enqueue(queue: 'disabled-test', payload: 'direct-msg') + assert msg_id + refute_empty msg_id + end + + def test_disabled_nonexistent_queue_raises + assert_raises(Fila::QueueNotFoundError) do + @client.enqueue(queue: 'no-such-queue-disabled', payload: 'fail') + end + end +end + +class TestBatchModeValidation < Minitest::Test + def test_invalid_batch_mode_raises + assert_raises(ArgumentError) do + Fila::Client.new('localhost:5555', batch_mode: :invalid) + end + end + + def test_valid_batch_modes_accepted + # These should not raise (but won't connect since server isn't on this port). + # Just verify argument validation passes. + %i[auto linger disabled].each do |mode| + client = Fila::Client.new('localhost:19999', batch_mode: mode) + client.close + end + end +end + +class TestCloseFlush < Minitest::Test + def setup + @server = TestServerHelper.start + end + + def teardown + TestServerHelper.stop(@server) if @server + end + + def test_close_drains_pending_messages + TestServerHelper.create_queue(@server, 'close-drain') + client = Fila::Client.new(@server[:addr]) + + # Enqueue a message, then close immediately. + msg_id = client.enqueue(queue: 'close-drain', payload: 'drain-me') + refute_empty msg_id + + client.close + + # Verify the message was persisted. + verify_client = Fila::Client.new(@server[:addr], batch_mode: :disabled) + received = false + verify_client.consume(queue: 'close-drain') do |msg| + assert_equal msg_id, msg.id + verify_client.ack(queue: 'close-drain', msg_id: msg.id) + received = true + break + end + assert received + verify_client.close + end + + def test_double_close_is_safe + client = Fila::Client.new(@server[:addr]) + client.close + client.close # Should not raise. + end +end diff --git a/test/test_client.rb b/test/test_client.rb index 7d0a01f..f39fe44 100644 --- a/test/test_client.rb +++ b/test/test_client.rb @@ -11,6 +11,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server end From 8bb8a0943d4df61b6a6acefa14a8f3fc8c04fba5 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 25 Mar 2026 00:07:59 -0300 Subject: [PATCH 11/19] feat: unified api surface for story 30.2 - copy new service.proto (BatchEnqueue RPC removed, EnqueueRequest now takes repeated EnqueueMessage, AckRequest/NackRequest take repeated messages, ConsumeResponse only has repeated messages field) - regenerate ruby proto code from new service.proto - replace batch_enqueue method with enqueue_many (no "batch" prefix) - enqueue wraps single message in EnqueueMessage + EnqueueRequest - ack/nack wrap in repeated AckMessage/NackMessage, parse first result - consume uses only repeated messages field (singular field removed) - rename BatchEnqueueResult to EnqueueResult - batcher uses unified Enqueue RPC for all batch sizes - update all tests to match new api surface - bump version to 0.4.0 --- lib/fila.rb | 2 +- lib/fila/batcher.rb | 55 +++------ lib/fila/client.rb | 110 ++++++++++------- ...ch_enqueue_result.rb => enqueue_result.rb} | 8 +- lib/fila/proto/fila/v1/service_pb.rb | 21 +++- lib/fila/proto/fila/v1/service_services_pb.rb | 2 +- lib/fila/version.rb | 2 +- proto/fila/v1/service.proto | 116 +++++++++++++++--- test/test_batch.rb | 42 +++---- 9 files changed, 228 insertions(+), 130 deletions(-) rename lib/fila/{batch_enqueue_result.rb => enqueue_result.rb} (79%) diff --git a/lib/fila.rb b/lib/fila.rb index 977a9a9..3a74a1b 100644 --- a/lib/fila.rb +++ b/lib/fila.rb @@ -3,6 +3,6 @@ require_relative 'fila/version' require_relative 'fila/errors' require_relative 'fila/consume_message' -require_relative 'fila/batch_enqueue_result' +require_relative 'fila/enqueue_result' require_relative 'fila/batcher' require_relative 'fila/client' diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 577f4d4..254f7de 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true module Fila - # Background batcher that collects enqueue requests and flushes them - # in batches via BatchEnqueue RPC. Supports auto (opportunistic) and - # linger (timer-based) modes. + # Background batcher that collects enqueue messages and flushes them + # in batches via the unified Enqueue RPC. Supports auto (opportunistic) + # and linger (timer-based) modes. # # @api private class Batcher # An item queued for batching, pairing a message with its result slot. - BatchItem = Struct.new(:request, :result_queue, keyword_init: true) + BatchItem = Struct.new(:message, :result_queue, keyword_init: true) # @param stub [Fila::V1::FilaService::Stub] gRPC stub # @param metadata [Hash] call metadata (auth headers) @@ -33,13 +33,13 @@ def initialize(stub:, metadata:, mode:, max_batch_size: 100, batch_size: 100, li # Submit a message for batched sending. Blocks until the batch # containing this message is flushed and the result is available. # - # @param request [Fila::V1::EnqueueRequest] the enqueue request + # @param message [Fila::V1::EnqueueMessage] the enqueue message # @return [String] message ID on success # @raise [Fila::QueueNotFoundError] if the queue does not exist # @raise [Fila::RPCError] for unexpected gRPC failures - def submit(request) + def submit(message) result_queue = Queue.new - item = BatchItem.new(request: request, result_queue: result_queue) + item = BatchItem.new(message: message, result_queue: result_queue) @mutex.synchronize do raise Fila::Error, 'batcher is closed' if @stopped @@ -128,43 +128,28 @@ def drain_nonblocking(batch) end end - # Flush a batch of items. Uses single Enqueue RPC for 1 message - # (preserves exact error types like QueueNotFoundError), and - # BatchEnqueue RPC for 2+ messages. + # Flush a batch of items via the unified Enqueue RPC. def flush_batch(items) - if items.size == 1 - flush_single(items.first) - else - flush_multi(items) - end - end - - def flush_single(item) - resp = @stub.enqueue(item.request, metadata: @metadata) - item.result_queue.push(resp.message_id) - rescue GRPC::NotFound => e - item.result_queue.push(QueueNotFoundError.new("enqueue: #{e.details}")) - rescue GRPC::BadStatus => e - item.result_queue.push(RPCError.new(e.code, e.details)) - rescue StandardError => e - item.result_queue.push(Fila::Error.new(e.message)) - end - - def flush_multi(items) - req = ::Fila::V1::BatchEnqueueRequest.new( - messages: items.map(&:request) + req = ::Fila::V1::EnqueueRequest.new( + messages: items.map(&:message) ) - resp = @stub.batch_enqueue(req, metadata: @metadata) + resp = @stub.enqueue(req, metadata: @metadata) results = resp.results items.each_with_index do |item, idx| result = results[idx] if result.nil? item.result_queue.push(Fila::Error.new('no result from server')) - elsif result.result == :success - item.result_queue.push(result.success.message_id) + elsif result.result == :message_id + item.result_queue.push(result.message_id) else - item.result_queue.push(RPCError.new(GRPC::Core::StatusCodes::INTERNAL, result.error)) + err = result.error + case err.code + when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND + item.result_queue.push(QueueNotFoundError.new("enqueue: #{err.message}")) + else + item.result_queue.push(RPCError.new(GRPC::Core::StatusCodes::INTERNAL, err.message)) + end end end rescue GRPC::BadStatus => e diff --git a/lib/fila/client.rb b/lib/fila/client.rb index ce4e8a8..e3cb12e 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -8,7 +8,7 @@ require_relative 'proto/fila/v1/service_services_pb' require_relative 'errors' require_relative 'consume_message' -require_relative 'batch_enqueue_result' +require_relative 'enqueue_result' require_relative 'batcher' module Fila @@ -69,7 +69,7 @@ def close @batcher = nil end - # Enqueue a message to a queue. + # Enqueue a single message to a queue. # # When batching is enabled (default), the message is submitted to # the background batcher. At low load each message is sent @@ -82,50 +82,50 @@ def close # @raise [QueueNotFoundError] if the queue does not exist # @raise [RPCError] for unexpected gRPC failures def enqueue(queue:, payload:, headers: nil) - req = ::Fila::V1::EnqueueRequest.new( + msg = ::Fila::V1::EnqueueMessage.new( queue: queue, headers: headers || {}, payload: payload ) if @batcher - @batcher.submit(req) + @batcher.submit(msg) else - enqueue_direct(req) + enqueue_single(msg) end end - # Enqueue a batch of messages in a single RPC call. + # Enqueue multiple messages in a single RPC call. # # Each message is independently validated and processed. A failed # message does not affect the others. Returns an array of - # BatchEnqueueResult with one result per input message, in order. + # EnqueueResult with one result per input message, in order. # # This bypasses the background batcher and always uses the - # BatchEnqueue RPC directly. + # Enqueue RPC directly. # # @param messages [Array] messages to enqueue; each hash has # keys :queue (String), :payload (String), and optionally # :headers (Hash) - # @return [Array] + # @return [Array] # @raise [RPCError] for transport-level gRPC failures - def batch_enqueue(messages) + def enqueue_many(messages) proto_messages = messages.map do |m| - ::Fila::V1::EnqueueRequest.new( + ::Fila::V1::EnqueueMessage.new( queue: m[:queue], headers: m[:headers] || {}, payload: m[:payload] ) end - req = ::Fila::V1::BatchEnqueueRequest.new(messages: proto_messages) - resp = @stub.batch_enqueue(req, metadata: call_metadata) + req = ::Fila::V1::EnqueueRequest.new(messages: proto_messages) + resp = @stub.enqueue(req, metadata: call_metadata) resp.results.map do |r| - if r.result == :success - BatchEnqueueResult.new(message_id: r.success.message_id) + if r.result == :message_id + EnqueueResult.new(message_id: r.message_id) else - BatchEnqueueResult.new(error: r.error) + EnqueueResult.new(error: r.error.message) end end rescue GRPC::BadStatus => e @@ -133,8 +133,6 @@ def batch_enqueue(messages) end # Open a streaming consumer. Yields messages as they arrive. - # Transparently unpacks batched delivery (repeated messages field) - # with fallback to singular message field. # Returns an Enumerator if no block given. def consume(queue:, &block) return enum_for(:consume, queue: queue) unless block @@ -149,11 +147,20 @@ def consume(queue:, &block) # @raise [MessageNotFoundError] if the message does not exist # @raise [RPCError] for unexpected gRPC failures def ack(queue:, msg_id:) - req = ::Fila::V1::AckRequest.new(queue: queue, message_id: msg_id) - @stub.ack(req, metadata: call_metadata) - nil - rescue GRPC::NotFound => e - raise MessageNotFoundError, "ack: #{e.details}" + msg = ::Fila::V1::AckMessage.new(queue: queue, message_id: msg_id) + req = ::Fila::V1::AckRequest.new(messages: [msg]) + resp = @stub.ack(req, metadata: call_metadata) + + result = resp.results.first + return if result.nil? || result.result == :success + + err = result.error + case err.code + when :ACK_ERROR_CODE_MESSAGE_NOT_FOUND + raise MessageNotFoundError, "ack: #{err.message}" + else + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "ack: #{err.message}") + end rescue GRPC::BadStatus => e raise RPCError.new(e.code, e.details) end @@ -166,11 +173,20 @@ def ack(queue:, msg_id:) # @raise [MessageNotFoundError] if the message does not exist # @raise [RPCError] for unexpected gRPC failures def nack(queue:, msg_id:, error:) - req = ::Fila::V1::NackRequest.new(queue: queue, message_id: msg_id, error: error) - @stub.nack(req, metadata: call_metadata) - nil - rescue GRPC::NotFound => e - raise MessageNotFoundError, "nack: #{e.details}" + msg = ::Fila::V1::NackMessage.new(queue: queue, message_id: msg_id, error: error) + req = ::Fila::V1::NackRequest.new(messages: [msg]) + resp = @stub.nack(req, metadata: call_metadata) + + result = resp.results.first + return if result.nil? || result.result == :success + + err = result.error + case err.code + when :NACK_ERROR_CODE_MESSAGE_NOT_FOUND + raise MessageNotFoundError, "nack: #{err.message}" + else + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "nack: #{err.message}") + end rescue GRPC::BadStatus => e raise RPCError.new(e.code, e.details) end @@ -200,11 +216,25 @@ def start_batcher(mode, max_batch_size, batch_size, linger_ms) ) end - def enqueue_direct(req) + # Send a single message via the unified Enqueue RPC. + def enqueue_single(msg) + req = ::Fila::V1::EnqueueRequest.new(messages: [msg]) resp = @stub.enqueue(req, metadata: call_metadata) - resp.message_id - rescue GRPC::NotFound => e - raise QueueNotFoundError, "enqueue: #{e.details}" + + result = resp.results.first + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? + + if result.result == :message_id + result.message_id + else + err = result.error + case err.code + when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND + raise QueueNotFoundError, "enqueue: #{err.message}" + else + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "enqueue: #{err.message}") + end + end rescue GRPC::BadStatus => e raise RPCError.new(e.code, e.details) end @@ -226,20 +256,10 @@ def consume_with_redirect(queue:, redirected:, &block) raise RPCError.new(e.code, e.details) end - # Unpack messages from a ConsumeResponse. Prefers the repeated - # messages field (batched delivery); falls back to singular message - # field for backward compatibility with older servers. + # Unpack messages from a ConsumeResponse. def yield_messages_from_response(resp, &block) - msgs = resp.messages - if msgs && !msgs.empty? - msgs.each do |msg| - next if msg.nil? || msg.id.empty? - - block.call(build_consume_message(msg)) - end - else - msg = resp.message - return if msg.nil? || msg.id.empty? + resp.messages.each do |msg| + next if msg.nil? || msg.id.empty? block.call(build_consume_message(msg)) end diff --git a/lib/fila/batch_enqueue_result.rb b/lib/fila/enqueue_result.rb similarity index 79% rename from lib/fila/batch_enqueue_result.rb rename to lib/fila/enqueue_result.rb index be9cf13..ce8b3db 100644 --- a/lib/fila/batch_enqueue_result.rb +++ b/lib/fila/enqueue_result.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true module Fila - # Result of a single message within a batch enqueue call. + # Result of a single message within an enqueue_many call. # - # Each message in a batch is independently validated and processed. + # Each message is independently validated and processed. # A failed message does not affect the others. # # @example - # results = client.batch_enqueue(messages) + # results = client.enqueue_many(messages) # results.each do |r| # if r.success? # puts "Enqueued: #{r.message_id}" @@ -15,7 +15,7 @@ module Fila # puts "Failed: #{r.error}" # end # end - class BatchEnqueueResult + class EnqueueResult # @return [String, nil] broker-assigned message ID on success attr_reader :message_id diff --git a/lib/fila/proto/fila/v1/service_pb.rb b/lib/fila/proto/fila/v1/service_pb.rb index 6120065..9751f67 100644 --- a/lib/fila/proto/fila/v1/service_pb.rb +++ b/lib/fila/proto/fila/v1/service_pb.rb @@ -7,23 +7,36 @@ require 'fila/v1/messages_pb' -descriptor_data = "\n\x15\x66ila/v1/service.proto\x12\x07\x66ila.v1\x1a\x16\x66ila/v1/messages.proto\"\x97\x01\n\x0e\x45nqueueRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x35\n\x07headers\x18\x02 \x03(\x0b\x32$.fila.v1.EnqueueRequest.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"%\n\x0f\x45nqueueResponse\x12\x12\n\nmessage_id\x18\x01 \x01(\t\"\x1f\n\x0e\x43onsumeRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"X\n\x0f\x43onsumeResponse\x12!\n\x07message\x18\x01 \x01(\x0b\x32\x10.fila.v1.Message\x12\"\n\x08messages\x18\x02 \x03(\x0b\x32\x10.fila.v1.Message\"/\n\nAckRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\"\r\n\x0b\x41\x63kResponse\"?\n\x0bNackRequest\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"\x0e\n\x0cNackResponse\"@\n\x13\x42\x61tchEnqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueRequest\"D\n\x14\x42\x61tchEnqueueResponse\x12,\n\x07results\x18\x01 \x03(\x0b\x32\x1b.fila.v1.BatchEnqueueResult\"\\\n\x12\x42\x61tchEnqueueResult\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x18.fila.v1.EnqueueResponseH\x00\x12\x0f\n\x05\x65rror\x18\x02 \x01(\tH\x00\x42\x08\n\x06result2\xbf\x02\n\x0b\x46ilaService\x12<\n\x07\x45nqueue\x12\x17.fila.v1.EnqueueRequest\x1a\x18.fila.v1.EnqueueResponse\x12K\n\x0c\x42\x61tchEnqueue\x12\x1c.fila.v1.BatchEnqueueRequest\x1a\x1d.fila.v1.BatchEnqueueResponse\x12>\n\x07\x43onsume\x12\x17.fila.v1.ConsumeRequest\x1a\x18.fila.v1.ConsumeResponse0\x01\x12\x30\n\x03\x41\x63k\x12\x13.fila.v1.AckRequest\x1a\x14.fila.v1.AckResponse\x12\x33\n\x04Nack\x12\x14.fila.v1.NackRequest\x1a\x15.fila.v1.NackResponseb\x06proto3" +descriptor_data = "\n\x15\x66ila/v1/service.proto\x12\x07\x66ila.v1\x1a\x16\x66ila/v1/messages.proto\"\x97\x01\n\x0e\x45nqueueMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x35\n\x07headers\x18\x02 \x03(\x0b\x32$.fila.v1.EnqueueMessage.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\";\n\x0e\x45nqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueMessage\"W\n\rEnqueueResult\x12\x14\n\nmessage_id\x18\x01 \x01(\tH\x00\x12&\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x15.fila.v1.EnqueueErrorH\x00\x42\x08\n\x06result\"H\n\x0c\x45nqueueError\x12\'\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x19.fila.v1.EnqueueErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x0f\x45nqueueResponse\x12\'\n\x07results\x18\x01 \x03(\x0b\x32\x16.fila.v1.EnqueueResult\"\x1f\n\x0e\x43onsumeRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"5\n\x0f\x43onsumeResponse\x12\"\n\x08messages\x18\x01 \x03(\x0b\x32\x10.fila.v1.Message\"/\n\nAckMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\"3\n\nAckRequest\x12%\n\x08messages\x18\x01 \x03(\x0b\x32\x13.fila.v1.AckMessage\"a\n\tAckResult\x12&\n\x07success\x18\x01 \x01(\x0b\x32\x13.fila.v1.AckSuccessH\x00\x12\"\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x11.fila.v1.AckErrorH\x00\x42\x08\n\x06result\"\x0c\n\nAckSuccess\"@\n\x08\x41\x63kError\x12#\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x15.fila.v1.AckErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0b\x41\x63kResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.fila.v1.AckResult\"?\n\x0bNackMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"5\n\x0bNackRequest\x12&\n\x08messages\x18\x01 \x03(\x0b\x32\x14.fila.v1.NackMessage\"d\n\nNackResult\x12\'\n\x07success\x18\x01 \x01(\x0b\x32\x14.fila.v1.NackSuccessH\x00\x12#\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x12.fila.v1.NackErrorH\x00\x42\x08\n\x06result\"\r\n\x0bNackSuccess\"B\n\tNackError\x12$\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x16.fila.v1.NackErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"4\n\x0cNackResponse\x12$\n\x07results\x18\x01 \x03(\x0b\x32\x13.fila.v1.NackResult\"Z\n\x14StreamEnqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueMessage\x12\x17\n\x0fsequence_number\x18\x02 \x01(\x04\"Y\n\x15StreamEnqueueResponse\x12\x17\n\x0fsequence_number\x18\x01 \x01(\x04\x12\'\n\x07results\x18\x02 \x03(\x0b\x32\x16.fila.v1.EnqueueResult*\xc4\x01\n\x10\x45nqueueErrorCode\x12\"\n\x1e\x45NQUEUE_ERROR_CODE_UNSPECIFIED\x10\x00\x12&\n\"ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND\x10\x01\x12\x1e\n\x1a\x45NQUEUE_ERROR_CODE_STORAGE\x10\x02\x12\x1a\n\x16\x45NQUEUE_ERROR_CODE_LUA\x10\x03\x12(\n$ENQUEUE_ERROR_CODE_PERMISSION_DENIED\x10\x04*\x96\x01\n\x0c\x41\x63kErrorCode\x12\x1e\n\x1a\x41\x43K_ERROR_CODE_UNSPECIFIED\x10\x00\x12$\n ACK_ERROR_CODE_MESSAGE_NOT_FOUND\x10\x01\x12\x1a\n\x16\x41\x43K_ERROR_CODE_STORAGE\x10\x02\x12$\n ACK_ERROR_CODE_PERMISSION_DENIED\x10\x03*\x9b\x01\n\rNackErrorCode\x12\x1f\n\x1bNACK_ERROR_CODE_UNSPECIFIED\x10\x00\x12%\n!NACK_ERROR_CODE_MESSAGE_NOT_FOUND\x10\x01\x12\x1b\n\x17NACK_ERROR_CODE_STORAGE\x10\x02\x12%\n!NACK_ERROR_CODE_PERMISSION_DENIED\x10\x03\x32\xc6\x02\n\x0b\x46ilaService\x12<\n\x07\x45nqueue\x12\x17.fila.v1.EnqueueRequest\x1a\x18.fila.v1.EnqueueResponse\x12R\n\rStreamEnqueue\x12\x1d.fila.v1.StreamEnqueueRequest\x1a\x1e.fila.v1.StreamEnqueueResponse(\x01\x30\x01\x12>\n\x07\x43onsume\x12\x17.fila.v1.ConsumeRequest\x1a\x18.fila.v1.ConsumeResponse0\x01\x12\x30\n\x03\x41\x63k\x12\x13.fila.v1.AckRequest\x1a\x14.fila.v1.AckResponse\x12\x33\n\x04Nack\x12\x14.fila.v1.NackRequest\x1a\x15.fila.v1.NackResponseb\x06proto3" pool = ::Google::Protobuf::DescriptorPool.generated_pool pool.add_serialized_file(descriptor_data) module Fila module V1 + EnqueueMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueMessage").msgclass EnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueRequest").msgclass + EnqueueResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueResult").msgclass + EnqueueError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueError").msgclass EnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueResponse").msgclass ConsumeRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ConsumeRequest").msgclass ConsumeResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ConsumeResponse").msgclass + AckMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckMessage").msgclass AckRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckRequest").msgclass + AckResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckResult").msgclass + AckSuccess = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckSuccess").msgclass + AckError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckError").msgclass AckResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckResponse").msgclass + NackMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackMessage").msgclass NackRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackRequest").msgclass + NackResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackResult").msgclass + NackSuccess = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackSuccess").msgclass + NackError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackError").msgclass NackResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackResponse").msgclass - BatchEnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueRequest").msgclass - BatchEnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueResponse").msgclass - BatchEnqueueResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.BatchEnqueueResult").msgclass + StreamEnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.StreamEnqueueRequest").msgclass + StreamEnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.StreamEnqueueResponse").msgclass + EnqueueErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueErrorCode").enummodule + AckErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckErrorCode").enummodule + NackErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackErrorCode").enummodule end end diff --git a/lib/fila/proto/fila/v1/service_services_pb.rb b/lib/fila/proto/fila/v1/service_services_pb.rb index 0ec0dcb..93d38ab 100644 --- a/lib/fila/proto/fila/v1/service_services_pb.rb +++ b/lib/fila/proto/fila/v1/service_services_pb.rb @@ -17,7 +17,7 @@ class Service self.service_name = 'fila.v1.FilaService' rpc :Enqueue, ::Fila::V1::EnqueueRequest, ::Fila::V1::EnqueueResponse - rpc :BatchEnqueue, ::Fila::V1::BatchEnqueueRequest, ::Fila::V1::BatchEnqueueResponse + rpc :StreamEnqueue, stream(::Fila::V1::StreamEnqueueRequest), stream(::Fila::V1::StreamEnqueueResponse) rpc :Consume, ::Fila::V1::ConsumeRequest, stream(::Fila::V1::ConsumeResponse) rpc :Ack, ::Fila::V1::AckRequest, ::Fila::V1::AckResponse rpc :Nack, ::Fila::V1::NackRequest, ::Fila::V1::NackResponse diff --git a/lib/fila/version.rb b/lib/fila/version.rb index e7c9e84..ecc3d06 100644 --- a/lib/fila/version.rb +++ b/lib/fila/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Fila - VERSION = '0.3.0' + VERSION = '0.4.0' end diff --git a/proto/fila/v1/service.proto b/proto/fila/v1/service.proto index fc0f710..7d1db79 100644 --- a/proto/fila/v1/service.proto +++ b/proto/fila/v1/service.proto @@ -6,20 +6,49 @@ import "fila/v1/messages.proto"; // Hot-path RPCs for producers and consumers. service FilaService { rpc Enqueue(EnqueueRequest) returns (EnqueueResponse); - rpc BatchEnqueue(BatchEnqueueRequest) returns (BatchEnqueueResponse); + rpc StreamEnqueue(stream StreamEnqueueRequest) returns (stream StreamEnqueueResponse); rpc Consume(ConsumeRequest) returns (stream ConsumeResponse); rpc Ack(AckRequest) returns (AckResponse); rpc Nack(NackRequest) returns (NackResponse); } -message EnqueueRequest { +// Individual message to enqueue. +message EnqueueMessage { string queue = 1; map headers = 2; bytes payload = 3; } +// Enqueue one or more messages. +message EnqueueRequest { + repeated EnqueueMessage messages = 1; +} + +// Per-message enqueue result. +message EnqueueResult { + oneof result { + string message_id = 1; + EnqueueError error = 2; + } +} + +// Typed enqueue error with structured error code. +message EnqueueError { + EnqueueErrorCode code = 1; + string message = 2; +} + +enum EnqueueErrorCode { + ENQUEUE_ERROR_CODE_UNSPECIFIED = 0; + ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND = 1; + ENQUEUE_ERROR_CODE_STORAGE = 2; + ENQUEUE_ERROR_CODE_LUA = 3; + ENQUEUE_ERROR_CODE_PERMISSION_DENIED = 4; +} + +// One result per input message. message EnqueueResponse { - string message_id = 1; + repeated EnqueueResult results = 1; } message ConsumeRequest { @@ -27,36 +56,87 @@ message ConsumeRequest { } message ConsumeResponse { - Message message = 1; // Single message (backward compatible, used when batch size is 1) - repeated Message messages = 2; // Batched messages (populated when server sends multiple at once) + repeated Message messages = 1; } -message AckRequest { +// Individual ack item. +message AckMessage { string queue = 1; string message_id = 2; } -message AckResponse {} +message AckRequest { + repeated AckMessage messages = 1; +} + +message AckResult { + oneof result { + AckSuccess success = 1; + AckError error = 2; + } +} -message NackRequest { +message AckSuccess {} + +message AckError { + AckErrorCode code = 1; + string message = 2; +} + +enum AckErrorCode { + ACK_ERROR_CODE_UNSPECIFIED = 0; + ACK_ERROR_CODE_MESSAGE_NOT_FOUND = 1; + ACK_ERROR_CODE_STORAGE = 2; + ACK_ERROR_CODE_PERMISSION_DENIED = 3; +} + +message AckResponse { + repeated AckResult results = 1; +} + +// Individual nack item. +message NackMessage { string queue = 1; string message_id = 2; string error = 3; } -message NackResponse {} +message NackRequest { + repeated NackMessage messages = 1; +} + +message NackResult { + oneof result { + NackSuccess success = 1; + NackError error = 2; + } +} -message BatchEnqueueRequest { - repeated EnqueueRequest messages = 1; +message NackSuccess {} + +message NackError { + NackErrorCode code = 1; + string message = 2; } -message BatchEnqueueResponse { - repeated BatchEnqueueResult results = 1; +enum NackErrorCode { + NACK_ERROR_CODE_UNSPECIFIED = 0; + NACK_ERROR_CODE_MESSAGE_NOT_FOUND = 1; + NACK_ERROR_CODE_STORAGE = 2; + NACK_ERROR_CODE_PERMISSION_DENIED = 3; } -message BatchEnqueueResult { - oneof result { - EnqueueResponse success = 1; - string error = 2; - } +message NackResponse { + repeated NackResult results = 1; +} + +// Stream enqueue — per-write batch with sequence tracking. +message StreamEnqueueRequest { + repeated EnqueueMessage messages = 1; + uint64 sequence_number = 2; +} + +message StreamEnqueueResponse { + uint64 sequence_number = 1; + repeated EnqueueResult results = 2; } diff --git a/test/test_batch.rb b/test/test_batch.rb index 5a8778d..5bdb4fb 100644 --- a/test/test_batch.rb +++ b/test/test_batch.rb @@ -4,7 +4,7 @@ return unless FILA_SERVER_AVAILABLE -class TestBatchEnqueue < Minitest::Test +class TestEnqueueMany < Minitest::Test def setup @server = TestServerHelper.start @client = Fila::Client.new(@server[:addr], batch_mode: :disabled) @@ -15,14 +15,14 @@ def teardown TestServerHelper.stop(@server) if @server end - def test_batch_enqueue_multiple_messages - TestServerHelper.create_queue(@server, 'batch-test') + def test_enqueue_many_multiple_messages + TestServerHelper.create_queue(@server, 'many-test') messages = 5.times.map do |i| - { queue: 'batch-test', payload: "batch-msg-#{i}", headers: { 'index' => i.to_s } } + { queue: 'many-test', payload: "many-msg-#{i}", headers: { 'index' => i.to_s } } end - results = @client.batch_enqueue(messages) + results = @client.enqueue_many(messages) assert_equal 5, results.size results.each do |r| @@ -32,19 +32,19 @@ def test_batch_enqueue_multiple_messages # Verify all messages are consumable. received_ids = [] - @client.consume(queue: 'batch-test') do |msg| + @client.consume(queue: 'many-test') do |msg| received_ids << msg.id - @client.ack(queue: 'batch-test', msg_id: msg.id) + @client.ack(queue: 'many-test', msg_id: msg.id) break if received_ids.size >= 5 end assert_equal 5, received_ids.size end - def test_batch_enqueue_single_message - TestServerHelper.create_queue(@server, 'batch-single') + def test_enqueue_many_single_message + TestServerHelper.create_queue(@server, 'many-single') - results = @client.batch_enqueue([ - { queue: 'batch-single', payload: 'solo' } + results = @client.enqueue_many([ + { queue: 'many-single', payload: 'solo' } ]) assert_equal 1, results.size @@ -52,21 +52,21 @@ def test_batch_enqueue_single_message refute_empty results.first.message_id end - def test_batch_enqueue_empty_array - results = @client.batch_enqueue([]) + def test_enqueue_many_empty_array + results = @client.enqueue_many([]) assert_equal 0, results.size end - def test_batch_enqueue_mixed_success_and_failure - TestServerHelper.create_queue(@server, 'batch-mixed') + def test_enqueue_many_mixed_success_and_failure + TestServerHelper.create_queue(@server, 'many-mixed') messages = [ - { queue: 'batch-mixed', payload: 'good-1' }, + { queue: 'many-mixed', payload: 'good-1' }, { queue: 'no-such-queue-xyz', payload: 'bad' }, - { queue: 'batch-mixed', payload: 'good-2' } + { queue: 'many-mixed', payload: 'good-2' } ] - results = @client.batch_enqueue(messages) + results = @client.enqueue_many(messages) assert_equal 3, results.size assert results[0].success?, "first message should succeed" @@ -76,16 +76,16 @@ def test_batch_enqueue_mixed_success_and_failure end end -class TestBatchEnqueueResult < Minitest::Test +class TestEnqueueResult < Minitest::Test def test_success_result - r = Fila::BatchEnqueueResult.new(message_id: 'abc-123') + r = Fila::EnqueueResult.new(message_id: 'abc-123') assert r.success? assert_equal 'abc-123', r.message_id assert_nil r.error end def test_error_result - r = Fila::BatchEnqueueResult.new(error: 'queue not found') + r = Fila::EnqueueResult.new(error: 'queue not found') refute r.success? assert_nil r.message_id assert_equal 'queue not found', r.error From 88a334abd493debdb5d32487dadd153210949449 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Wed, 25 Mar 2026 09:53:52 -0300 Subject: [PATCH 12/19] fix: address cubic finding and rubocop lint offenses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ack/nack: raise RPCError on nil result instead of silently returning (identified by cubic — nil means zero results, which is an error for single-message operations) - batcher: extract result_to_outcome and broadcast_error from flush_batch to satisfy AbcSize/CyclomaticComplexity/MethodLength/PerceivedComplexity - batcher: remove redundant begin block in pop_with_timeout - batcher: add rubocop:disable for Metrics/ClassLength (126 lines, thread management + two modes is inherently complex) - test_batch: fix array indentation and prefer single-quoted strings --- lib/fila/batcher.rb | 57 +++++++++++++++++++++------------------------ lib/fila/client.rb | 6 +++-- test/test_batch.rb | 14 +++++------ 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 254f7de..593d323 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -6,7 +6,7 @@ module Fila # and linger (timer-based) modes. # # @api private - class Batcher + class Batcher # rubocop:disable Metrics/ClassLength # An item queued for batching, pairing a message with its result slot. BatchItem = Struct.new(:message, :result_queue, keyword_init: true) @@ -130,34 +130,33 @@ def drain_nonblocking(batch) # Flush a batch of items via the unified Enqueue RPC. def flush_batch(items) - req = ::Fila::V1::EnqueueRequest.new( - messages: items.map(&:message) - ) - resp = @stub.enqueue(req, metadata: @metadata) - results = resp.results + req = ::Fila::V1::EnqueueRequest.new(messages: items.map(&:message)) + results = @stub.enqueue(req, metadata: @metadata).results items.each_with_index do |item, idx| - result = results[idx] - if result.nil? - item.result_queue.push(Fila::Error.new('no result from server')) - elsif result.result == :message_id - item.result_queue.push(result.message_id) - else - err = result.error - case err.code - when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND - item.result_queue.push(QueueNotFoundError.new("enqueue: #{err.message}")) - else - item.result_queue.push(RPCError.new(GRPC::Core::StatusCodes::INTERNAL, err.message)) - end - end + item.result_queue.push(result_to_outcome(results[idx])) end rescue GRPC::BadStatus => e - # Transport-level failure: all messages in this batch get the error. - err = RPCError.new(e.code, e.details) - items.each { |item| item.result_queue.push(err) } + broadcast_error(items, RPCError.new(e.code, e.details)) rescue StandardError => e - err = Fila::Error.new(e.message) + broadcast_error(items, Fila::Error.new(e.message)) + end + + # Convert a single proto EnqueueResult into a String (message_id) or Exception. + def result_to_outcome(result) + return Fila::Error.new('no result from server') if result.nil? + return result.message_id if result.result == :message_id + + err = result.error + case err.code + when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND + QueueNotFoundError.new("enqueue: #{err.message}") + else + RPCError.new(GRPC::Core::StatusCodes::INTERNAL, err.message) + end + end + + def broadcast_error(items, err) items.each { |item| item.result_queue.push(err) } end @@ -170,13 +169,11 @@ def current_time_ms def pop_with_timeout(timeout_ms) deadline = current_time_ms + timeout_ms loop do - begin - return @queue.pop(true) - rescue ThreadError - raise if current_time_ms >= deadline + return @queue.pop(true) + rescue ThreadError + raise if current_time_ms >= deadline - sleep(0.001) # 1ms polling interval - end + sleep(0.001) # 1ms polling interval end end end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index e3cb12e..53d6cff 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -152,7 +152,8 @@ def ack(queue:, msg_id:) resp = @stub.ack(req, metadata: call_metadata) result = resp.results.first - return if result.nil? || result.result == :success + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? + return if result.result == :success err = result.error case err.code @@ -178,7 +179,8 @@ def nack(queue:, msg_id:, error:) resp = @stub.nack(req, metadata: call_metadata) result = resp.results.first - return if result.nil? || result.result == :success + raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? + return if result.result == :success err = result.error case err.code diff --git a/test/test_batch.rb b/test/test_batch.rb index 5bdb4fb..62f1b41 100644 --- a/test/test_batch.rb +++ b/test/test_batch.rb @@ -43,9 +43,9 @@ def test_enqueue_many_multiple_messages def test_enqueue_many_single_message TestServerHelper.create_queue(@server, 'many-single') - results = @client.enqueue_many([ - { queue: 'many-single', payload: 'solo' } - ]) + results = @client.enqueue_many( + [{ queue: 'many-single', payload: 'solo' }] + ) assert_equal 1, results.size assert results.first.success? @@ -69,10 +69,10 @@ def test_enqueue_many_mixed_success_and_failure results = @client.enqueue_many(messages) assert_equal 3, results.size - assert results[0].success?, "first message should succeed" - refute results[1].success?, "second message should fail (nonexistent queue)" - assert results[1].error, "second message should have error description" - assert results[2].success?, "third message should succeed" + assert results[0].success?, 'first message should succeed' + refute results[1].success?, 'second message should fail (nonexistent queue)' + assert results[1].error, 'second message should have error description' + assert results[2].success?, 'third message should succeed' end end From b4829071f36ede345dd1b2da68c12d4e0816eb53 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 09:54:39 -0300 Subject: [PATCH 13/19] feat: replace grpc with fibp binary protocol migrate the ruby sdk from grpc to the fila binary protocol (fibp). adds tcp connection with tls/mtls support, fibp codec with all opcodes, admin operations, auth operations, and leader hint redirect. removes grpc and protobuf dependencies. bumps version to 0.5.0. --- .rubocop.yml | 4 +- README.md | 137 ++-- fila-client.gemspec | 6 +- lib/fila.rb | 3 + lib/fila/batcher.rb | 81 ++- lib/fila/client.rb | 610 +++++++++++++----- lib/fila/errors.rb | 33 +- lib/fila/fibp/codec.rb | 172 +++++ lib/fila/fibp/connection.rb | 310 +++++++++ lib/fila/fibp/opcodes.rb | 83 +++ lib/fila/proto/fila/v1/admin_pb.rb | 49 -- lib/fila/proto/fila/v1/admin_services_pb.rb | 39 -- lib/fila/proto/fila/v1/messages_pb.rb | 21 - lib/fila/proto/fila/v1/service_pb.rb | 42 -- lib/fila/proto/fila/v1/service_services_pb.rb | 29 - lib/fila/version.rb | 2 +- proto/fila/v1/admin.proto | 197 ------ proto/fila/v1/messages.proto | 28 - proto/fila/v1/service.proto | 142 ---- test/test_batch.rb | 9 - test/test_helper.rb | 43 +- test/test_tls_auth.rb | 28 +- 22 files changed, 1214 insertions(+), 854 deletions(-) create mode 100644 lib/fila/fibp/codec.rb create mode 100644 lib/fila/fibp/connection.rb create mode 100644 lib/fila/fibp/opcodes.rb delete mode 100644 lib/fila/proto/fila/v1/admin_pb.rb delete mode 100644 lib/fila/proto/fila/v1/admin_services_pb.rb delete mode 100644 lib/fila/proto/fila/v1/messages_pb.rb delete mode 100644 lib/fila/proto/fila/v1/service_pb.rb delete mode 100644 lib/fila/proto/fila/v1/service_services_pb.rb delete mode 100644 proto/fila/v1/admin.proto delete mode 100644 proto/fila/v1/messages.proto delete mode 100644 proto/fila/v1/service.proto diff --git a/.rubocop.yml b/.rubocop.yml index 106174c..ea1e5b8 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,9 +3,11 @@ AllCops: NewCops: enable SuggestExtensions: false Exclude: - - 'lib/fila/proto/**/*' - 'vendor/**/*' +Metrics/ClassLength: + Max: 300 + Metrics/MethodLength: Max: 25 Exclude: diff --git a/README.md b/README.md index 9c59120..e3e1d09 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # fila-ruby -Ruby client SDK for the [Fila](https://github.com/faisca/fila) message broker. +Ruby client SDK for the [Fila](https://github.com/faisca/fila) message broker using the FIBP binary protocol. ## Installation @@ -44,21 +44,9 @@ end client.close ``` -### TLS (system trust store) - -```ruby -require "fila" - -# TLS using the OS system trust store (e.g., server uses a public CA). -client = Fila::Client.new("localhost:5555", tls: true) -``` - ### TLS (custom CA) ```ruby -require "fila" - -# TLS with an explicit CA certificate (e.g., private/self-signed CA). client = Fila::Client.new("localhost:5555", ca_cert: File.read("ca.pem") ) @@ -67,16 +55,6 @@ client = Fila::Client.new("localhost:5555", ### mTLS (mutual TLS) ```ruby -require "fila" - -# Mutual TLS with system trust store. -client = Fila::Client.new("localhost:5555", - tls: true, - client_cert: File.read("client.pem"), - client_key: File.read("client-key.pem") -) - -# Mutual TLS with explicit CA certificate. client = Fila::Client.new("localhost:5555", ca_cert: File.read("ca.pem"), client_cert: File.read("client.pem"), @@ -87,9 +65,6 @@ client = Fila::Client.new("localhost:5555", ### API Key Authentication ```ruby -require "fila" - -# API key sent as Bearer token on every request. client = Fila::Client.new("localhost:5555", api_key: "fila_your_api_key_here" ) @@ -98,9 +73,6 @@ client = Fila::Client.new("localhost:5555", ### mTLS + API Key ```ruby -require "fila" - -# Full security: mTLS transport + API key authentication. client = Fila::Client.new("localhost:5555", ca_cert: File.read("ca.pem"), client_cert: File.read("client.pem"), @@ -109,9 +81,73 @@ client = Fila::Client.new("localhost:5555", ) ``` +### Batch Enqueue + +```ruby +results = client.enqueue_many([ + { queue: "orders", payload: "order-1", headers: { "tenant" => "acme" } }, + { queue: "orders", payload: "order-2" }, +]) + +results.each do |r| + if r.success? + puts "Enqueued: #{r.message_id}" + else + puts "Failed: #{r.error}" + end +end +``` + +### Admin Operations + +```ruby +# Create a queue. +client.create_queue(name: "my-queue") + +# Delete a queue. +client.delete_queue(queue: "my-queue") + +# Get queue statistics. +stats = client.get_stats(queue: "my-queue") + +# List all queues. +queues = client.list_queues + +# Runtime configuration. +client.set_config(key: "queues.my-queue.visibility_timeout_ms", value: "30000") +value = client.get_config(key: "queues.my-queue.visibility_timeout_ms") +entries = client.list_config(prefix: "queues.") + +# Redrive DLQ messages. +count = client.redrive(dlq_queue: "my-queue-dlq", count: 100) +``` + +### Auth Operations + +```ruby +# Create an API key. +result = client.create_api_key(name: "my-key", is_superadmin: false) +puts result[:key] + +# Revoke an API key. +client.revoke_api_key(key_id: result[:key_id]) + +# List API keys. +keys = client.list_api_keys + +# Set ACL permissions. +client.set_acl(key_id: "key-id", permissions: [ + { kind: "produce", pattern: "orders.*" }, + { kind: "consume", pattern: "orders.*" }, +]) + +# Get ACL permissions. +acl = client.get_acl(key_id: "key-id") +``` + ## API -### `Fila::Client.new(addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil)` +### `Fila::Client.new(addr, ...)` Connect to a Fila broker at the given address (e.g., `"localhost:5555"`). @@ -122,47 +158,46 @@ Connect to a Fila broker at the given address (e.g., `"localhost:5555"`). | `ca_cert:` | `String` or `nil` | PEM-encoded CA certificate for TLS (implies `tls: true`) | | `client_cert:` | `String` or `nil` | PEM-encoded client certificate for mTLS | | `client_key:` | `String` or `nil` | PEM-encoded client private key for mTLS | -| `api_key:` | `String` or `nil` | API key for Bearer token authentication | - -When no TLS/auth options are provided, the client connects over plaintext (backward compatible). When `tls: true` is set without `ca_cert:`, the OS system trust store is used for server certificate verification. +| `api_key:` | `String` or `nil` | API key sent during FIBP handshake | +| `batch_mode:` | `Symbol` | `:auto` (default), `:linger`, or `:disabled` | -### `client.enqueue(queue:, headers:, payload:)` +### `client.enqueue(queue:, payload:, headers: nil)` Enqueue a message. Returns the broker-assigned message ID (UUIDv7). +### `client.enqueue_many(messages)` + +Enqueue multiple messages in a single request. Returns an array of `Fila::EnqueueResult`. + ### `client.consume(queue:) { |msg| ... }` -Open a streaming consumer. Yields `Fila::ConsumeMessage` objects as they become available. If no block is given, returns an `Enumerator`. Nacked messages are redelivered on the same stream. +Open a streaming consumer. Yields `Fila::ConsumeMessage` objects. If no block is given, returns an `Enumerator`. ### `client.ack(queue:, msg_id:)` -Acknowledge a successfully processed message. The message is permanently removed. +Acknowledge a successfully processed message. ### `client.nack(queue:, msg_id:, error:)` -Negatively acknowledge a failed message. The message is requeued or routed to the dead-letter queue based on the queue's configuration. +Negatively acknowledge a failed message. ### `client.close` -Close the underlying gRPC channel. +Drain pending batches and close the TCP connection. ## Error Handling Per-operation error classes are raised for specific failure modes: -```ruby -begin - client.enqueue(queue: "missing-queue", payload: "test") -rescue Fila::QueueNotFoundError => e - # handle queue not found -end - -begin - client.ack(queue: "my-queue", msg_id: "missing-id") -rescue Fila::MessageNotFoundError => e - # handle message not found -end -``` +| Error Class | Description | +|---|---| +| `Fila::QueueNotFoundError` | Queue does not exist | +| `Fila::MessageNotFoundError` | Message not found or not leased | +| `Fila::QueueAlreadyExistsError` | Queue already exists | +| `Fila::AuthenticationError` | Missing or invalid API key | +| `Fila::ForbiddenError` | Insufficient permissions | +| `Fila::NotLeaderError` | Not the leader (includes leader hint) | +| `Fila::RPCError` | Transport or protocol failure | ## License diff --git a/fila-client.gemspec b/fila-client.gemspec index 98cbfad..2dfbc30 100644 --- a/fila-client.gemspec +++ b/fila-client.gemspec @@ -7,15 +7,13 @@ Gem::Specification.new do |spec| spec.version = Fila::VERSION spec.authors = ['Faisca'] spec.summary = 'Ruby client SDK for the Fila message broker' - spec.description = "Idiomatic Ruby client wrapping Fila's gRPC API for enqueue, consume, ack, and nack operations." + spec.description = "Idiomatic Ruby client for the Fila message broker using the FIBP binary protocol." spec.homepage = 'https://github.com/faiscadev/fila-ruby' spec.license = 'AGPL-3.0-or-later' spec.required_ruby_version = '>= 3.1' - spec.files = Dir['lib/**/*.rb', 'proto/**/*.proto', 'LICENSE', 'README.md'] + spec.files = Dir['lib/**/*.rb', 'LICENSE', 'README.md'] spec.require_paths = ['lib'] - spec.add_dependency 'google-protobuf', '~> 4.0' - spec.add_dependency 'grpc', '~> 1.60' spec.metadata['rubygems_mfa_required'] = 'true' end diff --git a/lib/fila.rb b/lib/fila.rb index 3a74a1b..39e9462 100644 --- a/lib/fila.rb +++ b/lib/fila.rb @@ -4,5 +4,8 @@ require_relative 'fila/errors' require_relative 'fila/consume_message' require_relative 'fila/enqueue_result' +require_relative 'fila/fibp/opcodes' +require_relative 'fila/fibp/codec' +require_relative 'fila/fibp/connection' require_relative 'fila/batcher' require_relative 'fila/client' diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 593d323..795bff1 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -2,23 +2,21 @@ module Fila # Background batcher that collects enqueue messages and flushes them - # in batches via the unified Enqueue RPC. Supports auto (opportunistic) + # in batches via the FIBP binary protocol. Supports auto (opportunistic) # and linger (timer-based) modes. # # @api private class Batcher # rubocop:disable Metrics/ClassLength - # An item queued for batching, pairing a message with its result slot. + # An item queued for batching, pairing a message hash with its result slot. BatchItem = Struct.new(:message, :result_queue, keyword_init: true) - # @param stub [Fila::V1::FilaService::Stub] gRPC stub - # @param metadata [Hash] call metadata (auth headers) + # @param conn [Fila::FIBP::Connection] FIBP connection # @param mode [Symbol] :auto or :linger # @param max_batch_size [Integer] cap on batch size (auto mode) # @param batch_size [Integer] batch size threshold (linger mode) # @param linger_ms [Integer] linger time in ms (linger mode) - def initialize(stub:, metadata:, mode:, max_batch_size: 100, batch_size: 100, linger_ms: 10) - @stub = stub - @metadata = metadata + def initialize(conn:, mode:, max_batch_size: 100, batch_size: 100, linger_ms: 10) + @conn = conn @mode = mode @max_batch_size = mode == :auto ? max_batch_size : batch_size @linger_ms = linger_ms @@ -33,10 +31,10 @@ def initialize(stub:, metadata:, mode:, max_batch_size: 100, batch_size: 100, li # Submit a message for batched sending. Blocks until the batch # containing this message is flushed and the result is available. # - # @param message [Fila::V1::EnqueueMessage] the enqueue message + # @param message [Hash] message hash with :queue, :headers, :payload # @return [String] message ID on success # @raise [Fila::QueueNotFoundError] if the queue does not exist - # @raise [Fila::RPCError] for unexpected gRPC failures + # @raise [Fila::RPCError] for unexpected failures def submit(message) result_queue = Queue.new item = BatchItem.new(message: message, result_queue: result_queue) @@ -47,7 +45,6 @@ def submit(message) @queue.push(item) end - # Block until the batcher flushes our batch and posts the result. outcome = result_queue.pop case outcome when String then outcome @@ -72,8 +69,6 @@ def run_loop end end - # Auto mode: block for the first message, then non-blocking drain - # any additional messages that have arrived, flush concurrently. def run_auto_loop loop do first = @queue.pop @@ -85,8 +80,6 @@ def run_auto_loop end end - # Linger mode: block for the first message, then wait up to linger_ms - # for more messages or until batch_size is reached. def run_linger_loop loop do first = @queue.pop @@ -116,9 +109,9 @@ def run_linger_loop def drain_nonblocking(batch) while batch.size < @max_batch_size begin - item = @queue.pop(true) # non_block = true + item = @queue.pop(true) if item == :shutdown - @queue.push(:shutdown) # re-enqueue so the loop sees it + @queue.push(:shutdown) break end batch << item @@ -128,31 +121,53 @@ def drain_nonblocking(batch) end end - # Flush a batch of items via the unified Enqueue RPC. + # Flush a batch of items via FIBP Enqueue. def flush_batch(items) - req = ::Fila::V1::EnqueueRequest.new(messages: items.map(&:message)) - results = @stub.enqueue(req, metadata: @metadata).results + messages = items.map(&:message) + payload = encode_enqueue_batch(messages) + opcode, resp = @conn.request(FIBP::Opcodes::ENQUEUE, payload) + + if opcode == FIBP::Opcodes::ERROR + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + message = reader.read_string + broadcast_error(items, RPCError.new(code, message)) + return + end + + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 items.each_with_index do |item, idx| - item.result_queue.push(result_to_outcome(results[idx])) + if idx < count + code = reader.read_u8 + msg_id = reader.read_string + item.result_queue.push(result_to_outcome(code, msg_id)) + else + item.result_queue.push(Fila::Error.new('no result from server')) + end end - rescue GRPC::BadStatus => e - broadcast_error(items, RPCError.new(e.code, e.details)) rescue StandardError => e broadcast_error(items, Fila::Error.new(e.message)) end - # Convert a single proto EnqueueResult into a String (message_id) or Exception. - def result_to_outcome(result) - return Fila::Error.new('no result from server') if result.nil? - return result.message_id if result.result == :message_id + def encode_enqueue_batch(messages) + buf = FIBP::Codec.encode_u32(messages.size) + messages.each do |m| + buf += FIBP::Codec.encode_string(m[:queue]) + + FIBP::Codec.encode_map(m[:headers] || {}) + + FIBP::Codec.encode_bytes(m[:payload]) + end + buf + end - err = result.error - case err.code - when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND - QueueNotFoundError.new("enqueue: #{err.message}") + def result_to_outcome(code, msg_id) + case code + when FIBP::ErrorCodes::OK then msg_id + when FIBP::ErrorCodes::QUEUE_NOT_FOUND + QueueNotFoundError.new("enqueue: queue not found") else - RPCError.new(GRPC::Core::StatusCodes::INTERNAL, err.message) + RPCError.new(code, "enqueue failed") end end @@ -164,8 +179,6 @@ def current_time_ms (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i end - # Pop from @queue with a timeout in milliseconds. - # Raises ThreadError if nothing is available within the timeout. def pop_with_timeout(timeout_ms) deadline = current_time_ms + timeout_ms loop do @@ -173,7 +186,7 @@ def pop_with_timeout(timeout_ms) rescue ThreadError raise if current_time_ms >= deadline - sleep(0.001) # 1ms polling interval + sleep(0.001) end end end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 53d6cff..2fd5d0d 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -1,20 +1,15 @@ # frozen_string_literal: true -require 'grpc' - -# Add proto directory to load path so generated requires resolve correctly. -$LOAD_PATH.unshift(File.expand_path('proto', __dir__)) unless $LOAD_PATH.include?(File.expand_path('proto', __dir__)) - -require_relative 'proto/fila/v1/service_services_pb' require_relative 'errors' require_relative 'consume_message' require_relative 'enqueue_result' require_relative 'batcher' +require_relative 'fibp/opcodes' +require_relative 'fibp/codec' +require_relative 'fibp/connection' module Fila - # Client for the Fila message broker. - # - # Wraps the hot-path gRPC operations: enqueue, consume, ack, nack. + # Client for the Fila message broker over the FIBP binary protocol. # # @example Plain-text, default auto-batching # client = Fila::Client.new("localhost:5555") @@ -22,12 +17,9 @@ module Fila # @example Batching disabled # client = Fila::Client.new("localhost:5555", batch_mode: :disabled) # - # @example Linger-based batching + # @example TLS with custom CA # client = Fila::Client.new("localhost:5555", - # batch_mode: :linger, linger_ms: 10, batch_size: 50) - # - # @example TLS with system trust store - # client = Fila::Client.new("localhost:5555", tls: true) + # ca_cert: File.read("ca.pem")) # # @example mTLS + API key # client = Fila::Client.new("localhost:5555", @@ -36,7 +28,6 @@ module Fila # client_key: File.read("client-key.pem"), # api_key: "fila_abc123") class Client # rubocop:disable Metrics/ClassLength - # Valid batch mode values. BATCH_MODES = %i[auto linger disabled].freeze private_constant :BATCH_MODES @@ -48,18 +39,23 @@ class Client # rubocop:disable Metrics/ClassLength # @param client_key [String, nil] PEM-encoded client key (mTLS) # @param api_key [String, nil] API key for authentication # @param batch_mode [Symbol] :auto (default), :linger, or :disabled - # @param max_batch_size [Integer] max batch size for auto mode (default: 100) - # @param batch_size [Integer] batch size for linger mode (default: 100) - # @param linger_ms [Integer] linger time in ms for linger mode (default: 10) + # @param max_batch_size [Integer] max batch size for auto mode + # @param batch_size [Integer] batch size for linger mode + # @param linger_ms [Integer] linger time in ms for linger mode def initialize( # rubocop:disable Metrics/ParameterLists addr, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil, batch_mode: :auto, max_batch_size: 100, batch_size: 100, linger_ms: 10 ) validate_batch_mode(batch_mode) + @addr = addr + @tls = tls + @ca_cert = ca_cert + @client_cert = client_cert + @client_key = client_key @api_key = api_key - @credentials = build_credentials(tls: tls, ca_cert: ca_cert, client_cert: client_cert, client_key: client_key) - @stub = ::Fila::V1::FilaService::Stub.new(addr, @credentials) + + @conn = build_connection(addr) @batcher = start_batcher(batch_mode, max_batch_size, batch_size, linger_ms) end @@ -67,26 +63,18 @@ def initialize( # rubocop:disable Metrics/ParameterLists def close @batcher&.close @batcher = nil + @conn&.close + @conn = nil end # Enqueue a single message to a queue. # - # When batching is enabled (default), the message is submitted to - # the background batcher. At low load each message is sent - # individually; at high load messages cluster into batches. - # # @param queue [String] target queue name # @param payload [String] message payload # @param headers [Hash, nil] optional headers # @return [String] broker-assigned message ID - # @raise [QueueNotFoundError] if the queue does not exist - # @raise [RPCError] for unexpected gRPC failures def enqueue(queue:, payload:, headers: nil) - msg = ::Fila::V1::EnqueueMessage.new( - queue: queue, - headers: headers || {}, - payload: payload - ) + msg = { queue: queue, headers: headers || {}, payload: payload } if @batcher @batcher.submit(msg) @@ -95,41 +83,21 @@ def enqueue(queue:, payload:, headers: nil) end end - # Enqueue multiple messages in a single RPC call. - # - # Each message is independently validated and processed. A failed - # message does not affect the others. Returns an array of - # EnqueueResult with one result per input message, in order. + # Enqueue multiple messages in a single request. # - # This bypasses the background batcher and always uses the - # Enqueue RPC directly. - # - # @param messages [Array] messages to enqueue; each hash has - # keys :queue (String), :payload (String), and optionally - # :headers (Hash) + # @param messages [Array] messages with :queue, :payload, :headers # @return [Array] - # @raise [RPCError] for transport-level gRPC failures def enqueue_many(messages) - proto_messages = messages.map do |m| - ::Fila::V1::EnqueueMessage.new( - queue: m[:queue], - headers: m[:headers] || {}, - payload: m[:payload] - ) - end + return [] if messages.empty? - req = ::Fila::V1::EnqueueRequest.new(messages: proto_messages) - resp = @stub.enqueue(req, metadata: call_metadata) + payload = encode_enqueue_batch(messages) + opcode, resp = @conn.request(FIBP::Opcodes::ENQUEUE, payload) - resp.results.map do |r| - if r.result == :message_id - EnqueueResult.new(message_id: r.message_id) - else - EnqueueResult.new(error: r.error.message) - end + if opcode == FIBP::Opcodes::ERROR + raise_from_error_frame(resp) end - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) + + decode_enqueue_results(resp) end # Open a streaming consumer. Yields messages as they arrive. @@ -144,26 +112,27 @@ def consume(queue:, &block) # # @param queue [String] queue the message belongs to # @param msg_id [String] ID of the message to acknowledge - # @raise [MessageNotFoundError] if the message does not exist - # @raise [RPCError] for unexpected gRPC failures def ack(queue:, msg_id:) - msg = ::Fila::V1::AckMessage.new(queue: queue, message_id: msg_id) - req = ::Fila::V1::AckRequest.new(messages: [msg]) - resp = @stub.ack(req, metadata: call_metadata) - - result = resp.results.first - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? - return if result.result == :success - - err = result.error - case err.code - when :ACK_ERROR_CODE_MESSAGE_NOT_FOUND - raise MessageNotFoundError, "ack: #{err.message}" + payload = FIBP::Codec.encode_u32(1) + + FIBP::Codec.encode_string(queue) + + FIBP::Codec.encode_string(msg_id) + opcode, resp = @conn.request(FIBP::Opcodes::ACK, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 + raise RPCError.new(0xFF, 'no result from server') if count.zero? + + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + case code + when FIBP::ErrorCodes::MESSAGE_NOT_FOUND + raise MessageNotFoundError, "ack: message not found" else - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "ack: #{err.message}") + raise RPCError.new(code, "ack failed") end - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) end # Negatively acknowledge a message that failed processing. @@ -171,31 +140,250 @@ def ack(queue:, msg_id:) # @param queue [String] queue the message belongs to # @param msg_id [String] ID of the message to nack # @param error [String] description of the failure - # @raise [MessageNotFoundError] if the message does not exist - # @raise [RPCError] for unexpected gRPC failures def nack(queue:, msg_id:, error:) - msg = ::Fila::V1::NackMessage.new(queue: queue, message_id: msg_id, error: error) - req = ::Fila::V1::NackRequest.new(messages: [msg]) - resp = @stub.nack(req, metadata: call_metadata) - - result = resp.results.first - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? - return if result.result == :success - - err = result.error - case err.code - when :NACK_ERROR_CODE_MESSAGE_NOT_FOUND - raise MessageNotFoundError, "nack: #{err.message}" + payload = FIBP::Codec.encode_u32(1) + + FIBP::Codec.encode_string(queue) + + FIBP::Codec.encode_string(msg_id) + + FIBP::Codec.encode_string(error) + opcode, resp = @conn.request(FIBP::Opcodes::NACK, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 + raise RPCError.new(0xFF, 'no result from server') if count.zero? + + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + case code + when FIBP::ErrorCodes::MESSAGE_NOT_FOUND + raise MessageNotFoundError, "nack: message not found" + else + raise RPCError.new(code, "nack failed") + end + end + + # --- Admin operations --- + + # Create a queue. + # + # @param name [String] queue name + # @param on_enqueue_script [String, nil] Lua enqueue script + # @param on_failure_script [String, nil] Lua failure script + # @param visibility_timeout_ms [Integer] visibility timeout (0 = server default) + # @return [String] created queue ID + def create_queue(name:, on_enqueue_script: nil, on_failure_script: nil, visibility_timeout_ms: 0) + payload = FIBP::Codec.encode_string(name) + + FIBP::Codec.encode_optional_string(on_enqueue_script) + + FIBP::Codec.encode_optional_string(on_failure_script) + + FIBP::Codec.encode_u64(visibility_timeout_ms) + opcode, resp = @conn.request(FIBP::Opcodes::CREATE_QUEUE, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + queue_id = reader.read_string + + case code + when FIBP::ErrorCodes::OK then queue_id + when FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS + raise QueueAlreadyExistsError, "queue '#{name}' already exists" else - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "nack: #{err.message}") + raise RPCError.new(code, "create queue failed") end - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) end - LEADER_ADDR_KEY = 'x-fila-leader-addr' + # Delete a queue. + # + # @param queue [String] queue name + def delete_queue(queue:) + payload = FIBP::Codec.encode_string(queue) + opcode, resp = @conn.request(FIBP::Opcodes::DELETE_QUEUE, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + raise_for_error_code(code, 'delete queue') + end + + # Get statistics for a queue. + # + # @param queue [String] queue name + # @return [Hash] queue statistics + def get_stats(queue:) + payload = FIBP::Codec.encode_string(queue) + opcode, resp = @conn.request(FIBP::Opcodes::GET_STATS, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + decode_stats_result(resp) + end + + # List all queues. + # + # @return [Hash] with :cluster_node_count and :queues array + def list_queues + opcode, resp = @conn.request(FIBP::Opcodes::LIST_QUEUES, '') + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + decode_list_queues_result(resp) + end + + # Set a runtime config key. + def set_config(key:, value:) + payload = FIBP::Codec.encode_string(key) + FIBP::Codec.encode_string(value) + opcode, resp = @conn.request(FIBP::Opcodes::SET_CONFIG, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + raise_for_error_code(code, 'set config') + end + + # Get a runtime config value. + def get_config(key:) + payload = FIBP::Codec.encode_string(key) + opcode, resp = @conn.request(FIBP::Opcodes::GET_CONFIG, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'get config') unless code == FIBP::ErrorCodes::OK + + reader.read_string + end + + # List config keys by prefix. + def list_config(prefix:) + payload = FIBP::Codec.encode_string(prefix) + opcode, resp = @conn.request(FIBP::Opcodes::LIST_CONFIG, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'list config') unless code == FIBP::ErrorCodes::OK + + count = reader.read_u16 + Array.new(count) do + { key: reader.read_string, value: reader.read_string } + end + end + + # Redrive messages from a DLQ back to their parent queue. + def redrive(dlq_queue:, count:) + payload = FIBP::Codec.encode_string(dlq_queue) + FIBP::Codec.encode_u64(count) + opcode, resp = @conn.request(FIBP::Opcodes::REDRIVE, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'redrive') unless code == FIBP::ErrorCodes::OK + + reader.read_u64 + end - private_constant :LEADER_ADDR_KEY + # --- Auth operations --- + + # Create an API key. + def create_api_key(name:, expires_at_ms: 0, is_superadmin: false) + payload = FIBP::Codec.encode_string(name) + + FIBP::Codec.encode_u64(expires_at_ms) + + FIBP::Codec.encode_bool(is_superadmin) + opcode, resp = @conn.request(FIBP::Opcodes::CREATE_API_KEY, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'create api key') unless code == FIBP::ErrorCodes::OK + + { key_id: reader.read_string, key: reader.read_string, is_superadmin: reader.read_bool } + end + + # Revoke an API key. + def revoke_api_key(key_id:) + payload = FIBP::Codec.encode_string(key_id) + opcode, resp = @conn.request(FIBP::Opcodes::REVOKE_API_KEY, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + raise_for_error_code(code, 'revoke api key') + end + + # List all API keys. + def list_api_keys + opcode, resp = @conn.request(FIBP::Opcodes::LIST_API_KEYS, '') + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'list api keys') unless code == FIBP::ErrorCodes::OK + + count = reader.read_u16 + Array.new(count) do + { + key_id: reader.read_string, name: reader.read_string, + created_at_ms: reader.read_u64, expires_at_ms: reader.read_u64, + is_superadmin: reader.read_bool + } + end + end + + # Set ACL permissions for an API key. + def set_acl(key_id:, permissions:) + payload = FIBP::Codec.encode_string(key_id) + + FIBP::Codec.encode_u16(permissions.size) + permissions.each do |perm| + payload += FIBP::Codec.encode_string(perm[:kind]) + + FIBP::Codec.encode_string(perm[:pattern]) + end + opcode, resp = @conn.request(FIBP::Opcodes::SET_ACL, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + return if code == FIBP::ErrorCodes::OK + + raise_for_error_code(code, 'set acl') + end + + # Get ACL permissions for an API key. + def get_acl(key_id:) + payload = FIBP::Codec.encode_string(key_id) + opcode, resp = @conn.request(FIBP::Opcodes::GET_ACL, payload) + + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR + + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'get acl') unless code == FIBP::ErrorCodes::OK + + key_id_resp = reader.read_string + is_superadmin = reader.read_bool + perm_count = reader.read_u16 + permissions = Array.new(perm_count) do + { kind: reader.read_string, pattern: reader.read_string } + end + { key_id: key_id_resp, is_superadmin: is_superadmin, permissions: permissions } + end private @@ -205,12 +393,30 @@ def validate_batch_mode(mode) raise ArgumentError, "invalid batch_mode: #{mode.inspect}, must be one of #{BATCH_MODES.inspect}" end + def build_connection(addr) + host, port_str = addr.split(':') + port = port_str.to_i + validate_tls_options + FIBP::Connection.new( + host: host, port: port, + tls: @tls, ca_cert: @ca_cert, + client_cert: @client_cert, client_key: @client_key, + api_key: @api_key + ) + end + + def validate_tls_options + tls_enabled = @tls || @ca_cert + return if tls_enabled || (!@client_cert && !@client_key) + + raise ArgumentError, 'tls: true or ca_cert is required when client_cert or client_key is provided' + end + def start_batcher(mode, max_batch_size, batch_size, linger_ms) return nil if mode == :disabled Batcher.new( - stub: @stub, - metadata: call_metadata, + conn: @conn, mode: mode, max_batch_size: max_batch_size, batch_size: batch_size, @@ -218,98 +424,176 @@ def start_batcher(mode, max_batch_size, batch_size, linger_ms) ) end - # Send a single message via the unified Enqueue RPC. def enqueue_single(msg) - req = ::Fila::V1::EnqueueRequest.new(messages: [msg]) - resp = @stub.enqueue(req, metadata: call_metadata) + payload = encode_enqueue_batch([msg]) + opcode, resp = @conn.request(FIBP::Opcodes::ENQUEUE, payload) - result = resp.results.first - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, 'no result from server') if result.nil? + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR - if result.result == :message_id - result.message_id + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 + raise RPCError.new(0xFF, 'no result from server') if count.zero? + + code = reader.read_u8 + msg_id = reader.read_string + + case code + when FIBP::ErrorCodes::OK then msg_id + when FIBP::ErrorCodes::QUEUE_NOT_FOUND + raise QueueNotFoundError, "enqueue: queue not found" else - err = result.error - case err.code - when :ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND - raise QueueNotFoundError, "enqueue: #{err.message}" - else - raise RPCError.new(GRPC::Core::StatusCodes::INTERNAL, "enqueue: #{err.message}") - end + raise RPCError.new(code, "enqueue failed") end - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) end - def consume_with_redirect(queue:, redirected:, &block) - stream = @stub.consume(::Fila::V1::ConsumeRequest.new(queue: queue), metadata: call_metadata) - stream.each do |resp| - yield_messages_from_response(resp, &block) + def encode_enqueue_batch(messages) + payload = FIBP::Codec.encode_u32(messages.size) + messages.each do |m| + payload += FIBP::Codec.encode_string(m[:queue]) + + FIBP::Codec.encode_map(m[:headers] || {}) + + FIBP::Codec.encode_bytes(m[:payload]) end - rescue GRPC::Cancelled then nil - rescue GRPC::NotFound => e - raise QueueNotFoundError, "consume: #{e.details}" - rescue GRPC::Unavailable => e - raise RPCError.new(e.code, e.details) if (leader_addr = extract_leader_addr(e)).nil? || redirected + payload + end - @stub = ::Fila::V1::FilaService::Stub.new(leader_addr, @credentials) - consume_with_redirect(queue: queue, redirected: true, &block) - rescue GRPC::BadStatus => e - raise RPCError.new(e.code, e.details) + def decode_enqueue_results(resp) + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 + Array.new(count) do + code = reader.read_u8 + msg_id = reader.read_string + if code == FIBP::ErrorCodes::OK + EnqueueResult.new(message_id: msg_id) + else + EnqueueResult.new(error: error_name(code)) + end + end end - # Unpack messages from a ConsumeResponse. - def yield_messages_from_response(resp, &block) - resp.messages.each do |msg| - next if msg.nil? || msg.id.empty? + def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/MethodLength + payload = FIBP::Codec.encode_string(queue) + + delivery_queue = Queue.new + consumer_done = false + rid = nil + + begin + rid, response = @conn.subscribe(FIBP::Opcodes::CONSUME, payload) do |_opcode, del_payload| + delivery_queue.push(del_payload) unless consumer_done + end + + opcode, resp_payload = response + raise_from_error_frame(resp_payload) if opcode == FIBP::Opcodes::ERROR - block.call(build_consume_message(msg)) + # Process deliveries from the queue. + loop do + del_payload = delivery_queue.pop + break if del_payload.nil? + + process_delivery(del_payload, &block) + end + rescue NotLeaderError => e + raise if redirected || e.leader_addr.nil? + + reconnect_to(e.leader_addr) + return consume_with_redirect(queue: queue, redirected: true, &block) + rescue LocalJumpError + nil # Consumer break end + ensure + consumer_done = true + @conn&.cancel_consume(rid) if rid end - def extract_leader_addr(err) - err.metadata[LEADER_ADDR_KEY] - rescue StandardError - nil + def process_delivery(payload, &block) + reader = FIBP::Codec::Reader.new(payload) + count = reader.read_u32 + count.times do + msg = decode_delivery_message(reader) + block.call(msg) + end end - def build_credentials(tls:, ca_cert:, client_cert:, client_key:) - tls_enabled = tls || ca_cert - validate_tls_options(tls_enabled, client_cert, client_key) - return :this_channel_is_insecure unless tls_enabled + def decode_delivery_message(reader) + msg_id = reader.read_string + queue = reader.read_string + headers = reader.read_map + payload = reader.read_bytes + fairness_key = reader.read_string + _weight = reader.read_u32 + _throttle_keys = reader.read_string_array + attempt_count = reader.read_u32 + _enqueued_at = reader.read_u64 + _leased_at = reader.read_u64 - build_channel_credentials(ca_cert, client_cert, client_key) + ConsumeMessage.new( + id: msg_id, + headers: headers, + payload: payload, + fairness_key: fairness_key, + attempt_count: attempt_count, + queue: queue + ) end - def validate_tls_options(tls_enabled, client_cert, client_key) - return if tls_enabled || (!client_cert && !client_key) - - raise ArgumentError, 'tls: true or ca_cert is required when client_cert or client_key is provided' + def reconnect_to(addr) + @conn&.close + @conn = build_connection(addr) end - def build_channel_credentials(ca_cert, client_cert, client_key) - if ca_cert then GRPC::Core::ChannelCredentials.new(ca_cert, client_key, client_cert) - elsif client_cert && client_key then GRPC::Core::ChannelCredentials.new(nil, client_key, client_cert) - else GRPC::Core::ChannelCredentials.new + def raise_from_error_frame(resp) + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + message = reader.read_string + metadata = reader.remaining.positive? ? reader.read_map : {} + + case code + when FIBP::ErrorCodes::QUEUE_NOT_FOUND + raise QueueNotFoundError, message + when FIBP::ErrorCodes::MESSAGE_NOT_FOUND + raise MessageNotFoundError, message + when FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS + raise QueueAlreadyExistsError, message + when FIBP::ErrorCodes::UNAUTHORIZED + raise AuthenticationError, message + when FIBP::ErrorCodes::FORBIDDEN + raise ForbiddenError, message + when FIBP::ErrorCodes::NOT_LEADER + raise NotLeaderError.new(message, leader_addr: metadata['leader_addr']) + when FIBP::ErrorCodes::API_KEY_NOT_FOUND + raise ApiKeyNotFoundError, message + else + raise RPCError.new(code, message) end end - def call_metadata - return {} unless @api_key - - { 'authorization' => "Bearer #{@api_key}" } + def raise_for_error_code(code, context) + case code + when FIBP::ErrorCodes::QUEUE_NOT_FOUND + raise QueueNotFoundError, "#{context}: queue not found" + when FIBP::ErrorCodes::MESSAGE_NOT_FOUND + raise MessageNotFoundError, "#{context}: message not found" + when FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS + raise QueueAlreadyExistsError, "#{context}: queue already exists" + when FIBP::ErrorCodes::UNAUTHORIZED + raise AuthenticationError, "#{context}: unauthorized" + when FIBP::ErrorCodes::FORBIDDEN + raise ForbiddenError, "#{context}: forbidden" + when FIBP::ErrorCodes::API_KEY_NOT_FOUND + raise ApiKeyNotFoundError, "#{context}: api key not found" + else + raise RPCError.new(code, "#{context} failed") + end end - def build_consume_message(msg) - metadata = msg.metadata - ConsumeMessage.new( - id: msg.id, - headers: msg.headers.to_h, - payload: msg.payload, - fairness_key: metadata&.fairness_key.to_s, - attempt_count: metadata&.attempt_count.to_i, - queue: metadata&.queue_id.to_s - ) + def error_name(code) + case code + when FIBP::ErrorCodes::QUEUE_NOT_FOUND then 'queue not found' + when FIBP::ErrorCodes::MESSAGE_NOT_FOUND then 'message not found' + when FIBP::ErrorCodes::UNAUTHORIZED then 'unauthorized' + when FIBP::ErrorCodes::FORBIDDEN then 'forbidden' + else "error code 0x#{code.to_s(16).rjust(2, '0')}" + end end end end diff --git a/lib/fila/errors.rb b/lib/fila/errors.rb index 2f3dcb8..b6307b8 100644 --- a/lib/fila/errors.rb +++ b/lib/fila/errors.rb @@ -10,16 +10,41 @@ class QueueNotFoundError < Error; end # Raised when the specified message does not exist. class MessageNotFoundError < Error; end - # Raised for unexpected gRPC failures, preserving status code and message. + # Raised when the queue already exists. + class QueueAlreadyExistsError < Error; end + + # Raised when authentication fails (missing or invalid API key). + class AuthenticationError < Error; end + + # Raised when the client lacks permission for the operation. + class ForbiddenError < Error; end + + # Raised when the server is not the leader for the target queue. + class NotLeaderError < Error + # @return [String, nil] address of the current leader + attr_reader :leader_addr + + # @param message [String] error message + # @param leader_addr [String, nil] leader address hint + def initialize(message, leader_addr: nil) + @leader_addr = leader_addr + super(message) + end + end + + # Raised when the API key is not found. + class ApiKeyNotFoundError < Error; end + + # Raised for transport or protocol failures, preserving the error code. class RPCError < Error - # @return [Integer] gRPC status code + # @return [Integer] FIBP error code attr_reader :code - # @param code [Integer] gRPC status code + # @param code [Integer] error code # @param message [String] error message def initialize(code, message) @code = code - super("rpc error (code = #{code}): #{message}") + super("rpc error (code = 0x#{code.to_s(16).rjust(2, '0')}): #{message}") end end end diff --git a/lib/fila/fibp/codec.rb b/lib/fila/fibp/codec.rb new file mode 100644 index 0000000..4e0fcd3 --- /dev/null +++ b/lib/fila/fibp/codec.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: true + +module Fila + module FIBP + # Low-level encoding/decoding primitives for the Fila binary protocol. + # + # All multi-byte integers are big-endian (network byte order). + # Strings are length-prefixed with u16; bytes with u32; maps with u16 count. + module Codec + module_function + + # --- Encoding primitives --- + + def encode_u8(val) + [val].pack('C') + end + + def encode_u16(val) + [val].pack('n') + end + + def encode_u32(val) + [val].pack('N') + end + + def encode_u64(val) + [val >> 32, val & 0xFFFFFFFF].pack('NN') + end + + def encode_i64(val) + [val >> 32, val & 0xFFFFFFFF].pack('NN') + end + + def encode_f64(val) + [val].pack('G') + end + + def encode_bool(val) + [val ? 1 : 0].pack('C') + end + + def encode_string(str) + bytes = str.to_s.encode('UTF-8').b + [bytes.bytesize].pack('n') + bytes + end + + def encode_bytes(data) + raw = data.to_s.b + [raw.bytesize].pack('N') + raw + end + + def encode_map(hash) + hash ||= {} + buf = [hash.size].pack('n') + hash.each do |k, v| + buf += encode_string(k) + encode_string(v) + end + buf + end + + def encode_string_array(arr) + arr ||= [] + buf = [arr.size].pack('n') + arr.each { |s| buf += encode_string(s) } + buf + end + + def encode_optional_string(val) + if val.nil? + encode_u8(0) + else + encode_u8(1) + encode_string(val) + end + end + + # Build a frame: [u32 frame_length][u8 opcode][u8 flags][u32 request_id][payload] + def encode_frame(opcode, request_id, payload, flags: 0) + body = [opcode, flags, request_id].pack('CCN') + payload + [body.bytesize].pack('N') + body + end + + # --- Decoding primitives --- + + # Reader wraps a binary string with a cursor for sequential reads. + class Reader + attr_reader :pos + + def initialize(data) + @data = data.b + @pos = 0 + end + + def remaining + @data.bytesize - @pos + end + + def read_raw(n) + raise ProtocolError, "unexpected end of frame (need #{n}, have #{remaining})" if remaining < n + + slice = @data.byteslice(@pos, n) + @pos += n + slice + end + + def read_u8 + read_raw(1).unpack1('C') + end + + def read_u16 + read_raw(2).unpack1('n') + end + + def read_u32 + read_raw(4).unpack1('N') + end + + def read_u64 + hi, lo = read_raw(8).unpack('NN') + (hi << 32) | lo + end + + def read_i64 + hi, lo = read_raw(8).unpack('NN') + val = (hi << 32) | lo + val >= (1 << 63) ? val - (1 << 64) : val + end + + def read_f64 + read_raw(8).unpack1('G') + end + + def read_bool + read_u8 != 0 + end + + def read_string + len = read_u16 + read_raw(len).force_encoding('UTF-8') + end + + def read_bytes + len = read_u32 + read_raw(len) + end + + def read_map + count = read_u16 + hash = {} + count.times do + k = read_string + v = read_string + hash[k] = v + end + hash + end + + def read_string_array + count = read_u16 + Array.new(count) { read_string } + end + + def read_optional_string + present = read_u8 + present == 1 ? read_string : nil + end + end + end + + # Raised for protocol-level decode errors. + class ProtocolError < Fila::Error; end + end +end diff --git a/lib/fila/fibp/connection.rb b/lib/fila/fibp/connection.rb new file mode 100644 index 0000000..2b28d58 --- /dev/null +++ b/lib/fila/fibp/connection.rb @@ -0,0 +1,310 @@ +# frozen_string_literal: true + +require 'socket' +require 'openssl' +require 'monitor' + +module Fila + module FIBP + # TCP connection with TLS support, FIBP framing, handshake, and + # request/response correlation. Supports both synchronous + # request/response and asynchronous delivery streaming. + class Connection # rubocop:disable Metrics/ClassLength + PROTOCOL_VERSION = 1 + DEFAULT_MAX_FRAME_SIZE = 16 * 1024 * 1024 # 16 MiB + + attr_reader :node_id, :max_frame_size + + # @param host [String] server hostname or IP + # @param port [Integer] server port + # @param tls [Boolean] enable TLS with system trust store + # @param ca_cert [String, nil] PEM CA certificate + # @param client_cert [String, nil] PEM client certificate (mTLS) + # @param client_key [String, nil] PEM client key (mTLS) + # @param api_key [String, nil] API key for handshake auth + def initialize(host:, port:, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) + @host = host + @port = port + @api_key = api_key + @tls_enabled = tls || !ca_cert.nil? + @ca_cert = ca_cert + @client_cert = client_cert + @client_key = client_key + + @request_id_counter = 0 + @pending = {} + @delivery_callbacks = {} + @monitor = Monitor.new + @read_monitor = Monitor.new + @write_monitor = Monitor.new + @closed = false + @continuation_buffers = {} + + connect! + perform_handshake + start_reader_thread + end + + # Send a request frame and wait for the response. + # + # @param opcode [Integer] request opcode + # @param payload [String] encoded payload bytes + # @return [Array(Integer, String)] [response_opcode, response_payload] + def request(opcode, payload) + raise Fila::Error, 'connection closed' if @closed + + rid = next_request_id + result_queue = Queue.new + @monitor.synchronize { @pending[rid] = result_queue } + + send_frame(opcode, rid, payload) + + response = result_queue.pop + raise response if response.is_a?(Exception) + + response + end + + # Register a delivery callback for a consume subscription. + # + # @param request_id [Integer] the consume request ID + # @param callback [Proc] called with (opcode, payload) for each delivery + # @return [Integer] the request_id + def subscribe(opcode, payload, &callback) + raise Fila::Error, 'connection closed' if @closed + + rid = next_request_id + result_queue = Queue.new + @monitor.synchronize do + @pending[rid] = result_queue + @delivery_callbacks[rid] = callback + end + + send_frame(opcode, rid, payload) + + # Wait for ConsumeOk or Error + response = result_queue.pop + raise response if response.is_a?(Exception) + + [rid, response] + end + + # Cancel a consume subscription. + def cancel_consume(request_id) + send_frame(Opcodes::CANCEL_CONSUME, request_id, '') + @monitor.synchronize { @delivery_callbacks.delete(request_id) } + end + + # Close the connection gracefully. + def close + return if @closed + + @closed = true + begin + send_frame(Opcodes::DISCONNECT, 0, '') + rescue StandardError + nil + end + @socket&.close + @reader_thread&.join(2) + + # Unblock any waiting requests. + @monitor.synchronize do + @pending.each_value { |q| q.push(Fila::Error.new('connection closed')) } + @pending.clear + @delivery_callbacks.clear + end + end + + private + + def connect! + tcp = TCPSocket.new(@host, @port) + tcp.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + + @socket = if @tls_enabled + wrap_tls(tcp) + else + tcp + end + end + + def wrap_tls(tcp) + ctx = OpenSSL::SSL::SSLContext.new + ctx.min_version = OpenSSL::SSL::TLS1_2_VERSION + + if @ca_cert + store = OpenSSL::X509::Store.new + store.add_cert(OpenSSL::X509::Certificate.new(@ca_cert)) + ctx.cert_store = store + else + ctx.set_params # uses system trust store + end + + if @client_cert && @client_key + ctx.cert = OpenSSL::X509::Certificate.new(@client_cert) + ctx.key = OpenSSL::PKey.read(@client_key) + end + + ssl = OpenSSL::SSL::SSLSocket.new(tcp, ctx) + ssl.hostname = @host + ssl.sync_close = true + ssl.connect + ssl + end + + def perform_handshake + payload = Codec.encode_u16(PROTOCOL_VERSION) + payload += Codec.encode_optional_string(@api_key) + + send_frame(Opcodes::HANDSHAKE, 0, payload) + + opcode, resp_payload = read_frame + if opcode == Opcodes::ERROR + reader = Codec::Reader.new(resp_payload) + code = reader.read_u8 + message = reader.read_string + raise_protocol_error(code, message, reader) + end + + unless opcode == Opcodes::HANDSHAKE_OK + raise ProtocolError, "expected HandshakeOk, got opcode 0x#{opcode.to_s(16)}" + end + + reader = Codec::Reader.new(resp_payload) + _version = reader.read_u16 + @node_id = reader.read_u64 + max_frame = reader.read_u32 + @max_frame_size = max_frame.zero? ? DEFAULT_MAX_FRAME_SIZE : max_frame + end + + def start_reader_thread + @reader_thread = Thread.new { reader_loop } + @reader_thread.abort_on_exception = false + end + + def reader_loop + until @closed + opcode, payload, request_id = read_frame_with_id + break if opcode.nil? + + handle_incoming(opcode, payload, request_id) + end + rescue IOError, Errno::ECONNRESET, OpenSSL::SSL::SSLError + # Connection closed + ensure + @closed = true + @monitor.synchronize do + err = Fila::Error.new('connection closed') + @pending.each_value { |q| q.push(err) } + @pending.clear + end + end + + def handle_incoming(opcode, payload, request_id) + if opcode == Opcodes::PING + send_frame(Opcodes::PONG, request_id, '') + return + end + + if opcode == Opcodes::DELIVERY + cb = @monitor.synchronize { @delivery_callbacks[request_id] } + cb&.call(opcode, payload) + return + end + + queue = @monitor.synchronize { @pending.delete(request_id) } + queue&.push([opcode, payload]) + end + + def send_frame(opcode, request_id, payload) + frame = Codec.encode_frame(opcode, request_id, payload) + @write_monitor.synchronize { write_all(frame) } + end + + def write_all(data) + total = 0 + while total < data.bytesize + written = @socket.write(data.byteslice(total..)) + total += written + end + end + + def read_frame + opcode, payload, _rid = read_frame_with_id + [opcode, payload] + end + + def read_frame_with_id + @read_monitor.synchronize do + loop do + length_bytes = read_exact(4) + return [nil, nil, nil] if length_bytes.nil? + + frame_length = length_bytes.unpack1('N') + body = read_exact(frame_length) + return [nil, nil, nil] if body.nil? + + opcode = body.getbyte(0) + flags = body.getbyte(1) + request_id = body.byteslice(2, 4).unpack1('N') + payload = body.byteslice(6..) + + continuation = (flags & FLAG_CONTINUATION) != 0 + + if continuation + @continuation_buffers[request_id] ||= { opcode: opcode, data: +''.b } + @continuation_buffers[request_id][:data] << payload + next + end + + if @continuation_buffers.key?(request_id) + buf = @continuation_buffers.delete(request_id) + payload = buf[:data] + payload + opcode = buf[:opcode] + end + + return [opcode, payload, request_id] + end + end + end + + def read_exact(n) + buf = +''.b + while buf.bytesize < n + chunk = @socket.read(n - buf.bytesize) + return nil if chunk.nil? || chunk.empty? + + buf << chunk + end + buf + end + + def next_request_id + @monitor.synchronize do + @request_id_counter = (@request_id_counter + 1) & 0xFFFFFFFF + @request_id_counter + end + end + + def raise_protocol_error(code, message, reader) + metadata = reader.remaining.positive? ? reader.read_map : {} + case code + when ErrorCodes::UNAUTHORIZED + raise Fila::AuthenticationError, message + when ErrorCodes::FORBIDDEN + raise Fila::ForbiddenError, message + when ErrorCodes::NOT_LEADER + leader_addr = metadata['leader_addr'] + raise Fila::NotLeaderError.new(message, leader_addr: leader_addr) + when ErrorCodes::QUEUE_NOT_FOUND + raise Fila::QueueNotFoundError, message + when ErrorCodes::UNSUPPORTED_VERSION + raise ProtocolError, "unsupported version: #{message}" + else + raise Fila::RPCError.new(code, message) + end + end + end + end +end diff --git a/lib/fila/fibp/opcodes.rb b/lib/fila/fibp/opcodes.rb new file mode 100644 index 0000000..fd408be --- /dev/null +++ b/lib/fila/fibp/opcodes.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Fila + module FIBP + # Protocol opcodes for the Fila binary protocol. + module Opcodes + # Control opcodes (0x00-0x0F) + HANDSHAKE = 0x01 + HANDSHAKE_OK = 0x02 + PING = 0x03 + PONG = 0x04 + DISCONNECT = 0x05 + + # Hot-path opcodes (0x10-0x1F) + ENQUEUE = 0x10 + ENQUEUE_RESULT = 0x11 + CONSUME = 0x12 + DELIVERY = 0x13 + CANCEL_CONSUME = 0x14 + ACK = 0x15 + ACK_RESULT = 0x16 + NACK = 0x17 + NACK_RESULT = 0x18 + CONSUME_OK = 0x19 + + # Error opcode + ERROR = 0xFE + + # Admin opcodes (0xFD downward) + CREATE_QUEUE = 0xFD + CREATE_QUEUE_RESULT = 0xFC + DELETE_QUEUE = 0xFB + DELETE_QUEUE_RESULT = 0xFA + GET_STATS = 0xF9 + GET_STATS_RESULT = 0xF8 + LIST_QUEUES = 0xF7 + LIST_QUEUES_RESULT = 0xF6 + SET_CONFIG = 0xF5 + SET_CONFIG_RESULT = 0xF4 + GET_CONFIG = 0xF3 + GET_CONFIG_RESULT = 0xF2 + LIST_CONFIG = 0xF1 + LIST_CONFIG_RESULT = 0xF0 + REDRIVE = 0xEF + REDRIVE_RESULT = 0xEE + CREATE_API_KEY = 0xED + CREATE_API_KEY_RESULT = 0xEC + REVOKE_API_KEY = 0xEB + REVOKE_API_KEY_RESULT = 0xEA + LIST_API_KEYS = 0xE9 + LIST_API_KEYS_RESULT = 0xE8 + SET_ACL = 0xE7 + SET_ACL_RESULT = 0xE6 + GET_ACL = 0xE5 + GET_ACL_RESULT = 0xE4 + end + + # Protocol error codes. + module ErrorCodes + OK = 0x00 + QUEUE_NOT_FOUND = 0x01 + MESSAGE_NOT_FOUND = 0x02 + QUEUE_ALREADY_EXISTS = 0x03 + LUA_COMPILATION = 0x04 + STORAGE_ERROR = 0x05 + NOT_A_DLQ = 0x06 + PARENT_QUEUE_NOT_FOUND = 0x07 + INVALID_CONFIG_VALUE = 0x08 + CHANNEL_FULL = 0x09 + UNAUTHORIZED = 0x0A + FORBIDDEN = 0x0B + NOT_LEADER = 0x0C + UNSUPPORTED_VERSION = 0x0D + INVALID_FRAME = 0x0E + API_KEY_NOT_FOUND = 0x0F + NODE_NOT_READY = 0x10 + INTERNAL_ERROR = 0xFF + end + + # Continuation flag in frame flags byte. + FLAG_CONTINUATION = 0x01 + end +end diff --git a/lib/fila/proto/fila/v1/admin_pb.rb b/lib/fila/proto/fila/v1/admin_pb.rb deleted file mode 100644 index 5138c83..0000000 --- a/lib/fila/proto/fila/v1/admin_pb.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: fila/v1/admin.proto - -require 'google/protobuf' - - -descriptor_data = "\n\x13\x66ila/v1/admin.proto\x12\x07\x66ila.v1\"H\n\x12\x43reateQueueRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x06\x63onfig\x18\x02 \x01(\x0b\x32\x14.fila.v1.QueueConfig\"b\n\x0bQueueConfig\x12\x19\n\x11on_enqueue_script\x18\x01 \x01(\t\x12\x19\n\x11on_failure_script\x18\x02 \x01(\t\x12\x1d\n\x15visibility_timeout_ms\x18\x03 \x01(\x04\"\'\n\x13\x43reateQueueResponse\x12\x10\n\x08queue_id\x18\x01 \x01(\t\"#\n\x12\x44\x65leteQueueRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"\x15\n\x13\x44\x65leteQueueResponse\".\n\x10SetConfigRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x13\n\x11SetConfigResponse\"\x1f\n\x10GetConfigRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"\"\n\x11GetConfigResponse\x12\r\n\x05value\x18\x01 \x01(\t\")\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"#\n\x11ListConfigRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\"P\n\x12ListConfigResponse\x12%\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x14.fila.v1.ConfigEntry\x12\x13\n\x0btotal_count\x18\x02 \x01(\r\" \n\x0fGetStatsRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"b\n\x13PerFairnessKeyStats\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\rpending_count\x18\x02 \x01(\x04\x12\x17\n\x0f\x63urrent_deficit\x18\x03 \x01(\x03\x12\x0e\n\x06weight\x18\x04 \x01(\r\"Z\n\x13PerThrottleKeyStats\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0e\n\x06tokens\x18\x02 \x01(\x01\x12\x17\n\x0frate_per_second\x18\x03 \x01(\x01\x12\r\n\x05\x62urst\x18\x04 \x01(\x01\"\x9f\x02\n\x10GetStatsResponse\x12\r\n\x05\x64\x65pth\x18\x01 \x01(\x04\x12\x11\n\tin_flight\x18\x02 \x01(\x04\x12\x1c\n\x14\x61\x63tive_fairness_keys\x18\x03 \x01(\x04\x12\x18\n\x10\x61\x63tive_consumers\x18\x04 \x01(\r\x12\x0f\n\x07quantum\x18\x05 \x01(\r\x12\x33\n\rper_key_stats\x18\x06 \x03(\x0b\x32\x1c.fila.v1.PerFairnessKeyStats\x12\x38\n\x12per_throttle_stats\x18\x07 \x03(\x0b\x32\x1c.fila.v1.PerThrottleKeyStats\x12\x16\n\x0eleader_node_id\x18\x08 \x01(\x04\x12\x19\n\x11replication_count\x18\t \x01(\r\"2\n\x0eRedriveRequest\x12\x11\n\tdlq_queue\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"#\n\x0fRedriveResponse\x12\x10\n\x08redriven\x18\x01 \x01(\x04\"\x13\n\x11ListQueuesRequest\"m\n\tQueueInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x04\x12\x11\n\tin_flight\x18\x03 \x01(\x04\x12\x18\n\x10\x61\x63tive_consumers\x18\x04 \x01(\r\x12\x16\n\x0eleader_node_id\x18\x05 \x01(\x04\"T\n\x12ListQueuesResponse\x12\"\n\x06queues\x18\x01 \x03(\x0b\x32\x12.fila.v1.QueueInfo\x12\x1a\n\x12\x63luster_node_count\x18\x02 \x01(\r\"Q\n\x13\x43reateApiKeyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rexpires_at_ms\x18\x02 \x01(\x04\x12\x15\n\ris_superadmin\x18\x03 \x01(\x08\"J\n\x14\x43reateApiKeyResponse\x12\x0e\n\x06key_id\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x15\n\ris_superadmin\x18\x03 \x01(\x08\"%\n\x13RevokeApiKeyRequest\x12\x0e\n\x06key_id\x18\x01 \x01(\t\"\x16\n\x14RevokeApiKeyResponse\"\x14\n\x12ListApiKeysRequest\"o\n\nApiKeyInfo\x12\x0e\n\x06key_id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x15\n\rcreated_at_ms\x18\x03 \x01(\x04\x12\x15\n\rexpires_at_ms\x18\x04 \x01(\x04\x12\x15\n\ris_superadmin\x18\x05 \x01(\x08\"8\n\x13ListApiKeysResponse\x12!\n\x04keys\x18\x01 \x03(\x0b\x32\x13.fila.v1.ApiKeyInfo\".\n\rAclPermission\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0f\n\x07pattern\x18\x02 \x01(\t\"L\n\rSetAclRequest\x12\x0e\n\x06key_id\x18\x01 \x01(\t\x12+\n\x0bpermissions\x18\x02 \x03(\x0b\x32\x16.fila.v1.AclPermission\"\x10\n\x0eSetAclResponse\"\x1f\n\rGetAclRequest\x12\x0e\n\x06key_id\x18\x01 \x01(\t\"d\n\x0eGetAclResponse\x12\x0e\n\x06key_id\x18\x01 \x01(\t\x12+\n\x0bpermissions\x18\x02 \x03(\x0b\x32\x16.fila.v1.AclPermission\x12\x15\n\ris_superadmin\x18\x03 \x01(\x08\x32\x8e\x07\n\tFilaAdmin\x12H\n\x0b\x43reateQueue\x12\x1b.fila.v1.CreateQueueRequest\x1a\x1c.fila.v1.CreateQueueResponse\x12H\n\x0b\x44\x65leteQueue\x12\x1b.fila.v1.DeleteQueueRequest\x1a\x1c.fila.v1.DeleteQueueResponse\x12\x42\n\tSetConfig\x12\x19.fila.v1.SetConfigRequest\x1a\x1a.fila.v1.SetConfigResponse\x12\x42\n\tGetConfig\x12\x19.fila.v1.GetConfigRequest\x1a\x1a.fila.v1.GetConfigResponse\x12\x45\n\nListConfig\x12\x1a.fila.v1.ListConfigRequest\x1a\x1b.fila.v1.ListConfigResponse\x12?\n\x08GetStats\x12\x18.fila.v1.GetStatsRequest\x1a\x19.fila.v1.GetStatsResponse\x12<\n\x07Redrive\x12\x17.fila.v1.RedriveRequest\x1a\x18.fila.v1.RedriveResponse\x12\x45\n\nListQueues\x12\x1a.fila.v1.ListQueuesRequest\x1a\x1b.fila.v1.ListQueuesResponse\x12K\n\x0c\x43reateApiKey\x12\x1c.fila.v1.CreateApiKeyRequest\x1a\x1d.fila.v1.CreateApiKeyResponse\x12K\n\x0cRevokeApiKey\x12\x1c.fila.v1.RevokeApiKeyRequest\x1a\x1d.fila.v1.RevokeApiKeyResponse\x12H\n\x0bListApiKeys\x12\x1b.fila.v1.ListApiKeysRequest\x1a\x1c.fila.v1.ListApiKeysResponse\x12\x39\n\x06SetAcl\x12\x16.fila.v1.SetAclRequest\x1a\x17.fila.v1.SetAclResponse\x12\x39\n\x06GetAcl\x12\x16.fila.v1.GetAclRequest\x1a\x17.fila.v1.GetAclResponseb\x06proto3" - -pool = ::Google::Protobuf::DescriptorPool.generated_pool -pool.add_serialized_file(descriptor_data) - -module Fila - module V1 - CreateQueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.CreateQueueRequest").msgclass - QueueConfig = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.QueueConfig").msgclass - CreateQueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.CreateQueueResponse").msgclass - DeleteQueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.DeleteQueueRequest").msgclass - DeleteQueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.DeleteQueueResponse").msgclass - SetConfigRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.SetConfigRequest").msgclass - SetConfigResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.SetConfigResponse").msgclass - GetConfigRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetConfigRequest").msgclass - GetConfigResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetConfigResponse").msgclass - ConfigEntry = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ConfigEntry").msgclass - ListConfigRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListConfigRequest").msgclass - ListConfigResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListConfigResponse").msgclass - GetStatsRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetStatsRequest").msgclass - PerFairnessKeyStats = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.PerFairnessKeyStats").msgclass - PerThrottleKeyStats = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.PerThrottleKeyStats").msgclass - GetStatsResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetStatsResponse").msgclass - RedriveRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.RedriveRequest").msgclass - RedriveResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.RedriveResponse").msgclass - ListQueuesRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListQueuesRequest").msgclass - QueueInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.QueueInfo").msgclass - ListQueuesResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListQueuesResponse").msgclass - CreateApiKeyRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.CreateApiKeyRequest").msgclass - CreateApiKeyResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.CreateApiKeyResponse").msgclass - RevokeApiKeyRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.RevokeApiKeyRequest").msgclass - RevokeApiKeyResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.RevokeApiKeyResponse").msgclass - ListApiKeysRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListApiKeysRequest").msgclass - ApiKeyInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ApiKeyInfo").msgclass - ListApiKeysResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ListApiKeysResponse").msgclass - AclPermission = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AclPermission").msgclass - SetAclRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.SetAclRequest").msgclass - SetAclResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.SetAclResponse").msgclass - GetAclRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetAclRequest").msgclass - GetAclResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.GetAclResponse").msgclass - end -end diff --git a/lib/fila/proto/fila/v1/admin_services_pb.rb b/lib/fila/proto/fila/v1/admin_services_pb.rb deleted file mode 100644 index 71de19e..0000000 --- a/lib/fila/proto/fila/v1/admin_services_pb.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# Source: fila/v1/admin.proto for package 'fila.v1' - -require 'grpc' -require 'fila/v1/admin_pb' - -module Fila - module V1 - module FilaAdmin - # Admin RPCs for operators and the CLI. - class Service - - include ::GRPC::GenericService - - self.marshal_class_method = :encode - self.unmarshal_class_method = :decode - self.service_name = 'fila.v1.FilaAdmin' - - rpc :CreateQueue, ::Fila::V1::CreateQueueRequest, ::Fila::V1::CreateQueueResponse - rpc :DeleteQueue, ::Fila::V1::DeleteQueueRequest, ::Fila::V1::DeleteQueueResponse - rpc :SetConfig, ::Fila::V1::SetConfigRequest, ::Fila::V1::SetConfigResponse - rpc :GetConfig, ::Fila::V1::GetConfigRequest, ::Fila::V1::GetConfigResponse - rpc :ListConfig, ::Fila::V1::ListConfigRequest, ::Fila::V1::ListConfigResponse - rpc :GetStats, ::Fila::V1::GetStatsRequest, ::Fila::V1::GetStatsResponse - rpc :Redrive, ::Fila::V1::RedriveRequest, ::Fila::V1::RedriveResponse - rpc :ListQueues, ::Fila::V1::ListQueuesRequest, ::Fila::V1::ListQueuesResponse - # API key management. CreateApiKey bypasses auth (bootstrap); others require a valid key. - rpc :CreateApiKey, ::Fila::V1::CreateApiKeyRequest, ::Fila::V1::CreateApiKeyResponse - rpc :RevokeApiKey, ::Fila::V1::RevokeApiKeyRequest, ::Fila::V1::RevokeApiKeyResponse - rpc :ListApiKeys, ::Fila::V1::ListApiKeysRequest, ::Fila::V1::ListApiKeysResponse - # Per-key ACL management. - rpc :SetAcl, ::Fila::V1::SetAclRequest, ::Fila::V1::SetAclResponse - rpc :GetAcl, ::Fila::V1::GetAclRequest, ::Fila::V1::GetAclResponse - end - - Stub = Service.rpc_stub_class - end - end -end diff --git a/lib/fila/proto/fila/v1/messages_pb.rb b/lib/fila/proto/fila/v1/messages_pb.rb deleted file mode 100644 index ae3674f..0000000 --- a/lib/fila/proto/fila/v1/messages_pb.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: fila/v1/messages.proto - -require 'google/protobuf' - -require 'google/protobuf/timestamp_pb' - - -descriptor_data = "\n\x16\x66ila/v1/messages.proto\x12\x07\x66ila.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xe2\x01\n\x07Message\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\x07headers\x18\x02 \x03(\x0b\x32\x1d.fila.v1.Message.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12*\n\x08metadata\x18\x04 \x01(\x0b\x32\x18.fila.v1.MessageMetadata\x12.\n\ntimestamps\x18\x05 \x01(\x0b\x32\x1a.fila.v1.MessageTimestamps\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"w\n\x0fMessageMetadata\x12\x14\n\x0c\x66\x61irness_key\x18\x01 \x01(\t\x12\x0e\n\x06weight\x18\x02 \x01(\r\x12\x15\n\rthrottle_keys\x18\x03 \x03(\t\x12\x15\n\rattempt_count\x18\x04 \x01(\r\x12\x10\n\x08queue_id\x18\x05 \x01(\t\"s\n\x11MessageTimestamps\x12/\n\x0b\x65nqueued_at\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12-\n\tleased_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestampb\x06proto3" - -pool = ::Google::Protobuf::DescriptorPool.generated_pool -pool.add_serialized_file(descriptor_data) - -module Fila - module V1 - Message = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.Message").msgclass - MessageMetadata = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.MessageMetadata").msgclass - MessageTimestamps = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.MessageTimestamps").msgclass - end -end diff --git a/lib/fila/proto/fila/v1/service_pb.rb b/lib/fila/proto/fila/v1/service_pb.rb deleted file mode 100644 index 9751f67..0000000 --- a/lib/fila/proto/fila/v1/service_pb.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: fila/v1/service.proto - -require 'google/protobuf' - -require 'fila/v1/messages_pb' - - -descriptor_data = "\n\x15\x66ila/v1/service.proto\x12\x07\x66ila.v1\x1a\x16\x66ila/v1/messages.proto\"\x97\x01\n\x0e\x45nqueueMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x35\n\x07headers\x18\x02 \x03(\x0b\x32$.fila.v1.EnqueueMessage.HeadersEntry\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\";\n\x0e\x45nqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueMessage\"W\n\rEnqueueResult\x12\x14\n\nmessage_id\x18\x01 \x01(\tH\x00\x12&\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x15.fila.v1.EnqueueErrorH\x00\x42\x08\n\x06result\"H\n\x0c\x45nqueueError\x12\'\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x19.fila.v1.EnqueueErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x0f\x45nqueueResponse\x12\'\n\x07results\x18\x01 \x03(\x0b\x32\x16.fila.v1.EnqueueResult\"\x1f\n\x0e\x43onsumeRequest\x12\r\n\x05queue\x18\x01 \x01(\t\"5\n\x0f\x43onsumeResponse\x12\"\n\x08messages\x18\x01 \x03(\x0b\x32\x10.fila.v1.Message\"/\n\nAckMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\"3\n\nAckRequest\x12%\n\x08messages\x18\x01 \x03(\x0b\x32\x13.fila.v1.AckMessage\"a\n\tAckResult\x12&\n\x07success\x18\x01 \x01(\x0b\x32\x13.fila.v1.AckSuccessH\x00\x12\"\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x11.fila.v1.AckErrorH\x00\x42\x08\n\x06result\"\x0c\n\nAckSuccess\"@\n\x08\x41\x63kError\x12#\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x15.fila.v1.AckErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0b\x41\x63kResponse\x12#\n\x07results\x18\x01 \x03(\x0b\x32\x12.fila.v1.AckResult\"?\n\x0bNackMessage\x12\r\n\x05queue\x18\x01 \x01(\t\x12\x12\n\nmessage_id\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"5\n\x0bNackRequest\x12&\n\x08messages\x18\x01 \x03(\x0b\x32\x14.fila.v1.NackMessage\"d\n\nNackResult\x12\'\n\x07success\x18\x01 \x01(\x0b\x32\x14.fila.v1.NackSuccessH\x00\x12#\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x12.fila.v1.NackErrorH\x00\x42\x08\n\x06result\"\r\n\x0bNackSuccess\"B\n\tNackError\x12$\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x16.fila.v1.NackErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"4\n\x0cNackResponse\x12$\n\x07results\x18\x01 \x03(\x0b\x32\x13.fila.v1.NackResult\"Z\n\x14StreamEnqueueRequest\x12)\n\x08messages\x18\x01 \x03(\x0b\x32\x17.fila.v1.EnqueueMessage\x12\x17\n\x0fsequence_number\x18\x02 \x01(\x04\"Y\n\x15StreamEnqueueResponse\x12\x17\n\x0fsequence_number\x18\x01 \x01(\x04\x12\'\n\x07results\x18\x02 \x03(\x0b\x32\x16.fila.v1.EnqueueResult*\xc4\x01\n\x10\x45nqueueErrorCode\x12\"\n\x1e\x45NQUEUE_ERROR_CODE_UNSPECIFIED\x10\x00\x12&\n\"ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND\x10\x01\x12\x1e\n\x1a\x45NQUEUE_ERROR_CODE_STORAGE\x10\x02\x12\x1a\n\x16\x45NQUEUE_ERROR_CODE_LUA\x10\x03\x12(\n$ENQUEUE_ERROR_CODE_PERMISSION_DENIED\x10\x04*\x96\x01\n\x0c\x41\x63kErrorCode\x12\x1e\n\x1a\x41\x43K_ERROR_CODE_UNSPECIFIED\x10\x00\x12$\n ACK_ERROR_CODE_MESSAGE_NOT_FOUND\x10\x01\x12\x1a\n\x16\x41\x43K_ERROR_CODE_STORAGE\x10\x02\x12$\n ACK_ERROR_CODE_PERMISSION_DENIED\x10\x03*\x9b\x01\n\rNackErrorCode\x12\x1f\n\x1bNACK_ERROR_CODE_UNSPECIFIED\x10\x00\x12%\n!NACK_ERROR_CODE_MESSAGE_NOT_FOUND\x10\x01\x12\x1b\n\x17NACK_ERROR_CODE_STORAGE\x10\x02\x12%\n!NACK_ERROR_CODE_PERMISSION_DENIED\x10\x03\x32\xc6\x02\n\x0b\x46ilaService\x12<\n\x07\x45nqueue\x12\x17.fila.v1.EnqueueRequest\x1a\x18.fila.v1.EnqueueResponse\x12R\n\rStreamEnqueue\x12\x1d.fila.v1.StreamEnqueueRequest\x1a\x1e.fila.v1.StreamEnqueueResponse(\x01\x30\x01\x12>\n\x07\x43onsume\x12\x17.fila.v1.ConsumeRequest\x1a\x18.fila.v1.ConsumeResponse0\x01\x12\x30\n\x03\x41\x63k\x12\x13.fila.v1.AckRequest\x1a\x14.fila.v1.AckResponse\x12\x33\n\x04Nack\x12\x14.fila.v1.NackRequest\x1a\x15.fila.v1.NackResponseb\x06proto3" - -pool = ::Google::Protobuf::DescriptorPool.generated_pool -pool.add_serialized_file(descriptor_data) - -module Fila - module V1 - EnqueueMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueMessage").msgclass - EnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueRequest").msgclass - EnqueueResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueResult").msgclass - EnqueueError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueError").msgclass - EnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueResponse").msgclass - ConsumeRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ConsumeRequest").msgclass - ConsumeResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.ConsumeResponse").msgclass - AckMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckMessage").msgclass - AckRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckRequest").msgclass - AckResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckResult").msgclass - AckSuccess = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckSuccess").msgclass - AckError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckError").msgclass - AckResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckResponse").msgclass - NackMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackMessage").msgclass - NackRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackRequest").msgclass - NackResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackResult").msgclass - NackSuccess = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackSuccess").msgclass - NackError = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackError").msgclass - NackResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackResponse").msgclass - StreamEnqueueRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.StreamEnqueueRequest").msgclass - StreamEnqueueResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.StreamEnqueueResponse").msgclass - EnqueueErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.EnqueueErrorCode").enummodule - AckErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.AckErrorCode").enummodule - NackErrorCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("fila.v1.NackErrorCode").enummodule - end -end diff --git a/lib/fila/proto/fila/v1/service_services_pb.rb b/lib/fila/proto/fila/v1/service_services_pb.rb deleted file mode 100644 index 93d38ab..0000000 --- a/lib/fila/proto/fila/v1/service_services_pb.rb +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# Source: fila/v1/service.proto for package 'fila.v1' - -require 'grpc' -require 'fila/v1/service_pb' - -module Fila - module V1 - module FilaService - # Hot-path RPCs for producers and consumers. - class Service - - include ::GRPC::GenericService - - self.marshal_class_method = :encode - self.unmarshal_class_method = :decode - self.service_name = 'fila.v1.FilaService' - - rpc :Enqueue, ::Fila::V1::EnqueueRequest, ::Fila::V1::EnqueueResponse - rpc :StreamEnqueue, stream(::Fila::V1::StreamEnqueueRequest), stream(::Fila::V1::StreamEnqueueResponse) - rpc :Consume, ::Fila::V1::ConsumeRequest, stream(::Fila::V1::ConsumeResponse) - rpc :Ack, ::Fila::V1::AckRequest, ::Fila::V1::AckResponse - rpc :Nack, ::Fila::V1::NackRequest, ::Fila::V1::NackResponse - end - - Stub = Service.rpc_stub_class - end - end -end diff --git a/lib/fila/version.rb b/lib/fila/version.rb index ecc3d06..efdd9b7 100644 --- a/lib/fila/version.rb +++ b/lib/fila/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Fila - VERSION = '0.4.0' + VERSION = '0.5.0' end diff --git a/proto/fila/v1/admin.proto b/proto/fila/v1/admin.proto deleted file mode 100644 index 886e58d..0000000 --- a/proto/fila/v1/admin.proto +++ /dev/null @@ -1,197 +0,0 @@ -syntax = "proto3"; -package fila.v1; - -// Admin RPCs for operators and the CLI. -service FilaAdmin { - rpc CreateQueue(CreateQueueRequest) returns (CreateQueueResponse); - rpc DeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse); - rpc SetConfig(SetConfigRequest) returns (SetConfigResponse); - rpc GetConfig(GetConfigRequest) returns (GetConfigResponse); - rpc ListConfig(ListConfigRequest) returns (ListConfigResponse); - rpc GetStats(GetStatsRequest) returns (GetStatsResponse); - rpc Redrive(RedriveRequest) returns (RedriveResponse); - rpc ListQueues(ListQueuesRequest) returns (ListQueuesResponse); - - // API key management. CreateApiKey bypasses auth (bootstrap); others require a valid key. - rpc CreateApiKey(CreateApiKeyRequest) returns (CreateApiKeyResponse); - rpc RevokeApiKey(RevokeApiKeyRequest) returns (RevokeApiKeyResponse); - rpc ListApiKeys(ListApiKeysRequest) returns (ListApiKeysResponse); - - // Per-key ACL management. - rpc SetAcl(SetAclRequest) returns (SetAclResponse); - rpc GetAcl(GetAclRequest) returns (GetAclResponse); -} - -message CreateQueueRequest { - string name = 1; - QueueConfig config = 2; -} - -message QueueConfig { - string on_enqueue_script = 1; - string on_failure_script = 2; - uint64 visibility_timeout_ms = 3; -} - -message CreateQueueResponse { - string queue_id = 1; -} - -message DeleteQueueRequest { - string queue = 1; -} - -message DeleteQueueResponse {} - -message SetConfigRequest { - string key = 1; - string value = 2; -} - -message SetConfigResponse {} - -message GetConfigRequest { - string key = 1; -} - -message GetConfigResponse { - string value = 1; -} - -message ConfigEntry { - string key = 1; - string value = 2; -} - -message ListConfigRequest { - string prefix = 1; -} - -message ListConfigResponse { - repeated ConfigEntry entries = 1; - uint32 total_count = 2; -} - -message GetStatsRequest { - string queue = 1; -} - -message PerFairnessKeyStats { - string key = 1; - uint64 pending_count = 2; - int64 current_deficit = 3; - uint32 weight = 4; -} - -message PerThrottleKeyStats { - string key = 1; - double tokens = 2; - double rate_per_second = 3; - double burst = 4; -} - -message GetStatsResponse { - uint64 depth = 1; - uint64 in_flight = 2; - uint64 active_fairness_keys = 3; - uint32 active_consumers = 4; - uint32 quantum = 5; - repeated PerFairnessKeyStats per_key_stats = 6; - repeated PerThrottleKeyStats per_throttle_stats = 7; - // Cluster fields (0 when not in cluster mode). - uint64 leader_node_id = 8; - uint32 replication_count = 9; -} - -message RedriveRequest { - string dlq_queue = 1; - uint64 count = 2; -} - -message RedriveResponse { - uint64 redriven = 1; -} - -message ListQueuesRequest {} - -message QueueInfo { - string name = 1; - uint64 depth = 2; - uint64 in_flight = 3; - uint32 active_consumers = 4; - uint64 leader_node_id = 5; -} - -message ListQueuesResponse { - repeated QueueInfo queues = 1; - uint32 cluster_node_count = 2; -} - -// --- API Key Management --- - -message CreateApiKeyRequest { - /// Human-readable label for the key. - string name = 1; - /// Optional Unix timestamp (milliseconds) after which the key expires. - /// 0 means no expiration. - uint64 expires_at_ms = 2; - /// When true, the key bypasses all ACL checks (superadmin). - bool is_superadmin = 3; -} - -message CreateApiKeyResponse { - /// Opaque key ID for management operations (revoke, list, set-acl). - string key_id = 1; - /// Plaintext API key. Returned once — store it securely. - string key = 2; - /// Whether this key has superadmin privileges. - bool is_superadmin = 3; -} - -message RevokeApiKeyRequest { - string key_id = 1; -} - -message RevokeApiKeyResponse {} - -message ListApiKeysRequest {} - -message ApiKeyInfo { - string key_id = 1; - string name = 2; - uint64 created_at_ms = 3; - /// 0 means no expiration. - uint64 expires_at_ms = 4; - bool is_superadmin = 5; -} - -message ListApiKeysResponse { - repeated ApiKeyInfo keys = 1; -} - -// --- ACL Management --- - -/// A single permission grant: kind (produce/consume/admin) + queue pattern. -message AclPermission { - /// One of: "produce", "consume", "admin". - string kind = 1; - /// Queue name or wildcard ("*" or "orders.*"). - string pattern = 2; -} - -message SetAclRequest { - string key_id = 1; - repeated AclPermission permissions = 2; -} - -message SetAclResponse {} - -message GetAclRequest { - string key_id = 1; -} - -message GetAclResponse { - string key_id = 1; - repeated AclPermission permissions = 2; - bool is_superadmin = 3; -} diff --git a/proto/fila/v1/messages.proto b/proto/fila/v1/messages.proto deleted file mode 100644 index a0709cf..0000000 --- a/proto/fila/v1/messages.proto +++ /dev/null @@ -1,28 +0,0 @@ -syntax = "proto3"; -package fila.v1; - -import "google/protobuf/timestamp.proto"; - -// Core message envelope persisted in the broker. -message Message { - string id = 1; - map headers = 2; - bytes payload = 3; - MessageMetadata metadata = 4; - MessageTimestamps timestamps = 5; -} - -// Broker-assigned scheduling metadata. -message MessageMetadata { - string fairness_key = 1; - uint32 weight = 2; - repeated string throttle_keys = 3; - uint32 attempt_count = 4; - string queue_id = 5; -} - -// Lifecycle timestamps attached to every message. -message MessageTimestamps { - google.protobuf.Timestamp enqueued_at = 1; - google.protobuf.Timestamp leased_at = 2; -} diff --git a/proto/fila/v1/service.proto b/proto/fila/v1/service.proto deleted file mode 100644 index 7d1db79..0000000 --- a/proto/fila/v1/service.proto +++ /dev/null @@ -1,142 +0,0 @@ -syntax = "proto3"; -package fila.v1; - -import "fila/v1/messages.proto"; - -// Hot-path RPCs for producers and consumers. -service FilaService { - rpc Enqueue(EnqueueRequest) returns (EnqueueResponse); - rpc StreamEnqueue(stream StreamEnqueueRequest) returns (stream StreamEnqueueResponse); - rpc Consume(ConsumeRequest) returns (stream ConsumeResponse); - rpc Ack(AckRequest) returns (AckResponse); - rpc Nack(NackRequest) returns (NackResponse); -} - -// Individual message to enqueue. -message EnqueueMessage { - string queue = 1; - map headers = 2; - bytes payload = 3; -} - -// Enqueue one or more messages. -message EnqueueRequest { - repeated EnqueueMessage messages = 1; -} - -// Per-message enqueue result. -message EnqueueResult { - oneof result { - string message_id = 1; - EnqueueError error = 2; - } -} - -// Typed enqueue error with structured error code. -message EnqueueError { - EnqueueErrorCode code = 1; - string message = 2; -} - -enum EnqueueErrorCode { - ENQUEUE_ERROR_CODE_UNSPECIFIED = 0; - ENQUEUE_ERROR_CODE_QUEUE_NOT_FOUND = 1; - ENQUEUE_ERROR_CODE_STORAGE = 2; - ENQUEUE_ERROR_CODE_LUA = 3; - ENQUEUE_ERROR_CODE_PERMISSION_DENIED = 4; -} - -// One result per input message. -message EnqueueResponse { - repeated EnqueueResult results = 1; -} - -message ConsumeRequest { - string queue = 1; -} - -message ConsumeResponse { - repeated Message messages = 1; -} - -// Individual ack item. -message AckMessage { - string queue = 1; - string message_id = 2; -} - -message AckRequest { - repeated AckMessage messages = 1; -} - -message AckResult { - oneof result { - AckSuccess success = 1; - AckError error = 2; - } -} - -message AckSuccess {} - -message AckError { - AckErrorCode code = 1; - string message = 2; -} - -enum AckErrorCode { - ACK_ERROR_CODE_UNSPECIFIED = 0; - ACK_ERROR_CODE_MESSAGE_NOT_FOUND = 1; - ACK_ERROR_CODE_STORAGE = 2; - ACK_ERROR_CODE_PERMISSION_DENIED = 3; -} - -message AckResponse { - repeated AckResult results = 1; -} - -// Individual nack item. -message NackMessage { - string queue = 1; - string message_id = 2; - string error = 3; -} - -message NackRequest { - repeated NackMessage messages = 1; -} - -message NackResult { - oneof result { - NackSuccess success = 1; - NackError error = 2; - } -} - -message NackSuccess {} - -message NackError { - NackErrorCode code = 1; - string message = 2; -} - -enum NackErrorCode { - NACK_ERROR_CODE_UNSPECIFIED = 0; - NACK_ERROR_CODE_MESSAGE_NOT_FOUND = 1; - NACK_ERROR_CODE_STORAGE = 2; - NACK_ERROR_CODE_PERMISSION_DENIED = 3; -} - -message NackResponse { - repeated NackResult results = 1; -} - -// Stream enqueue — per-write batch with sequence tracking. -message StreamEnqueueRequest { - repeated EnqueueMessage messages = 1; - uint64 sequence_number = 2; -} - -message StreamEnqueueResponse { - uint64 sequence_number = 1; - repeated EnqueueResult results = 2; -} diff --git a/test/test_batch.rb b/test/test_batch.rb index 62f1b41..295c6ce 100644 --- a/test/test_batch.rb +++ b/test/test_batch.rb @@ -222,15 +222,6 @@ def test_invalid_batch_mode_raises Fila::Client.new('localhost:5555', batch_mode: :invalid) end end - - def test_valid_batch_modes_accepted - # These should not raise (but won't connect since server isn't on this port). - # Just verify argument validation passes. - %i[auto linger disabled].each do |mode| - client = Fila::Client.new('localhost:19999', batch_mode: mode) - client.close - end - end end class TestCloseFlush < Minitest::Test diff --git a/test/test_helper.rb b/test/test_helper.rb index c88a349..99d0ab5 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -3,12 +3,9 @@ require 'minitest/autorun' require 'tmpdir' require 'socket' -require 'grpc' $LOAD_PATH.unshift File.expand_path('../lib', __dir__) -$LOAD_PATH.unshift File.expand_path('../lib/fila/proto', __dir__) require 'fila' -require_relative '../lib/fila/proto/fila/v1/admin_services_pb' FILA_SERVER_BIN = ENV.fetch('FILA_SERVER_BIN') do File.join(__dir__, '..', '..', 'fila', 'target', 'release', 'fila-server') @@ -25,10 +22,9 @@ def self.find_free_port # Start a fila-server instance. # - # @param tls_config [Hash, nil] optional TLS configuration with keys: - # :ca_cert_path, :server_cert_path, :server_key_path + # @param tls_config [Hash, nil] optional TLS configuration # @param bootstrap_apikey [String, nil] optional bootstrap API key - # @return [Hash] server info with :addr, :pid, :data_dir, :admin_stub + # @return [Hash] server info with :addr, :pid, :data_dir, etc. def self.start(tls_config: nil, bootstrap_apikey: nil) port = find_free_port addr = "127.0.0.1:#{port}" @@ -62,28 +58,24 @@ def self.start(tls_config: nil, bootstrap_apikey: nil) ) end - # Build credentials for admin stub. - # client_ca_cert_path is always needed to verify server cert; ca_cert_path is only for mTLS. - credentials = :this_channel_is_insecure + # Build connection options for admin operations. + conn_opts = { tls: false } if tls_config ca_path = tls_config[:client_ca_cert_path] || tls_config[:ca_cert_path] if ca_path - ca_cert = File.read(ca_path) - client_key = tls_config[:client_key_path] ? File.read(tls_config[:client_key_path]) : nil - client_cert = tls_config[:client_cert_path] ? File.read(tls_config[:client_cert_path]) : nil - credentials = GRPC::Core::ChannelCredentials.new(ca_cert, client_key, client_cert) + conn_opts[:ca_cert] = File.read(ca_path) + conn_opts[:client_cert] = File.read(tls_config[:client_cert_path]) if tls_config[:client_cert_path] + conn_opts[:client_key] = File.read(tls_config[:client_key_path]) if tls_config[:client_key_path] end end - - admin_metadata = {} - admin_metadata['authorization'] = "Bearer #{bootstrap_apikey}" if bootstrap_apikey + conn_opts[:api_key] = bootstrap_apikey if bootstrap_apikey # Wait for server ready. deadline = Time.now + 10 ready = false while Time.now < deadline begin - try_list_queues(addr, credentials: credentials, metadata: admin_metadata) + try_list_queues(addr, conn_opts) ready = true break rescue StandardError @@ -103,14 +95,11 @@ def self.start(tls_config: nil, bootstrap_apikey: nil) raise "fila-server failed to start within 10s on #{addr}\nConfig:\n#{toml}\nStderr:\n#{stderr_output}" end - admin_stub = ::Fila::V1::FilaAdmin::Stub.new(addr, credentials) - { addr: addr, pid: pid, data_dir: data_dir, - admin_stub: admin_stub, - admin_metadata: admin_metadata + conn_opts: conn_opts } end @@ -123,12 +112,14 @@ def self.stop(server) end def self.create_queue(server, name) - req = ::Fila::V1::CreateQueueRequest.new(name: name, config: {}) - server[:admin_stub].create_queue(req, metadata: server[:admin_metadata] || {}) + client = Fila::Client.new(server[:addr], batch_mode: :disabled, **server[:conn_opts]) + client.create_queue(name: name) + client.close end - def self.try_list_queues(addr, credentials: :this_channel_is_insecure, metadata: {}) - stub = ::Fila::V1::FilaAdmin::Stub.new(addr, credentials) - stub.list_queues(::Fila::V1::ListQueuesRequest.new, metadata: metadata) + def self.try_list_queues(addr, conn_opts) + client = Fila::Client.new(addr, batch_mode: :disabled, **conn_opts) + client.list_queues + client.close end end diff --git a/test/test_tls_auth.rb b/test/test_tls_auth.rb index 7aad862..6132977 100644 --- a/test/test_tls_auth.rb +++ b/test/test_tls_auth.rb @@ -79,6 +79,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server end @@ -91,14 +92,13 @@ def test_enqueue_with_api_key end def test_enqueue_without_api_key_rejected - client_no_key = Fila::Client.new(@server[:addr]) TestServerHelper.create_queue(@server, 'auth-reject-queue') - # Without API key, the server should reject the request with Unauthenticated. - err = assert_raises(Fila::RPCError) do + err = assert_raises(Fila::AuthenticationError) do + client_no_key = Fila::Client.new(@server[:addr]) client_no_key.enqueue(queue: 'auth-reject-queue', payload: 'should fail') end - assert_equal 16, err.code # GRPC::Core::StatusCodes::UNAUTHENTICATED + assert_match(/unauthorized/i, err.message) end def test_consume_with_api_key @@ -121,8 +121,6 @@ def setup @cert_dir = Dir.mktmpdir('fila-certs-') @certs = CertHelper.generate_certs(@cert_dir) - # Server-only TLS: omit ca_cert_path so server does not require client certs. - # client_ca_cert_path is used by the test client to verify the server cert. @server = TestServerHelper.start( tls_config: { server_cert_path: @certs[:server_cert], @@ -138,6 +136,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server FileUtils.rm_rf(@cert_dir) if @cert_dir end @@ -189,6 +188,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server FileUtils.rm_rf(@cert_dir) if @cert_dir end @@ -208,8 +208,6 @@ def setup @certs = CertHelper.generate_certs(@cert_dir) @bootstrap_key = 'tls-bootstrap-key-67890' - # Server-only TLS + API key: omit ca_cert_path so server does not require client certs. - # client_ca_cert_path is used by the test client to verify the server cert. @server = TestServerHelper.start( tls_config: { server_cert_path: @certs[:server_cert], @@ -227,6 +225,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server FileUtils.rm_rf(@cert_dir) if @cert_dir end @@ -240,16 +239,16 @@ def test_enqueue_with_tls_and_api_key end def test_no_api_key_over_tls_rejected - client_no_key = Fila::Client.new( - @server[:addr], - ca_cert: File.read(@certs[:ca_cert]) - ) TestServerHelper.create_queue(@server, 'tls-auth-reject-queue') - err = assert_raises(Fila::RPCError) do + err = assert_raises(Fila::AuthenticationError) do + client_no_key = Fila::Client.new( + @server[:addr], + ca_cert: File.read(@certs[:ca_cert]) + ) client_no_key.enqueue(queue: 'tls-auth-reject-queue', payload: 'should fail') end - assert_equal 16, err.code # UNAUTHENTICATED + assert_match(/unauthorized/i, err.message) end end @@ -260,6 +259,7 @@ def setup end def teardown + @client&.close TestServerHelper.stop(@server) if @server end From 6f9a966dcad1ba97ad593463365e480174ceea9c Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:05:18 -0300 Subject: [PATCH 14/19] fix: resolve rubocop lint offenses and test helper timeout --- fila-client.gemspec | 2 +- lib/fila/batcher.rb | 47 ++++++++--------- lib/fila/client.rb | 101 +++++++++++++++++++----------------- lib/fila/fibp/codec.rb | 10 ++-- lib/fila/fibp/connection.rb | 60 +++++++++++---------- test/test_helper.rb | 13 +++-- 6 files changed, 122 insertions(+), 111 deletions(-) diff --git a/fila-client.gemspec b/fila-client.gemspec index 2dfbc30..1881081 100644 --- a/fila-client.gemspec +++ b/fila-client.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |spec| spec.version = Fila::VERSION spec.authors = ['Faisca'] spec.summary = 'Ruby client SDK for the Fila message broker' - spec.description = "Idiomatic Ruby client for the Fila message broker using the FIBP binary protocol." + spec.description = 'Idiomatic Ruby client for the Fila message broker using the FIBP binary protocol.' spec.homepage = 'https://github.com/faiscadev/fila-ruby' spec.license = 'AGPL-3.0-or-later' spec.required_ruby_version = '>= 3.1' diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 795bff1..5221125 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -6,7 +6,7 @@ module Fila # and linger (timer-based) modes. # # @api private - class Batcher # rubocop:disable Metrics/ClassLength + class Batcher # An item queued for batching, pairing a message hash with its result slot. BatchItem = Struct.new(:message, :result_queue, keyword_init: true) @@ -123,32 +123,31 @@ def drain_nonblocking(batch) # Flush a batch of items via FIBP Enqueue. def flush_batch(items) - messages = items.map(&:message) - payload = encode_enqueue_batch(messages) - opcode, resp = @conn.request(FIBP::Opcodes::ENQUEUE, payload) - - if opcode == FIBP::Opcodes::ERROR - reader = FIBP::Codec::Reader.new(resp) - code = reader.read_u8 - message = reader.read_string - broadcast_error(items, RPCError.new(code, message)) - return - end + opcode, resp = send_enqueue_batch(items) + handle_flush_error(items, resp) && return if opcode == FIBP::Opcodes::ERROR + + dispatch_results(items, resp) + rescue StandardError => e + broadcast_error(items, Fila::Error.new(e.message)) + end + + def send_enqueue_batch(items) + payload = encode_enqueue_batch(items.map(&:message)) + @conn.request(FIBP::Opcodes::ENQUEUE, payload) + end + def handle_flush_error(items, resp) reader = FIBP::Codec::Reader.new(resp) - count = reader.read_u32 + broadcast_error(items, RPCError.new(reader.read_u8, reader.read_string)) + end + def dispatch_results(items, resp) + reader = FIBP::Codec::Reader.new(resp) + count = reader.read_u32 items.each_with_index do |item, idx| - if idx < count - code = reader.read_u8 - msg_id = reader.read_string - item.result_queue.push(result_to_outcome(code, msg_id)) - else - item.result_queue.push(Fila::Error.new('no result from server')) - end + outcome = idx < count ? result_to_outcome(reader.read_u8, reader.read_string) : Fila::Error.new('no result from server') + item.result_queue.push(outcome) end - rescue StandardError => e - broadcast_error(items, Fila::Error.new(e.message)) end def encode_enqueue_batch(messages) @@ -165,9 +164,9 @@ def result_to_outcome(code, msg_id) case code when FIBP::ErrorCodes::OK then msg_id when FIBP::ErrorCodes::QUEUE_NOT_FOUND - QueueNotFoundError.new("enqueue: queue not found") + QueueNotFoundError.new('enqueue: queue not found') else - RPCError.new(code, "enqueue failed") + RPCError.new(code, 'enqueue failed') end end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 2fd5d0d..136994f 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -93,9 +93,7 @@ def enqueue_many(messages) payload = encode_enqueue_batch(messages) opcode, resp = @conn.request(FIBP::Opcodes::ENQUEUE, payload) - if opcode == FIBP::Opcodes::ERROR - raise_from_error_frame(resp) - end + raise_from_error_frame(resp) if opcode == FIBP::Opcodes::ERROR decode_enqueue_results(resp) end @@ -129,9 +127,9 @@ def ack(queue:, msg_id:) case code when FIBP::ErrorCodes::MESSAGE_NOT_FOUND - raise MessageNotFoundError, "ack: message not found" + raise MessageNotFoundError, 'ack: message not found' else - raise RPCError.new(code, "ack failed") + raise RPCError.new(code, 'ack failed') end end @@ -158,9 +156,9 @@ def nack(queue:, msg_id:, error:) case code when FIBP::ErrorCodes::MESSAGE_NOT_FOUND - raise MessageNotFoundError, "nack: message not found" + raise MessageNotFoundError, 'nack: message not found' else - raise RPCError.new(code, "nack failed") + raise RPCError.new(code, 'nack failed') end end @@ -191,7 +189,7 @@ def create_queue(name:, on_enqueue_script: nil, on_failure_script: nil, visibili when FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS raise QueueAlreadyExistsError, "queue '#{name}' already exists" else - raise RPCError.new(code, "create queue failed") + raise RPCError.new(code, 'create queue failed') end end @@ -440,9 +438,9 @@ def enqueue_single(msg) case code when FIBP::ErrorCodes::OK then msg_id when FIBP::ErrorCodes::QUEUE_NOT_FOUND - raise QueueNotFoundError, "enqueue: queue not found" + raise QueueNotFoundError, 'enqueue: queue not found' else - raise RPCError.new(code, "enqueue failed") + raise RPCError.new(code, 'enqueue failed') end end @@ -470,39 +468,45 @@ def decode_enqueue_results(resp) end end - def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/MethodLength + def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics payload = FIBP::Codec.encode_string(queue) - delivery_queue = Queue.new consumer_done = false rid = nil - begin - rid, response = @conn.subscribe(FIBP::Opcodes::CONSUME, payload) do |_opcode, del_payload| - delivery_queue.push(del_payload) unless consumer_done - end + rid, response = subscribe_to_queue(payload, delivery_queue, consumer_done) + check_consume_response(response) + consume_delivery_loop(delivery_queue, &block) + rescue NotLeaderError => e + raise if redirected || e.leader_addr.nil? + + reconnect_to(e.leader_addr) + consume_with_redirect(queue: queue, redirected: true, &block) + rescue LocalJumpError + nil # Consumer break + ensure + consumer_done = true + @conn&.cancel_consume(rid) if rid + end - opcode, resp_payload = response - raise_from_error_frame(resp_payload) if opcode == FIBP::Opcodes::ERROR + def subscribe_to_queue(payload, delivery_queue, consumer_done) + @conn.subscribe(FIBP::Opcodes::CONSUME, payload) do |_opcode, del_payload| + delivery_queue.push(del_payload) unless consumer_done + end + end - # Process deliveries from the queue. - loop do - del_payload = delivery_queue.pop - break if del_payload.nil? + def check_consume_response(response) + opcode, resp_payload = response + raise_from_error_frame(resp_payload) if opcode == FIBP::Opcodes::ERROR + end - process_delivery(del_payload, &block) - end - rescue NotLeaderError => e - raise if redirected || e.leader_addr.nil? + def consume_delivery_loop(delivery_queue, &block) + loop do + del_payload = delivery_queue.pop + break if del_payload.nil? - reconnect_to(e.leader_addr) - return consume_with_redirect(queue: queue, redirected: true, &block) - rescue LocalJumpError - nil # Consumer break + process_delivery(del_payload, &block) end - ensure - consumer_done = true - @conn&.cancel_consume(rid) if rid end def process_delivery(payload, &block) @@ -541,30 +545,29 @@ def reconnect_to(addr) @conn = build_connection(addr) end + ERROR_CODE_TO_CLASS = { + FIBP::ErrorCodes::QUEUE_NOT_FOUND => QueueNotFoundError, + FIBP::ErrorCodes::MESSAGE_NOT_FOUND => MessageNotFoundError, + FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS => QueueAlreadyExistsError, + FIBP::ErrorCodes::UNAUTHORIZED => AuthenticationError, + FIBP::ErrorCodes::FORBIDDEN => ForbiddenError, + FIBP::ErrorCodes::API_KEY_NOT_FOUND => ApiKeyNotFoundError + }.freeze + def raise_from_error_frame(resp) reader = FIBP::Codec::Reader.new(resp) code = reader.read_u8 message = reader.read_string metadata = reader.remaining.positive? ? reader.read_map : {} - case code - when FIBP::ErrorCodes::QUEUE_NOT_FOUND - raise QueueNotFoundError, message - when FIBP::ErrorCodes::MESSAGE_NOT_FOUND - raise MessageNotFoundError, message - when FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS - raise QueueAlreadyExistsError, message - when FIBP::ErrorCodes::UNAUTHORIZED - raise AuthenticationError, message - when FIBP::ErrorCodes::FORBIDDEN - raise ForbiddenError, message - when FIBP::ErrorCodes::NOT_LEADER + if code == FIBP::ErrorCodes::NOT_LEADER raise NotLeaderError.new(message, leader_addr: metadata['leader_addr']) - when FIBP::ErrorCodes::API_KEY_NOT_FOUND - raise ApiKeyNotFoundError, message - else - raise RPCError.new(code, message) end + + klass = ERROR_CODE_TO_CLASS[code] + raise klass, message if klass + + raise RPCError.new(code, message) end def raise_for_error_code(code, context) diff --git a/lib/fila/fibp/codec.rb b/lib/fila/fibp/codec.rb index 4e0fcd3..c5f87df 100644 --- a/lib/fila/fibp/codec.rb +++ b/lib/fila/fibp/codec.rb @@ -94,11 +94,11 @@ def remaining @data.bytesize - @pos end - def read_raw(n) - raise ProtocolError, "unexpected end of frame (need #{n}, have #{remaining})" if remaining < n + def read_raw(size) + raise ProtocolError, "unexpected end of frame (need #{size}, have #{remaining})" if remaining < size - slice = @data.byteslice(@pos, n) - @pos += n + slice = @data.byteslice(@pos, size) + @pos += size slice end @@ -129,7 +129,7 @@ def read_f64 read_raw(8).unpack1('G') end - def read_bool + def read_bool # rubocop:disable Naming/PredicateMethod read_u8 != 0 end diff --git a/lib/fila/fibp/connection.rb b/lib/fila/fibp/connection.rb index 2b28d58..fe7d0b4 100644 --- a/lib/fila/fibp/connection.rb +++ b/lib/fila/fibp/connection.rb @@ -9,7 +9,7 @@ module FIBP # TCP connection with TLS support, FIBP framing, handshake, and # request/response correlation. Supports both synchronous # request/response and asynchronous delivery streaming. - class Connection # rubocop:disable Metrics/ClassLength + class Connection PROTOCOL_VERSION = 1 DEFAULT_MAX_FRAME_SIZE = 16 * 1024 * 1024 # 16 MiB @@ -22,7 +22,7 @@ class Connection # rubocop:disable Metrics/ClassLength # @param client_cert [String, nil] PEM client certificate (mTLS) # @param client_key [String, nil] PEM client key (mTLS) # @param api_key [String, nil] API key for handshake auth - def initialize(host:, port:, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) + def initialize(host:, port:, tls: false, ca_cert: nil, client_cert: nil, client_key: nil, api_key: nil) # rubocop:disable Metrics/ParameterLists @host = host @port = port @api_key = api_key @@ -235,44 +235,48 @@ def read_frame [opcode, payload] end - def read_frame_with_id + def read_frame_with_id # rubocop:disable Metrics/AbcSize @read_monitor.synchronize do loop do - length_bytes = read_exact(4) - return [nil, nil, nil] if length_bytes.nil? + opcode, flags, request_id, payload = read_raw_frame + return [nil, nil, nil] if opcode.nil? - frame_length = length_bytes.unpack1('N') - body = read_exact(frame_length) - return [nil, nil, nil] if body.nil? + next buffer_continuation(request_id, opcode, payload) if flags.anybits?(FLAG_CONTINUATION) - opcode = body.getbyte(0) - flags = body.getbyte(1) - request_id = body.byteslice(2, 4).unpack1('N') - payload = body.byteslice(6..) + opcode, payload = reassemble_continuation(request_id, opcode, payload) + return [opcode, payload, request_id] + end + end + end + + def read_raw_frame + length_bytes = read_exact(4) + return [nil, nil, nil, nil] if length_bytes.nil? - continuation = (flags & FLAG_CONTINUATION) != 0 + body = read_exact(length_bytes.unpack1('N')) + return [nil, nil, nil, nil] if body.nil? - if continuation - @continuation_buffers[request_id] ||= { opcode: opcode, data: +''.b } - @continuation_buffers[request_id][:data] << payload - next - end + [body.getbyte(0), body.getbyte(1), body.byteslice(2, 4).unpack1('N'), body.byteslice(6..)] + end - if @continuation_buffers.key?(request_id) - buf = @continuation_buffers.delete(request_id) - payload = buf[:data] + payload - opcode = buf[:opcode] - end + def buffer_continuation(request_id, opcode, payload) + @continuation_buffers[request_id] ||= { opcode: opcode, data: +''.b } + @continuation_buffers[request_id][:data] << payload + end - return [opcode, payload, request_id] - end + def reassemble_continuation(request_id, opcode, payload) + if @continuation_buffers.key?(request_id) + buf = @continuation_buffers.delete(request_id) + [buf[:opcode], buf[:data] + payload] + else + [opcode, payload] end end - def read_exact(n) + def read_exact(size) buf = +''.b - while buf.bytesize < n - chunk = @socket.read(n - buf.bytesize) + while buf.bytesize < size + chunk = @socket.read(size - buf.bytesize) return nil if chunk.nil? || chunk.empty? buf << chunk diff --git a/test/test_helper.rb b/test/test_helper.rb index 99d0ab5..62f2da1 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -3,6 +3,7 @@ require 'minitest/autorun' require 'tmpdir' require 'socket' +require 'timeout' $LOAD_PATH.unshift File.expand_path('../lib', __dir__) require 'fila' @@ -75,11 +76,11 @@ def self.start(tls_config: nil, bootstrap_apikey: nil) ready = false while Time.now < deadline begin - try_list_queues(addr, conn_opts) + Timeout.timeout(3) { try_list_queues(addr, conn_opts) } ready = true break rescue StandardError - sleep 0.05 + sleep 0.1 end end @@ -112,14 +113,18 @@ def self.stop(server) end def self.create_queue(server, name) + client = nil client = Fila::Client.new(server[:addr], batch_mode: :disabled, **server[:conn_opts]) client.create_queue(name: name) - client.close + ensure + client&.close end def self.try_list_queues(addr, conn_opts) + client = nil client = Fila::Client.new(addr, batch_mode: :disabled, **conn_opts) client.list_queues - client.close + ensure + client&.close end end From 4a3e0cfa5aeb57d528dd2a448360868747209c6d Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:08:44 -0300 Subject: [PATCH 15/19] fix: remaining rubocop offenses (line length, constant scoping, modifier if) --- lib/fila/batcher.rb | 6 +++++- lib/fila/client.rb | 36 ++++++++++++++++++------------------ lib/fila/fibp/connection.rb | 2 +- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 5221125..7c60316 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -145,7 +145,11 @@ def dispatch_results(items, resp) reader = FIBP::Codec::Reader.new(resp) count = reader.read_u32 items.each_with_index do |item, idx| - outcome = idx < count ? result_to_outcome(reader.read_u8, reader.read_string) : Fila::Error.new('no result from server') + outcome = if idx < count + result_to_outcome(reader.read_u8, reader.read_string) + else + Fila::Error.new('no result from server') + end item.result_queue.push(outcome) end end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 136994f..6994884 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -383,6 +383,17 @@ def get_acl(key_id:) { key_id: key_id_resp, is_superadmin: is_superadmin, permissions: permissions } end + ERROR_CODE_TO_CLASS = { + FIBP::ErrorCodes::QUEUE_NOT_FOUND => QueueNotFoundError, + FIBP::ErrorCodes::MESSAGE_NOT_FOUND => MessageNotFoundError, + FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS => QueueAlreadyExistsError, + FIBP::ErrorCodes::UNAUTHORIZED => AuthenticationError, + FIBP::ErrorCodes::FORBIDDEN => ForbiddenError, + FIBP::ErrorCodes::API_KEY_NOT_FOUND => ApiKeyNotFoundError + }.freeze + + private_constant :ERROR_CODE_TO_CLASS + private def validate_batch_mode(mode) @@ -468,13 +479,13 @@ def decode_enqueue_results(resp) end end - def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics + def consume_with_redirect(queue:, redirected:, &block) payload = FIBP::Codec.encode_string(queue) delivery_queue = Queue.new - consumer_done = false + done = [false] # mutable container for closure capture rid = nil - rid, response = subscribe_to_queue(payload, delivery_queue, consumer_done) + rid, response = subscribe_to_queue(payload, delivery_queue, done) check_consume_response(response) consume_delivery_loop(delivery_queue, &block) rescue NotLeaderError => e @@ -485,13 +496,13 @@ def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics rescue LocalJumpError nil # Consumer break ensure - consumer_done = true + done[0] = true @conn&.cancel_consume(rid) if rid end - def subscribe_to_queue(payload, delivery_queue, consumer_done) + def subscribe_to_queue(payload, delivery_queue, done) @conn.subscribe(FIBP::Opcodes::CONSUME, payload) do |_opcode, del_payload| - delivery_queue.push(del_payload) unless consumer_done + delivery_queue.push(del_payload) unless done[0] end end @@ -545,24 +556,13 @@ def reconnect_to(addr) @conn = build_connection(addr) end - ERROR_CODE_TO_CLASS = { - FIBP::ErrorCodes::QUEUE_NOT_FOUND => QueueNotFoundError, - FIBP::ErrorCodes::MESSAGE_NOT_FOUND => MessageNotFoundError, - FIBP::ErrorCodes::QUEUE_ALREADY_EXISTS => QueueAlreadyExistsError, - FIBP::ErrorCodes::UNAUTHORIZED => AuthenticationError, - FIBP::ErrorCodes::FORBIDDEN => ForbiddenError, - FIBP::ErrorCodes::API_KEY_NOT_FOUND => ApiKeyNotFoundError - }.freeze - def raise_from_error_frame(resp) reader = FIBP::Codec::Reader.new(resp) code = reader.read_u8 message = reader.read_string metadata = reader.remaining.positive? ? reader.read_map : {} - if code == FIBP::ErrorCodes::NOT_LEADER - raise NotLeaderError.new(message, leader_addr: metadata['leader_addr']) - end + raise NotLeaderError.new(message, leader_addr: metadata['leader_addr']) if code == FIBP::ErrorCodes::NOT_LEADER klass = ERROR_CODE_TO_CLASS[code] raise klass, message if klass diff --git a/lib/fila/fibp/connection.rb b/lib/fila/fibp/connection.rb index fe7d0b4..625b43f 100644 --- a/lib/fila/fibp/connection.rb +++ b/lib/fila/fibp/connection.rb @@ -235,7 +235,7 @@ def read_frame [opcode, payload] end - def read_frame_with_id # rubocop:disable Metrics/AbcSize + def read_frame_with_id @read_monitor.synchronize do loop do opcode, flags, request_id, payload = read_raw_frame From fc57ce6e631c8c74dad15c96d64f7ec2b76f2828 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:13:55 -0300 Subject: [PATCH 16/19] fix: add last error detail to server startup failure message --- test/test_helper.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 62f2da1..635c496 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -74,12 +74,14 @@ def self.start(tls_config: nil, bootstrap_apikey: nil) # Wait for server ready. deadline = Time.now + 10 ready = false + last_error = nil while Time.now < deadline begin Timeout.timeout(3) { try_list_queues(addr, conn_opts) } ready = true break - rescue StandardError + rescue StandardError => e + last_error = e sleep 0.1 end end @@ -93,7 +95,8 @@ def self.start(tls_config: nil, bootstrap_apikey: nil) '' end FileUtils.rm_rf(data_dir) - raise "fila-server failed to start within 10s on #{addr}\nConfig:\n#{toml}\nStderr:\n#{stderr_output}" + raise "fila-server failed to start within 10s on #{addr}\nConfig:\n#{toml}\n" \ + "Stderr:\n#{stderr_output}\nLast error: #{last_error&.class}: #{last_error&.message}" end { From 301ff1c66378784f76c5bebd3e4f2444a3cffccd Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:21:38 -0300 Subject: [PATCH 17/19] fix: address 10 cubic review findings --- lib/fila/batcher.rb | 8 +++-- lib/fila/client.rb | 68 +++++++++++++++++++++++++++++++++++-- lib/fila/fibp/connection.rb | 1 + test/test_tls_auth.rb | 6 ++++ 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/lib/fila/batcher.rb b/lib/fila/batcher.rb index 7c60316..1616013 100644 --- a/lib/fila/batcher.rb +++ b/lib/fila/batcher.rb @@ -10,6 +10,8 @@ class Batcher # An item queued for batching, pairing a message hash with its result slot. BatchItem = Struct.new(:message, :result_queue, keyword_init: true) + attr_writer :conn + # @param conn [Fila::FIBP::Connection] FIBP connection # @param mode [Symbol] :auto or :linger # @param max_batch_size [Integer] cap on batch size (auto mode) @@ -157,9 +159,9 @@ def dispatch_results(items, resp) def encode_enqueue_batch(messages) buf = FIBP::Codec.encode_u32(messages.size) messages.each do |m| - buf += FIBP::Codec.encode_string(m[:queue]) + - FIBP::Codec.encode_map(m[:headers] || {}) + - FIBP::Codec.encode_bytes(m[:payload]) + buf << FIBP::Codec.encode_string(m[:queue]) + buf << FIBP::Codec.encode_map(m[:headers] || {}) + buf << FIBP::Codec.encode_bytes(m[:payload]) end buf end diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 6994884..16cf648 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -458,9 +458,9 @@ def enqueue_single(msg) def encode_enqueue_batch(messages) payload = FIBP::Codec.encode_u32(messages.size) messages.each do |m| - payload += FIBP::Codec.encode_string(m[:queue]) + - FIBP::Codec.encode_map(m[:headers] || {}) + - FIBP::Codec.encode_bytes(m[:payload]) + payload << FIBP::Codec.encode_string(m[:queue]) + payload << FIBP::Codec.encode_map(m[:headers] || {}) + payload << FIBP::Codec.encode_bytes(m[:payload]) end payload end @@ -491,6 +491,9 @@ def consume_with_redirect(queue:, redirected:, &block) rescue NotLeaderError => e raise if redirected || e.leader_addr.nil? + done[0] = true + @conn&.cancel_consume(rid) if rid + rid = nil reconnect_to(e.leader_addr) consume_with_redirect(queue: queue, redirected: true, &block) rescue LocalJumpError @@ -554,6 +557,7 @@ def decode_delivery_message(reader) def reconnect_to(addr) @conn&.close @conn = build_connection(addr) + @batcher.conn = @conn if @batcher end def raise_from_error_frame(resp) @@ -589,6 +593,64 @@ def raise_for_error_code(code, context) end end + def decode_stats_result(resp) + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'get stats') unless code == FIBP::ErrorCodes::OK + + result = { + depth: reader.read_u64, + in_flight: reader.read_u64, + active_fairness_keys: reader.read_u64, + active_consumers: reader.read_u32, + quantum: reader.read_u32, + leader_node_id: reader.read_u64, + replication_count: reader.read_u32 + } + + per_key_count = reader.read_u16 + result[:per_key_stats] = Array.new(per_key_count) do + { + key: reader.read_string, + pending_count: reader.read_u64, + current_deficit: reader.read_i64, + weight: reader.read_u32 + } + end + + per_throttle_count = reader.read_u16 + result[:per_throttle_stats] = Array.new(per_throttle_count) do + { + key: reader.read_string, + tokens: reader.read_f64, + rate_per_second: reader.read_f64, + burst: reader.read_f64 + } + end + + result + end + + def decode_list_queues_result(resp) + reader = FIBP::Codec::Reader.new(resp) + code = reader.read_u8 + raise_for_error_code(code, 'list queues') unless code == FIBP::ErrorCodes::OK + + cluster_node_count = reader.read_u32 + queue_count = reader.read_u16 + queues = Array.new(queue_count) do + { + name: reader.read_string, + depth: reader.read_u64, + in_flight: reader.read_u64, + active_consumers: reader.read_u32, + leader_node_id: reader.read_u64 + } + end + + { cluster_node_count: cluster_node_count, queues: queues } + end + def error_name(code) case code when FIBP::ErrorCodes::QUEUE_NOT_FOUND then 'queue not found' diff --git a/lib/fila/fibp/connection.rb b/lib/fila/fibp/connection.rb index 625b43f..97dfcbe 100644 --- a/lib/fila/fibp/connection.rb +++ b/lib/fila/fibp/connection.rb @@ -137,6 +137,7 @@ def wrap_tls(tcp) store = OpenSSL::X509::Store.new store.add_cert(OpenSSL::X509::Certificate.new(@ca_cert)) ctx.cert_store = store + ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER else ctx.set_params # uses system trust store end diff --git a/test/test_tls_auth.rb b/test/test_tls_auth.rb index 6132977..17be157 100644 --- a/test/test_tls_auth.rb +++ b/test/test_tls_auth.rb @@ -94,11 +94,14 @@ def test_enqueue_with_api_key def test_enqueue_without_api_key_rejected TestServerHelper.create_queue(@server, 'auth-reject-queue') + client_no_key = nil err = assert_raises(Fila::AuthenticationError) do client_no_key = Fila::Client.new(@server[:addr]) client_no_key.enqueue(queue: 'auth-reject-queue', payload: 'should fail') end assert_match(/unauthorized/i, err.message) + ensure + client_no_key&.close end def test_consume_with_api_key @@ -241,6 +244,7 @@ def test_enqueue_with_tls_and_api_key def test_no_api_key_over_tls_rejected TestServerHelper.create_queue(@server, 'tls-auth-reject-queue') + client_no_key = nil err = assert_raises(Fila::AuthenticationError) do client_no_key = Fila::Client.new( @server[:addr], @@ -249,6 +253,8 @@ def test_no_api_key_over_tls_rejected client_no_key.enqueue(queue: 'tls-auth-reject-queue', payload: 'should fail') end assert_match(/unauthorized/i, err.message) + ensure + client_no_key&.close end end From eb4d575ce41b25df9eba37fa66fb40a6e0b4efb6 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:24:32 -0300 Subject: [PATCH 18/19] fix: correct opcode values to match server, split decode_stats_result, fix auth test --- lib/fila/client.rb | 48 ++++++++++++++++++---------------------- lib/fila/fibp/opcodes.rb | 16 +++++++------- test/test_tls_auth.rb | 4 ++-- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 16cf648..359bef0 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -479,7 +479,7 @@ def decode_enqueue_results(resp) end end - def consume_with_redirect(queue:, redirected:, &block) + def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/CyclomaticComplexity payload = FIBP::Codec.encode_string(queue) delivery_queue = Queue.new done = [false] # mutable container for closure capture @@ -598,37 +598,33 @@ def decode_stats_result(resp) code = reader.read_u8 raise_for_error_code(code, 'get stats') unless code == FIBP::ErrorCodes::OK - result = { - depth: reader.read_u64, - in_flight: reader.read_u64, - active_fairness_keys: reader.read_u64, - active_consumers: reader.read_u32, - quantum: reader.read_u32, - leader_node_id: reader.read_u64, + result = read_stats_base(reader) + result[:per_key_stats] = read_per_key_stats(reader) + result[:per_throttle_stats] = read_per_throttle_stats(reader) + result + end + + def read_stats_base(reader) + { + depth: reader.read_u64, in_flight: reader.read_u64, + active_fairness_keys: reader.read_u64, active_consumers: reader.read_u32, + quantum: reader.read_u32, leader_node_id: reader.read_u64, replication_count: reader.read_u32 } + end - per_key_count = reader.read_u16 - result[:per_key_stats] = Array.new(per_key_count) do - { - key: reader.read_string, - pending_count: reader.read_u64, - current_deficit: reader.read_i64, - weight: reader.read_u32 - } + def read_per_key_stats(reader) + Array.new(reader.read_u16) do + { key: reader.read_string, pending_count: reader.read_u64, + current_deficit: reader.read_i64, weight: reader.read_u32 } end + end - per_throttle_count = reader.read_u16 - result[:per_throttle_stats] = Array.new(per_throttle_count) do - { - key: reader.read_string, - tokens: reader.read_f64, - rate_per_second: reader.read_f64, - burst: reader.read_f64 - } + def read_per_throttle_stats(reader) + Array.new(reader.read_u16) do + { key: reader.read_string, tokens: reader.read_f64, + rate_per_second: reader.read_f64, burst: reader.read_f64 } end - - result end def decode_list_queues_result(resp) diff --git a/lib/fila/fibp/opcodes.rb b/lib/fila/fibp/opcodes.rb index fd408be..c426144 100644 --- a/lib/fila/fibp/opcodes.rb +++ b/lib/fila/fibp/opcodes.rb @@ -2,7 +2,7 @@ module Fila module FIBP - # Protocol opcodes for the Fila binary protocol. + # Protocol opcodes matching the server's fila-fibp crate. module Opcodes # Control opcodes (0x00-0x0F) HANDSHAKE = 0x01 @@ -15,13 +15,13 @@ module Opcodes ENQUEUE = 0x10 ENQUEUE_RESULT = 0x11 CONSUME = 0x12 - DELIVERY = 0x13 - CANCEL_CONSUME = 0x14 - ACK = 0x15 - ACK_RESULT = 0x16 - NACK = 0x17 - NACK_RESULT = 0x18 - CONSUME_OK = 0x19 + CONSUME_OK = 0x13 + DELIVERY = 0x14 + CANCEL_CONSUME = 0x15 + ACK = 0x16 + ACK_RESULT = 0x17 + NACK = 0x18 + NACK_RESULT = 0x19 # Error opcode ERROR = 0xFE diff --git a/test/test_tls_auth.rb b/test/test_tls_auth.rb index 17be157..ca61f59 100644 --- a/test/test_tls_auth.rb +++ b/test/test_tls_auth.rb @@ -99,7 +99,7 @@ def test_enqueue_without_api_key_rejected client_no_key = Fila::Client.new(@server[:addr]) client_no_key.enqueue(queue: 'auth-reject-queue', payload: 'should fail') end - assert_match(/unauthorized/i, err.message) + assert_match(/api key required|unauthorized/i, err.message) ensure client_no_key&.close end @@ -252,7 +252,7 @@ def test_no_api_key_over_tls_rejected ) client_no_key.enqueue(queue: 'tls-auth-reject-queue', payload: 'should fail') end - assert_match(/unauthorized/i, err.message) + assert_match(/api key required|unauthorized/i, err.message) ensure client_no_key&.close end From 2da74e78bba93176e76a737a06f6ab16af71a630 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Sat, 4 Apr 2026 10:37:23 -0300 Subject: [PATCH 19/19] fix: resolve cyclomatic complexity in consume_with_redirect extract handle_not_leader_redirect helper to bring cyclomatic complexity under threshold, removing rubocop:disable comment. use anonymous block forwarding per rubocop style rules. --- lib/fila/client.rb | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/fila/client.rb b/lib/fila/client.rb index 359bef0..3ebc9c8 100644 --- a/lib/fila/client.rb +++ b/lib/fila/client.rb @@ -479,7 +479,7 @@ def decode_enqueue_results(resp) end end - def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics/CyclomaticComplexity + def consume_with_redirect(queue:, redirected:, &) payload = FIBP::Codec.encode_string(queue) delivery_queue = Queue.new done = [false] # mutable container for closure capture @@ -487,15 +487,9 @@ def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics rid, response = subscribe_to_queue(payload, delivery_queue, done) check_consume_response(response) - consume_delivery_loop(delivery_queue, &block) + consume_delivery_loop(delivery_queue, &) rescue NotLeaderError => e - raise if redirected || e.leader_addr.nil? - - done[0] = true - @conn&.cancel_consume(rid) if rid - rid = nil - reconnect_to(e.leader_addr) - consume_with_redirect(queue: queue, redirected: true, &block) + rid = handle_not_leader_redirect(e, rid, done, redirected, queue, &) rescue LocalJumpError nil # Consumer break ensure @@ -503,6 +497,16 @@ def consume_with_redirect(queue:, redirected:, &block) # rubocop:disable Metrics @conn&.cancel_consume(rid) if rid end + def handle_not_leader_redirect(error, rid, done, redirected, queue, &) + raise error if redirected || error.leader_addr.nil? + + done[0] = true + @conn&.cancel_consume(rid) if rid + reconnect_to(error.leader_addr) + consume_with_redirect(queue: queue, redirected: true, &) + nil + end + def subscribe_to_queue(payload, delivery_queue, done) @conn.subscribe(FIBP::Opcodes::CONSUME, payload) do |_opcode, del_payload| delivery_queue.push(del_payload) unless done[0]