From 9249b7ccf5cfcec4bf1526778f412baebcceb064 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 27 Feb 2026 19:45:46 +0000 Subject: [PATCH 01/28] create LLM request source Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- .bazelrc | 1 + source/request_source/BUILD | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/.bazelrc b/.bazelrc index d967b74ef..eeda635ce 100644 --- a/.bazelrc +++ b/.bazelrc @@ -511,3 +511,4 @@ try-import %workspace%/clang.bazelrc try-import %workspace%/user.bazelrc try-import %workspace%/local_tsan.bazelrc +common --credential_helper=us-python.pkg.dev=/usr/local/google/home/graysonlafleur/gar_cred_helper.sh diff --git a/source/request_source/BUILD b/source/request_source/BUILD index 9fdbf5151..e0b5ffc6c 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -4,6 +4,8 @@ load( "envoy_package", ) +load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") + licenses(["notice"]) # Apache 2 envoy_package() @@ -31,3 +33,44 @@ envoy_cc_library( "@envoy//source/exe:platform_impl_lib", ], ) + +envoy_cc_library( + name = "llm_request_source_plugin_impl", + srcs = [ + "llm_request_source_plugin_impl.cc", + ], + hdrs = [ + "llm_request_source_plugin_impl.h", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + "//include/nighthawk/request_source:request_source_plugin_config_factory_lib", + "//source/common:nighthawk_common_lib", + "//source/common:request_impl_lib", + "//source/common:request_source_impl_lib", + "@envoy//source/common/common:thread_lib_with_external_headers", + "@envoy//source/common/protobuf:message_validator_lib_with_external_headers", + "@envoy//source/common/protobuf:protobuf_with_external_headers", + "@envoy//source/common/protobuf:utility_lib_with_external_headers", + "@envoy//source/exe:platform_header_lib_with_external_headers", + "@envoy//source/exe:platform_impl_lib", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/random", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", + ], +) + +api_cc_py_proto_library( + name = "llm_request_source_plugin_proto", + srcs = [ + "llm_request_source_plugin.proto", + ], + deps = [ + "//api/client:base", + ], +) From 31517e1a4d8b2112f6531920b74028df9d678d35 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:47:18 +0000 Subject: [PATCH 02/28] add untracked files Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- .../llm_request_source_plugin.proto | 22 +++ .../llm_request_source_plugin_impl.cc | 131 ++++++++++++++++++ .../llm_request_source_plugin_impl.h | 90 ++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 source/request_source/llm_request_source_plugin.proto create mode 100644 source/request_source/llm_request_source_plugin_impl.cc create mode 100644 source/request_source/llm_request_source_plugin_impl.h diff --git a/source/request_source/llm_request_source_plugin.proto b/source/request_source/llm_request_source_plugin.proto new file mode 100644 index 000000000..0c21e5b19 --- /dev/null +++ b/source/request_source/llm_request_source_plugin.proto @@ -0,0 +1,22 @@ +edition = "2024"; + +package nighthawk; + +import "api/client/options.proto"; + +option features.utf8_validation = NONE; + +// Config for `LlmRequestSourcePlugin`. +message LlmRequestSourcePluginConfig { + // Model to use for the request. This field is required. + string model_name = 1; + + // Number of tokens to generate in the request. + int32 req_token_count = 2 [default = 10]; + + // Maximum number of tokens to return in the response. + int32 resp_max_tokens = 3 [default = 10]; + + // The options_list will be used to apply headers to the request. + nighthawk.client.RequestOptionsList options_list = 4; +} diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc new file mode 100644 index 000000000..faeede6aa --- /dev/null +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -0,0 +1,131 @@ +#include "source/request_source/llm_request_source_plugin_impl.h" + +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/random/random.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" + +#include "source/request_source/llm_request_source_plugin.pb.h" + +#include "external/envoy/source/common/protobuf/protobuf.h" +#include "external/envoy/source/common/protobuf/utility.h" +#include "external/envoy/src/api/envoy/config/core/v3/base.pb.h" +#include "external/envoy/src/api/envoy/config/core/v3/extension.pb.h" +#include "external/envoy/src/envoy/api/api.h" +#include "external/envoy/src/envoy/http/header_map.h" +#include "external/envoy/src/envoy/registry/registry.h" +#include "external/envoy/src/source/common/http/header_map_impl.h" +#include "external/envoy/src/source/common/protobuf/utility.h" + +#include "api/client/options.pb.h" +#include "api/request_source/request_source_plugin.pb.h" +#include "source/common/request.h" +#include "source/common/request_source.h" +#include "nighthawk/request_source/request_source_plugin_config_factory.h" +#include "source/common/request_impl.h" + +namespace nighthawk { +namespace { + +absl::Status ValidateConfig(const LlmRequestSourcePluginConfig& config) { + if (config.model_name().empty()) { + return absl::InvalidArgumentError("Model name is required."); + } + + return absl::OkStatus(); +} + +std::string GenerateRandomPrompt(int num_tokens) { + static const char charset[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + // The final string to be built. + std::string result_string; + + // Random number generator for selecting characters from the charset. + absl::BitGen bitgen; + + for (int i = 0; i < num_tokens; ++i) { + // Append a random character from the charset. + result_string += charset[absl::Uniform(bitgen, 0, 62)]; + + // Add a space between tokens. This is a naive way to calculate the number + // of tokens in the string as generally spaces delineate tokens. + if (i < num_tokens - 1) { + result_string += ' '; + } + } + + return result_string; +} + +} // namespace + +Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { + return [this]() -> std::unique_ptr { + Envoy::Http::RequestHeaderMapPtr headers = + Envoy::Http::RequestHeaderMapImpl::create(); + Envoy::Http::HeaderMapImpl::copyFrom(*headers, *header_); + + // TODO(b/436267941): Add support for multiple messages. + std::string body = absl::StrFormat(R"json( + { + "model": "%s", + "max_tokens": %d, + "messages": [ + { + "role": "user", + "content": "%s" + } + ] + } + )json", + model_name_, resp_max_tokens_, + GenerateRandomPrompt(req_token_count_)); + + headers->setMethod(envoy::config::core::v3::RequestMethod_Name( + envoy::config::core::v3::RequestMethod::POST)); + headers->setContentType("application/json"); + headers->setContentLength(body.size()); + + auto path_key = Envoy::Http::LowerCaseString(":path"); + headers->setCopy(path_key, "/v1/completions"); + + return std::make_unique(std::move(headers), body); + }; +} + +Nighthawk::RequestSourcePtr LlmRequestSourcePluginFactory::createRequestSourcePlugin( + const Envoy::Protobuf::Message& message, Envoy::Api::Api&, + Envoy::Http::RequestHeaderMapPtr header) { + const auto* any = Envoy::Protobuf::DynamicCastToGenerated(&message); + LlmRequestSourcePluginConfig llm_config; + THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(any, llm_config)); + THROW_IF_NOT_OK(ValidateConfig(llm_config)); + + for (const nighthawk::client::RequestOptions& request_option : + llm_config.options_list().options()) { + for (const envoy::config::core::v3::HeaderValueOption& option_header : + request_option.request_headers()) { + auto lower_case_key = + Envoy::Http::LowerCaseString(option_header.header().key()); + header->setCopy(lower_case_key, option_header.header().value()); + } + } + + return std::make_unique( + std::string(llm_config.model_name()), llm_config.req_token_count(), + llm_config.resp_max_tokens(), std::move(header)); +}; + +REGISTER_FACTORY(LlmRequestSourcePluginFactory, + Nighthawk::RequestSourcePluginConfigFactory); + +} // namespace nighthawk diff --git a/source/request_source/llm_request_source_plugin_impl.h b/source/request_source/llm_request_source_plugin_impl.h new file mode 100644 index 000000000..08159efaa --- /dev/null +++ b/source/request_source/llm_request_source_plugin_impl.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include + +#include "source/request_source/llm_request_source_plugin.pb.h" + +#include "absl/log/log.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" + +#include "external/envoy/src/api/envoy/config/core/v3/extension.pb.h" +#include "external/envoy/src/envoy/api/api.h" +#include "external/envoy/src/envoy/http/header_map.h" +#include "external/envoy/src/source/common/common/logger.h" +#include "external/envoy/src/source/common/protobuf/protobuf.h" +#include "external/envoy/source/common/common/logger.h" + +#include "api/client/options.pb.h" +#include "nighthawk/common/request_source.h" +#include "nighthawk/request_source/request_source_plugin_config_factory.h" + +namespace nighthawk { + +constexpr inline absl::string_view kLlmRequestSourcePluginName = + "nighthawk.request_source.llm"; + +// A Nighthawk RequestSource that generates requests for LLM inference. +// +// The request source generates requests with the following characteristics: +// - The request body is a JSON object with the following fields: +// - model: The name of the model to use for inference. +// - max_tokens: The maximum number of tokens to return in the response. +// - messages: A list with a single JSON object containing the following +// fields: +// - role: "user" +// - content: A string containing `req_token_count` randomly generated +// tokens. +// - The request headers are copied from the provided header map with the +// following modifications: +// - Method: POST +// - Content-Type: application/json +// - Content-Length: The length of the request body. +// - :path: /v1/completions +class LlmRequestSourcePlugin + : public Nighthawk::RequestSource, + public Envoy::Logger::Loggable { + public: + explicit LlmRequestSourcePlugin(std::string model_name, int req_token_count, + int resp_max_tokens, + Envoy::Http::RequestHeaderMapPtr header) + : model_name_(model_name), + req_token_count_(req_token_count), + resp_max_tokens_(resp_max_tokens), + header_(std::move(header)) {}; + + Nighthawk::RequestGenerator get() override; + void initOnThread() override {}; + void destroyOnThread() override {}; + + private: + // Model to use for the request. + std::string model_name_; + // Number of tokens to generate in the request. + int req_token_count_; + // Maximum number of tokens from the model to return in the response. + int resp_max_tokens_; + // The options_list will be used to apply headers to the request. + std::unique_ptr options_list_; + Envoy::Http::RequestHeaderMapPtr header_; +}; + +class LlmRequestSourcePluginFactory + : public virtual Nighthawk::RequestSourcePluginConfigFactory { + public: + std::string name() const override { + return std::string(kLlmRequestSourcePluginName); + } + + Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + Nighthawk::RequestSourcePtr createRequestSourcePlugin( + const Envoy::Protobuf::Message&, Envoy::Api::Api&, + Envoy::Http::RequestHeaderMapPtr header) override; +}; + +} // namespace nighthawk From 00f04462c6cc0b5edbfeaa961aaa9dd24836d624 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:09:19 +0000 Subject: [PATCH 03/28] fix build issues, add to general build rule, and add documentation Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- LLM_LOAD_GENERATION.md | 53 +++++++++++++++++++ source/BUILD | 0 source/client/BUILD | 2 + source/request_source/BUILD | 21 ++++---- .../llm_request_source_plugin.proto | 8 ++- .../llm_request_source_plugin_impl.cc | 22 ++++---- .../llm_request_source_plugin_impl.h | 9 ++-- 7 files changed, 86 insertions(+), 29 deletions(-) create mode 100644 LLM_LOAD_GENERATION.md create mode 100644 source/BUILD diff --git a/LLM_LOAD_GENERATION.md b/LLM_LOAD_GENERATION.md new file mode 100644 index 000000000..739856858 --- /dev/null +++ b/LLM_LOAD_GENERATION.md @@ -0,0 +1,53 @@ +# Load Testing with LLM-formatted Requests + +## Overview + +If you would like to perform a load test against an LLM backend, using +LLM-formatted requests, there is an LLM Request Source plugin that can +emulate that workload. These request bodies looks like the following: + +``` +{ + "model": "Qwen/Qwen2.5-1.5B-Instruct", + "max_tokens": 10, + "messages": [ + { + "role": "user", + "content": "L Q 5 i x D q v p X" + } + ] +} +``` + +This is generated based on input you provide. The 4 inputs are: + +1. Model Name (required) + - Name of the LLM model the requests are being sent to +2. Request Token Count (default 0) + - Number of "tokens" generated for the request +3. Response Max Token Count (default 0) + - Maximum number of tokens for the model to respond with +4. [Request Options List](https://github.com/envoyproxy/nighthawk/blob/09d64d769972513989a95766a98e28f5d6bb05c2/api/client/options.proto#L32) (optional) + - This allows you to add headers and choose request method of the requests + - Header 'Content-Type: application/json' added by default + - Ignore the "request_body_size" and "json_body" in this field + - Must use ":authority" header instead of ":host" + +It's also important to note that all requests are routed to the path "/v1/completions". +There is not currently a way to override this through the CLI. If you need to use +a different path, you will need to edit it in source/request_source/llm_request_source_plugin_impl.cc. + +The config for running with this request source is passed into the "--request-source-plugin-config" flag. +Here is an example of how that flag might look for running a load test with this LLM Request Source plugin: +``` +--request-source-plugin-config "{name:\"nighthawk.request_source.llm\",typed_config:{\"@type\":\"type.googleapis.com/nighthawk.LlmRequestSourcePluginConfig\", model_name: \"Qwen/Qwen2.5-1.5B-Instruct\", req_token_count: 10, resp_max_tokens: 10, options_list:{options:[{request_headers:[{header:{key:\":authority\",value:\"team1.example.com\"}}]}]}}}" +``` + +Please be conscious about using escape characters in your string. + +## Tokenizer + +We do not use a real tokenizer for generating tokens in the requests. Instead, +we do a naive "tokenizer" where each "token" is just a random character in the +range of [A-Za-z0-9]. This means that the length of the requested message will +always be 2*req_token_count-1. diff --git a/source/BUILD b/source/BUILD new file mode 100644 index 000000000..e69de29bb diff --git a/source/client/BUILD b/source/client/BUILD index 35efabe9c..fc7314a1c 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -104,6 +104,8 @@ envoy_cc_library( "//source/common:nighthawk_service_client_impl", "//source/common:request_source_impl_lib", "//source/request_source:request_options_list_plugin_impl", + "//source/request_source:llm_request_source_plugin_cc_proto", + "//source/request_source:llm_request_source_plugin_impl", "//source/user_defined_output:user_defined_output_plugin_creator", "@envoy//envoy/config:xds_manager_interface", "@envoy//envoy/http:protocol_interface_with_external_headers", diff --git a/source/request_source/BUILD b/source/request_source/BUILD index e0b5ffc6c..6f2606eae 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -34,6 +34,17 @@ envoy_cc_library( ], ) +api_cc_py_proto_library( + name = "llm_request_source_plugin", + srcs = [ + "llm_request_source_plugin.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "//api/client:base", + ], +) + envoy_cc_library( name = "llm_request_source_plugin_impl", srcs = [ @@ -45,6 +56,7 @@ envoy_cc_library( repository = "@envoy", visibility = ["//visibility:public"], deps = [ + ":llm_request_source_plugin_cc_proto", "//include/nighthawk/request_source:request_source_plugin_config_factory_lib", "//source/common:nighthawk_common_lib", "//source/common:request_impl_lib", @@ -65,12 +77,3 @@ envoy_cc_library( ], ) -api_cc_py_proto_library( - name = "llm_request_source_plugin_proto", - srcs = [ - "llm_request_source_plugin.proto", - ], - deps = [ - "//api/client:base", - ], -) diff --git a/source/request_source/llm_request_source_plugin.proto b/source/request_source/llm_request_source_plugin.proto index 0c21e5b19..cc04fa8c6 100644 --- a/source/request_source/llm_request_source_plugin.proto +++ b/source/request_source/llm_request_source_plugin.proto @@ -1,21 +1,19 @@ -edition = "2024"; +syntax = "proto3"; package nighthawk; import "api/client/options.proto"; -option features.utf8_validation = NONE; - // Config for `LlmRequestSourcePlugin`. message LlmRequestSourcePluginConfig { // Model to use for the request. This field is required. string model_name = 1; // Number of tokens to generate in the request. - int32 req_token_count = 2 [default = 10]; + int32 req_token_count = 2; // Maximum number of tokens to return in the response. - int32 resp_max_tokens = 3 [default = 10]; + int32 resp_max_tokens = 3; // The options_list will be used to apply headers to the request. nighthawk.client.RequestOptionsList options_list = 4; diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index faeede6aa..a6746f830 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -14,18 +14,18 @@ #include "external/envoy/source/common/protobuf/protobuf.h" #include "external/envoy/source/common/protobuf/utility.h" -#include "external/envoy/src/api/envoy/config/core/v3/base.pb.h" -#include "external/envoy/src/api/envoy/config/core/v3/extension.pb.h" -#include "external/envoy/src/envoy/api/api.h" -#include "external/envoy/src/envoy/http/header_map.h" -#include "external/envoy/src/envoy/registry/registry.h" -#include "external/envoy/src/source/common/http/header_map_impl.h" -#include "external/envoy/src/source/common/protobuf/utility.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/core/v3/extension.pb.h" +#include "envoy/api/api.h" +#include "envoy/http/header_map.h" +#include "envoy/registry/registry.h" +#include "external/envoy/source/common/http/header_map_impl.h" +#include "external/envoy/source/common/protobuf/utility.h" #include "api/client/options.pb.h" #include "api/request_source/request_source_plugin.pb.h" -#include "source/common/request.h" -#include "source/common/request_source.h" +#include "nighthawk/common/request.h" +#include "nighthawk/common/request_source.h" #include "nighthawk/request_source/request_source_plugin_config_factory.h" #include "source/common/request_impl.h" @@ -98,6 +98,8 @@ Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { auto path_key = Envoy::Http::LowerCaseString(":path"); headers->setCopy(path_key, "/v1/completions"); + ENVOY_LOG(info, body); + return std::make_unique(std::move(headers), body); }; } @@ -107,7 +109,7 @@ Nighthawk::RequestSourcePtr LlmRequestSourcePluginFactory::createRequestSourcePl Envoy::Http::RequestHeaderMapPtr header) { const auto* any = Envoy::Protobuf::DynamicCastToGenerated(&message); LlmRequestSourcePluginConfig llm_config; - THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(any, llm_config)); + THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(*any, llm_config)); THROW_IF_NOT_OK(ValidateConfig(llm_config)); for (const nighthawk::client::RequestOptions& request_option : diff --git a/source/request_source/llm_request_source_plugin_impl.h b/source/request_source/llm_request_source_plugin_impl.h index 08159efaa..85d6ea3cb 100644 --- a/source/request_source/llm_request_source_plugin_impl.h +++ b/source/request_source/llm_request_source_plugin_impl.h @@ -10,12 +10,11 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "external/envoy/src/api/envoy/config/core/v3/extension.pb.h" -#include "external/envoy/src/envoy/api/api.h" -#include "external/envoy/src/envoy/http/header_map.h" -#include "external/envoy/src/source/common/common/logger.h" -#include "external/envoy/src/source/common/protobuf/protobuf.h" +#include "envoy/config/core/v3/extension.pb.h" +#include "envoy/api/api.h" +#include "envoy/http/header_map.h" #include "external/envoy/source/common/common/logger.h" +#include "external/envoy/source/common/protobuf/protobuf.h" #include "api/client/options.pb.h" #include "nighthawk/common/request_source.h" From aad5daddf78c2d529f0b82e0f2014a34ea9b30de Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:54:52 +0000 Subject: [PATCH 04/28] update documentation Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- LLM_LOAD_GENERATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LLM_LOAD_GENERATION.md b/LLM_LOAD_GENERATION.md index 739856858..4663dc420 100644 --- a/LLM_LOAD_GENERATION.md +++ b/LLM_LOAD_GENERATION.md @@ -49,5 +49,5 @@ Please be conscious about using escape characters in your string. We do not use a real tokenizer for generating tokens in the requests. Instead, we do a naive "tokenizer" where each "token" is just a random character in the -range of [A-Za-z0-9]. This means that the length of the requested message will -always be 2*req_token_count-1. +range of [A-Za-z0-9] with a space between each. This means that the length of +the requested message will always be 2*req_token_count-1. From 07ef012e934c795c9ca3dc51a9a26e9f2e7c2613 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:56:45 +0000 Subject: [PATCH 05/28] update documentation Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- LLM_LOAD_GENERATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLM_LOAD_GENERATION.md b/LLM_LOAD_GENERATION.md index 4663dc420..2919acd39 100644 --- a/LLM_LOAD_GENERATION.md +++ b/LLM_LOAD_GENERATION.md @@ -31,7 +31,7 @@ This is generated based on input you provide. The 4 inputs are: - This allows you to add headers and choose request method of the requests - Header 'Content-Type: application/json' added by default - Ignore the "request_body_size" and "json_body" in this field - - Must use ":authority" header instead of ":host" + - If a host name is required, use ":authority" header instead of ":host" It's also important to note that all requests are routed to the path "/v1/completions". There is not currently a way to override this through the CLI. If you need to use From 494be94eeeb3ab2ef744219d6d19c0cd3b3609b5 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:26:05 +0000 Subject: [PATCH 06/28] clang-format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- .../llm_request_source_plugin_impl.cc | 48 +++++++++---------- .../llm_request_source_plugin_impl.h | 42 +++++++--------- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index a6746f830..6bd2bd884 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -12,14 +12,13 @@ #include "source/request_source/llm_request_source_plugin.pb.h" -#include "external/envoy/source/common/protobuf/protobuf.h" -#include "external/envoy/source/common/protobuf/utility.h" +#include "envoy/api/api.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/extension.pb.h" -#include "envoy/api/api.h" #include "envoy/http/header_map.h" #include "envoy/registry/registry.h" #include "external/envoy/source/common/http/header_map_impl.h" +#include "external/envoy/source/common/protobuf/protobuf.h" #include "external/envoy/source/common/protobuf/utility.h" #include "api/client/options.pb.h" @@ -41,10 +40,9 @@ absl::Status ValidateConfig(const LlmRequestSourcePluginConfig& config) { } std::string GenerateRandomPrompt(int num_tokens) { - static const char charset[] = - "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; + static const char charset[] = "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; // The final string to be built. std::string result_string; @@ -66,16 +64,16 @@ std::string GenerateRandomPrompt(int num_tokens) { return result_string; } -} // namespace +} // namespace Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { return [this]() -> std::unique_ptr { - Envoy::Http::RequestHeaderMapPtr headers = - Envoy::Http::RequestHeaderMapImpl::create(); + Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); Envoy::Http::HeaderMapImpl::copyFrom(*headers, *header_); // TODO(b/436267941): Add support for multiple messages. - std::string body = absl::StrFormat(R"json( + std::string body = + absl::StrFormat(R"json( { "model": "%s", "max_tokens": %d, @@ -87,11 +85,10 @@ Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { ] } )json", - model_name_, resp_max_tokens_, - GenerateRandomPrompt(req_token_count_)); + model_name_, resp_max_tokens_, GenerateRandomPrompt(req_token_count_)); - headers->setMethod(envoy::config::core::v3::RequestMethod_Name( - envoy::config::core::v3::RequestMethod::POST)); + headers->setMethod( + envoy::config::core::v3::RequestMethod_Name(envoy::config::core::v3::RequestMethod::POST)); headers->setContentType("application/json"); headers->setContentLength(body.size()); @@ -104,9 +101,10 @@ Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { }; } -Nighthawk::RequestSourcePtr LlmRequestSourcePluginFactory::createRequestSourcePlugin( - const Envoy::Protobuf::Message& message, Envoy::Api::Api&, - Envoy::Http::RequestHeaderMapPtr header) { +Nighthawk::RequestSourcePtr +LlmRequestSourcePluginFactory::createRequestSourcePlugin(const Envoy::Protobuf::Message& message, + Envoy::Api::Api&, + Envoy::Http::RequestHeaderMapPtr header) { const auto* any = Envoy::Protobuf::DynamicCastToGenerated(&message); LlmRequestSourcePluginConfig llm_config; THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(*any, llm_config)); @@ -116,18 +114,16 @@ Nighthawk::RequestSourcePtr LlmRequestSourcePluginFactory::createRequestSourcePl llm_config.options_list().options()) { for (const envoy::config::core::v3::HeaderValueOption& option_header : request_option.request_headers()) { - auto lower_case_key = - Envoy::Http::LowerCaseString(option_header.header().key()); + auto lower_case_key = Envoy::Http::LowerCaseString(option_header.header().key()); header->setCopy(lower_case_key, option_header.header().value()); } } - return std::make_unique( - std::string(llm_config.model_name()), llm_config.req_token_count(), - llm_config.resp_max_tokens(), std::move(header)); + return std::make_unique(std::string(llm_config.model_name()), + llm_config.req_token_count(), + llm_config.resp_max_tokens(), std::move(header)); }; -REGISTER_FACTORY(LlmRequestSourcePluginFactory, - Nighthawk::RequestSourcePluginConfigFactory); +REGISTER_FACTORY(LlmRequestSourcePluginFactory, Nighthawk::RequestSourcePluginConfigFactory); -} // namespace nighthawk +} // namespace nighthawk diff --git a/source/request_source/llm_request_source_plugin_impl.h b/source/request_source/llm_request_source_plugin_impl.h index 85d6ea3cb..6c3ef25d3 100644 --- a/source/request_source/llm_request_source_plugin_impl.h +++ b/source/request_source/llm_request_source_plugin_impl.h @@ -10,8 +10,8 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "envoy/config/core/v3/extension.pb.h" #include "envoy/api/api.h" +#include "envoy/config/core/v3/extension.pb.h" #include "envoy/http/header_map.h" #include "external/envoy/source/common/common/logger.h" #include "external/envoy/source/common/protobuf/protobuf.h" @@ -22,8 +22,7 @@ namespace nighthawk { -constexpr inline absl::string_view kLlmRequestSourcePluginName = - "nighthawk.request_source.llm"; +constexpr inline absl::string_view kLlmRequestSourcePluginName = "nighthawk.request_source.llm"; // A Nighthawk RequestSource that generates requests for LLM inference. // @@ -42,23 +41,19 @@ constexpr inline absl::string_view kLlmRequestSourcePluginName = // - Content-Type: application/json // - Content-Length: The length of the request body. // - :path: /v1/completions -class LlmRequestSourcePlugin - : public Nighthawk::RequestSource, - public Envoy::Logger::Loggable { - public: - explicit LlmRequestSourcePlugin(std::string model_name, int req_token_count, - int resp_max_tokens, - Envoy::Http::RequestHeaderMapPtr header) - : model_name_(model_name), - req_token_count_(req_token_count), - resp_max_tokens_(resp_max_tokens), - header_(std::move(header)) {}; +class LlmRequestSourcePlugin : public Nighthawk::RequestSource, + public Envoy::Logger::Loggable { +public: + explicit LlmRequestSourcePlugin(std::string model_name, int req_token_count, int resp_max_tokens, + Envoy::Http::RequestHeaderMapPtr header) + : model_name_(model_name), req_token_count_(req_token_count), + resp_max_tokens_(resp_max_tokens), header_(std::move(header)) {}; Nighthawk::RequestGenerator get() override; void initOnThread() override {}; void destroyOnThread() override {}; - private: +private: // Model to use for the request. std::string model_name_; // Number of tokens to generate in the request. @@ -70,20 +65,17 @@ class LlmRequestSourcePlugin Envoy::Http::RequestHeaderMapPtr header_; }; -class LlmRequestSourcePluginFactory - : public virtual Nighthawk::RequestSourcePluginConfigFactory { - public: - std::string name() const override { - return std::string(kLlmRequestSourcePluginName); - } +class LlmRequestSourcePluginFactory : public virtual Nighthawk::RequestSourcePluginConfigFactory { +public: + std::string name() const override { return std::string(kLlmRequestSourcePluginName); } Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); } - Nighthawk::RequestSourcePtr createRequestSourcePlugin( - const Envoy::Protobuf::Message&, Envoy::Api::Api&, - Envoy::Http::RequestHeaderMapPtr header) override; + Nighthawk::RequestSourcePtr + createRequestSourcePlugin(const Envoy::Protobuf::Message&, Envoy::Api::Api&, + Envoy::Http::RequestHeaderMapPtr header) override; }; -} // namespace nighthawk +} // namespace nighthawk From b7ee4bb2eaade4a02099895d8ae4f8d6244bc53e Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:47:30 +0000 Subject: [PATCH 07/28] format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/client/BUILD | 4 ++-- source/request_source/BUILD | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/client/BUILD b/source/client/BUILD index f329cca17..97cc32a61 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -103,10 +103,10 @@ envoy_cc_library( "//source/common:nighthawk_common_lib", "//source/common:nighthawk_service_client_impl", "//source/common:request_source_impl_lib", - "//source/request_source:request_options_list_plugin_impl", "//source/request_source:llm_request_source_plugin_cc_proto", "//source/request_source:llm_request_source_plugin_impl", - "//source/user_defined_output:user_defined_output_plugin_creator", + "//source/request_source:request_options_list_plugin_impl", + "//source/user_defined_output:user_defined_output_plugin_creator", "@envoy//envoy/config:xds_manager_interface", "@envoy//envoy/http:protocol_interface_with_external_headers", "@envoy//envoy/network:address_interface", diff --git a/source/request_source/BUILD b/source/request_source/BUILD index 6f2606eae..544ad8c91 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -56,24 +56,24 @@ envoy_cc_library( repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":llm_request_source_plugin_cc_proto", + ":llm_request_source_plugin_cc_proto", "//include/nighthawk/request_source:request_source_plugin_config_factory_lib", "//source/common:nighthawk_common_lib", "//source/common:request_impl_lib", "//source/common:request_source_impl_lib", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/random", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/strings:string_view", "@envoy//source/common/common:thread_lib_with_external_headers", "@envoy//source/common/protobuf:message_validator_lib_with_external_headers", "@envoy//source/common/protobuf:protobuf_with_external_headers", "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//source/exe:platform_header_lib_with_external_headers", "@envoy//source/exe:platform_impl_lib", - "@com_google_absl//absl/log", - "@com_google_absl//absl/log:check", - "@com_google_absl//absl/random", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/strings:string_view", ], ) From d773ce17668d18d34389e73c6eaf5193d6ac4374 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:55:00 +0000 Subject: [PATCH 08/28] format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/client/BUILD | 6 +++--- source/request_source/BUILD | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/client/BUILD b/source/client/BUILD index 97cc32a61..1520ef409 100644 --- a/source/client/BUILD +++ b/source/client/BUILD @@ -103,10 +103,10 @@ envoy_cc_library( "//source/common:nighthawk_common_lib", "//source/common:nighthawk_service_client_impl", "//source/common:request_source_impl_lib", - "//source/request_source:llm_request_source_plugin_cc_proto", - "//source/request_source:llm_request_source_plugin_impl", + "//source/request_source:llm_request_source_plugin_cc_proto", + "//source/request_source:llm_request_source_plugin_impl", "//source/request_source:request_options_list_plugin_impl", - "//source/user_defined_output:user_defined_output_plugin_creator", + "//source/user_defined_output:user_defined_output_plugin_creator", "@envoy//envoy/config:xds_manager_interface", "@envoy//envoy/http:protocol_interface_with_external_headers", "@envoy//envoy/network:address_interface", diff --git a/source/request_source/BUILD b/source/request_source/BUILD index 544ad8c91..cb7f4b95b 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -3,7 +3,6 @@ load( "envoy_cc_library", "envoy_package", ) - load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library") licenses(["notice"]) # Apache 2 @@ -76,4 +75,3 @@ envoy_cc_library( "@envoy//source/exe:platform_impl_lib", ], ) - From fce11acafc9f418f33ae3dc4805d577f36edfbf6 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:20:26 +0000 Subject: [PATCH 09/28] fix namespace Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/llm_request_source_plugin_impl.cc | 8 ++++---- source/request_source/llm_request_source_plugin_impl.h | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index 6bd2bd884..7a57bcfe6 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -28,10 +28,10 @@ #include "nighthawk/request_source/request_source_plugin_config_factory.h" #include "source/common/request_impl.h" -namespace nighthawk { +namespace Nighthawk { namespace { -absl::Status ValidateConfig(const LlmRequestSourcePluginConfig& config) { +absl::Status ValidateConfig(const nighthawk::LlmRequestSourcePluginConfig& config) { if (config.model_name().empty()) { return absl::InvalidArgumentError("Model name is required."); } @@ -106,7 +106,7 @@ LlmRequestSourcePluginFactory::createRequestSourcePlugin(const Envoy::Protobuf:: Envoy::Api::Api&, Envoy::Http::RequestHeaderMapPtr header) { const auto* any = Envoy::Protobuf::DynamicCastToGenerated(&message); - LlmRequestSourcePluginConfig llm_config; + nighthawk::LlmRequestSourcePluginConfig llm_config; THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(*any, llm_config)); THROW_IF_NOT_OK(ValidateConfig(llm_config)); @@ -126,4 +126,4 @@ LlmRequestSourcePluginFactory::createRequestSourcePlugin(const Envoy::Protobuf:: REGISTER_FACTORY(LlmRequestSourcePluginFactory, Nighthawk::RequestSourcePluginConfigFactory); -} // namespace nighthawk +} // namespace Nighthawk diff --git a/source/request_source/llm_request_source_plugin_impl.h b/source/request_source/llm_request_source_plugin_impl.h index 6c3ef25d3..8ba93770d 100644 --- a/source/request_source/llm_request_source_plugin_impl.h +++ b/source/request_source/llm_request_source_plugin_impl.h @@ -20,7 +20,7 @@ #include "nighthawk/common/request_source.h" #include "nighthawk/request_source/request_source_plugin_config_factory.h" -namespace nighthawk { +namespace Nighthawk { constexpr inline absl::string_view kLlmRequestSourcePluginName = "nighthawk.request_source.llm"; @@ -70,7 +70,7 @@ class LlmRequestSourcePluginFactory : public virtual Nighthawk::RequestSourcePlu std::string name() const override { return std::string(kLlmRequestSourcePluginName); } Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); + return std::make_unique(); } Nighthawk::RequestSourcePtr @@ -78,4 +78,4 @@ class LlmRequestSourcePluginFactory : public virtual Nighthawk::RequestSourcePlu Envoy::Http::RequestHeaderMapPtr header) override; }; -} // namespace nighthawk +} // namespace Nighthawk From ef7f06dd7d02905399a3257f28ef536787ea5487 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Wed, 4 Mar 2026 19:58:23 +0000 Subject: [PATCH 10/28] remove bashrc accidental change Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- .bazelrc | 2 -- 1 file changed, 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index e26be4257..e4927fd34 100644 --- a/.bazelrc +++ b/.bazelrc @@ -531,5 +531,3 @@ try-import %workspace%/repo.bazelrc try-import %workspace%/clang.bazelrc try-import %workspace%/user.bazelrc try-import %workspace%/local_tsan.bazelrc - -common --credential_helper=us-python.pkg.dev=/usr/local/google/home/graysonlafleur/gar_cred_helper.sh From be948e073139a882d967665754806740dae69846 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Wed, 4 Mar 2026 20:10:28 +0000 Subject: [PATCH 11/28] update with comments Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- LLM_LOAD_GENERATION.md | 16 ++++++++-------- .../llm_request_source_plugin_impl.cc | 14 ++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/LLM_LOAD_GENERATION.md b/LLM_LOAD_GENERATION.md index 2919acd39..77fb247c7 100644 --- a/LLM_LOAD_GENERATION.md +++ b/LLM_LOAD_GENERATION.md @@ -3,8 +3,9 @@ ## Overview If you would like to perform a load test against an LLM backend, using -LLM-formatted requests, there is an LLM Request Source plugin that can -emulate that workload. These request bodies looks like the following: +the [Completions API spec](https://developers.openai.com/api/docs/guides/completions/), +there is an LLM Request Source plugin that can emulate that workload. +These request bodies looks like the following: ``` { @@ -29,13 +30,12 @@ This is generated based on input you provide. The 4 inputs are: - Maximum number of tokens for the model to respond with 4. [Request Options List](https://github.com/envoyproxy/nighthawk/blob/09d64d769972513989a95766a98e28f5d6bb05c2/api/client/options.proto#L32) (optional) - This allows you to add headers and choose request method of the requests - - Header 'Content-Type: application/json' added by default - - Ignore the "request_body_size" and "json_body" in this field - - If a host name is required, use ":authority" header instead of ":host" -It's also important to note that all requests are routed to the path "/v1/completions". -There is not currently a way to override this through the CLI. If you need to use -a different path, you will need to edit it in source/request_source/llm_request_source_plugin_impl.cc. +A few additional details about the request options list: + +1. Header 'Content-Type: application/json' added by default +2. Ignore the "request_body_size" and "json_body" in this field +3. If a host name is required, use ":authority" header instead of ":host" The config for running with this request source is passed into the "--request-source-plugin-config" flag. Here is an example of how that flag might look for running a load test with this LLM Request Source plugin: diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index 7a57bcfe6..45b5dcd38 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -39,11 +39,12 @@ absl::Status ValidateConfig(const nighthawk::LlmRequestSourcePluginConfig& confi return absl::OkStatus(); } -std::string GenerateRandomPrompt(int num_tokens) { - static const char charset[] = "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; +constexpr absl::string_view kCharset = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; +std::string GenerateRandomPrompt(int num_tokens) { // The final string to be built. std::string result_string; @@ -52,7 +53,7 @@ std::string GenerateRandomPrompt(int num_tokens) { for (int i = 0; i < num_tokens; ++i) { // Append a random character from the charset. - result_string += charset[absl::Uniform(bitgen, 0, 62)]; + result_string += kCharset[absl::Uniform(bitgen, 0, kCharset.length())]; // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. @@ -71,7 +72,6 @@ Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); Envoy::Http::HeaderMapImpl::copyFrom(*headers, *header_); - // TODO(b/436267941): Add support for multiple messages. std::string body = absl::StrFormat(R"json( { @@ -95,8 +95,6 @@ Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() { auto path_key = Envoy::Http::LowerCaseString(":path"); headers->setCopy(path_key, "/v1/completions"); - ENVOY_LOG(info, body); - return std::make_unique(std::move(headers), body); }; } From c88b31dc2a524dc0def068ac33f30e5a59ec4a30 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Wed, 4 Mar 2026 20:18:44 +0000 Subject: [PATCH 12/28] fix format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/llm_request_source_plugin_impl.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index 45b5dcd38..d7fa6a5d3 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -39,10 +39,9 @@ absl::Status ValidateConfig(const nighthawk::LlmRequestSourcePluginConfig& confi return absl::OkStatus(); } -constexpr absl::string_view kCharset = - "0123456789" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz"; +constexpr absl::string_view kCharset = "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; std::string GenerateRandomPrompt(int num_tokens) { // The final string to be built. From c9e6a1b9f94ec42ad55d6e04ab86de240653b058 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Wed, 4 Mar 2026 20:36:44 +0000 Subject: [PATCH 13/28] update documentation structure Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- README.md | 5 +++++ .../root/howto/LLM_LOAD_GENERATION.md | 0 docs/root/overview.md | 4 ++++ 3 files changed, 9 insertions(+) rename LLM_LOAD_GENERATION.md => docs/root/howto/LLM_LOAD_GENERATION.md (100%) diff --git a/README.md b/README.md index 715b342cc..bf7ec95c8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ Nighthawk currently offers: See [navigating the codebase](docs/root/navigating_the_codebase.md) for a description of the directory structure. +## Additional Documentation + +See the [howto](docs/root/howto) directory for documentation aimed at specific +use-cases. + ## Building Nighthawk ### Prerequisites diff --git a/LLM_LOAD_GENERATION.md b/docs/root/howto/LLM_LOAD_GENERATION.md similarity index 100% rename from LLM_LOAD_GENERATION.md rename to docs/root/howto/LLM_LOAD_GENERATION.md diff --git a/docs/root/overview.md b/docs/root/overview.md index 700281176..efdba94d8 100644 --- a/docs/root/overview.md +++ b/docs/root/overview.md @@ -147,6 +147,10 @@ to fire off should look like. A couple of implementations exist: - a request source [plugin](https://github.com/envoyproxy/nighthawk/blob/9f97c2d9cb86b84a158ccba33832d135e1b96c7a/source/request_source/request_options_list_plugin_impl.h#L94) which replays requests from memory. +- a request source + [plugin](https://github.com/envoyproxy/nighthawk/blob/9f97c2d9cb86b84a158ccba33832d135e1b96c7a/source/request_source/llm_request_source_plugin_impl.h) + which creates requests based on the Completions API spec. See + [howto](howto/LLM_LOAD_GENERATION.md) for more details. ### StreamDecoder From 5456fe205a21d624fbfd9e9554012ff6f2db2ac4 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:53:57 +0000 Subject: [PATCH 14/28] add test cases for llm request source Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 15 +++ .../llm_request_source_plugin_test.cc | 92 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 test/request_source/llm_request_source_plugin_test.cc diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 28d234ccc..6778a7ad0 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -58,3 +58,18 @@ envoy_cc_test( "@envoy//test/mocks/api:api_mocks", ], ) + +envoy_cc_test( + name = "llm_request_source_plugin_test", + srcs = ["llm_request_source_plugin_test.cc"], + repository = "@envoy", + deps = [ + "//test/test_common:proto_matchers", + "//source/common:request_impl_lib", + "//source/common:request_source_impl_lib", + "//source/request_source:llm_request_source_plugin_impl", + "//source/request_source:llm_request_source_plugin_cc_proto", + "@envoy//source/common/http:header_map_lib_with_external_headers", + "@envoy//test/mocks/api:api_mocks", + ], +) diff --git a/test/request_source/llm_request_source_plugin_test.cc b/test/request_source/llm_request_source_plugin_test.cc new file mode 100644 index 000000000..89c029aca --- /dev/null +++ b/test/request_source/llm_request_source_plugin_test.cc @@ -0,0 +1,92 @@ +#include +#include +#include +#include + +#include "source/request_source/llm_request_source_plugin.pb.h" +#include "source/request_source/llm_request_source_plugin_impl.h" + +#include "test/test_common/proto_matchers.h" + +#include "nighthawk/common/request.h" +#include "nighthawk/common/request_source.h" +#include "source/common/common/assert.h" +#include "source/common/json/json_loader.h" + +#include "external/envoy/source/common/http/header_map_impl.h" +#include "external/envoy/test/mocks/api/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Nighthawk { +namespace { + +using ::Envoy::Protobuf::TextFormat; +using ::testing::NiceMock; + +TEST(LlmRequestSourcePluginTest, TestLlmRequestSourcePlugin) { + nighthawk::LlmRequestSourcePluginConfig config; + TextFormat::ParseFromString(R"pb( + model_name: "test_model" + req_token_count: 100 + resp_max_tokens: 100 + options_list { + options { + request_headers { header { key: "test_header" value: "test_value" } } + } + } + )pb", + &config); + Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); + LlmRequestSourcePluginFactory factory; + NiceMock mock_api; + google::protobuf::Any config_wrapper; + config_wrapper.PackFrom(config); + Nighthawk::RequestSourcePtr llm_request_source = + factory.createRequestSourcePlugin(config_wrapper, mock_api, std::move(headers)); + ASSERT_NE(llm_request_source, nullptr); + + Nighthawk::RequestGenerator request_generator = llm_request_source->get(); + std::unique_ptr request = request_generator(); + + Envoy::Json::ObjectSharedPtr json_object = + Envoy::Json::Factory::loadFromString(request->body()).value(); + + EXPECT_EQ(json_object->getString("model").value(), "test_model"); + EXPECT_EQ(json_object->getInteger("max_tokens").value(), 100); + + std::vector messages = + json_object->getObjectArray("messages").value(); + Envoy::Json::ObjectSharedPtr first_message_obj = messages[0]; + std::string content = first_message_obj->getString("content").value(); + std::vector tokens = absl::StrSplit(content, ' '); + EXPECT_EQ(tokens.size(), 100); + EXPECT_EQ(request->header()->get(Envoy::Http::LowerCaseString("test_header")).size(), 1); +} + +TEST(LlmRequestSourcePluginTest, TestLlmRequestSourcePluginFactory) { + nighthawk::LlmRequestSourcePluginConfig config; + TextFormat::ParseFromString(R"pb( + model_name: "test_model" + req_token_count: 100 + resp_max_tokens: 100 + options_list { + options { + request_headers { header { key: "test_header" value: "test_value" } } + } + } + )pb", + &config); + Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); + LlmRequestSourcePluginFactory factory; + NiceMock mock_api; + google::protobuf::Any config_wrapper; + config_wrapper.PackFrom(config); + Nighthawk::RequestSourcePtr llm_request_source = + factory.createRequestSourcePlugin(config_wrapper, mock_api, std::move(headers)); + ASSERT_NE(llm_request_source, nullptr); +} + +} // namespace +} // namespace Nighthawk From e6196993ea77dcd70680f54869ac9183a37414d0 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:22:58 +0000 Subject: [PATCH 15/28] format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 6778a7ad0..1c4c896c2 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -64,12 +64,12 @@ envoy_cc_test( srcs = ["llm_request_source_plugin_test.cc"], repository = "@envoy", deps = [ - "//test/test_common:proto_matchers", - "//source/common:request_impl_lib", + "//source/common:request_impl_lib", "//source/common:request_source_impl_lib", - "//source/request_source:llm_request_source_plugin_impl", - "//source/request_source:llm_request_source_plugin_cc_proto", - "@envoy//source/common/http:header_map_lib_with_external_headers", - "@envoy//test/mocks/api:api_mocks", + "//source/request_source:llm_request_source_plugin_cc_proto", + "//source/request_source:llm_request_source_plugin_impl", + "//test/test_common:proto_matchers", + "@envoy//source/common/http:header_map_lib_with_external_headers", + "@envoy//test/mocks/api:api_mocks", ], ) From 845bf7753fcb19916a2a607c02d936681c63d17c Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:35:39 +0000 Subject: [PATCH 16/28] add direct dependency Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/llm_request_source_plugin_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/request_source/llm_request_source_plugin_test.cc b/test/request_source/llm_request_source_plugin_test.cc index 89c029aca..90c27dd27 100644 --- a/test/request_source/llm_request_source_plugin_test.cc +++ b/test/request_source/llm_request_source_plugin_test.cc @@ -14,6 +14,7 @@ #include "source/common/json/json_loader.h" #include "external/envoy/source/common/http/header_map_impl.h" +#include "external/envoy/source/common/protobuf/protobuf.h" #include "external/envoy/test/mocks/api/mocks.h" #include "gmock/gmock.h" From 605f98bd9b911f700faa0342c2135b2e442cd43f Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:27:01 +0000 Subject: [PATCH 17/28] nits Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- .bazelrc | 1 + source/request_source/llm_request_source_plugin.proto | 4 ++-- source/request_source/llm_request_source_plugin_impl.cc | 5 +---- source/request_source/llm_request_source_plugin_impl.h | 4 +++- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bazelrc b/.bazelrc index e4927fd34..088d99341 100644 --- a/.bazelrc +++ b/.bazelrc @@ -531,3 +531,4 @@ try-import %workspace%/repo.bazelrc try-import %workspace%/clang.bazelrc try-import %workspace%/user.bazelrc try-import %workspace%/local_tsan.bazelrc + diff --git a/source/request_source/llm_request_source_plugin.proto b/source/request_source/llm_request_source_plugin.proto index cc04fa8c6..b5f8c1c5c 100644 --- a/source/request_source/llm_request_source_plugin.proto +++ b/source/request_source/llm_request_source_plugin.proto @@ -9,10 +9,10 @@ message LlmRequestSourcePluginConfig { // Model to use for the request. This field is required. string model_name = 1; - // Number of tokens to generate in the request. + // Number of tokens to generate in the request. Defaults to 0. int32 req_token_count = 2; - // Maximum number of tokens to return in the response. + // Maximum number of tokens to return in the response. Defaults to 0. int32 resp_max_tokens = 3; // The options_list will be used to apply headers to the request. diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index d7fa6a5d3..b4ad5953d 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -44,10 +44,7 @@ constexpr absl::string_view kCharset = "0123456789" "abcdefghijklmnopqrstuvwxyz"; std::string GenerateRandomPrompt(int num_tokens) { - // The final string to be built. std::string result_string; - - // Random number generator for selecting characters from the charset. absl::BitGen bitgen; for (int i = 0; i < num_tokens; ++i) { @@ -57,7 +54,7 @@ std::string GenerateRandomPrompt(int num_tokens) { // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. if (i < num_tokens - 1) { - result_string += ' '; + result_string += ' '; } } diff --git a/source/request_source/llm_request_source_plugin_impl.h b/source/request_source/llm_request_source_plugin_impl.h index 8ba93770d..5057fdd08 100644 --- a/source/request_source/llm_request_source_plugin_impl.h +++ b/source/request_source/llm_request_source_plugin_impl.h @@ -24,7 +24,7 @@ namespace Nighthawk { constexpr inline absl::string_view kLlmRequestSourcePluginName = "nighthawk.request_source.llm"; -// A Nighthawk RequestSource that generates requests for LLM inference. +// A Nighthawk RequestSource that generates completions API requests. // // The request source generates requests with the following characteristics: // - The request body is a JSON object with the following fields: @@ -62,9 +62,11 @@ class LlmRequestSourcePlugin : public Nighthawk::RequestSource, int resp_max_tokens_; // The options_list will be used to apply headers to the request. std::unique_ptr options_list_; + // Headers for the request. Envoy::Http::RequestHeaderMapPtr header_; }; +// Factory class for creating LlmRequestSourcePlugin objects. class LlmRequestSourcePluginFactory : public virtual Nighthawk::RequestSourcePluginConfigFactory { public: std::string name() const override { return std::string(kLlmRequestSourcePluginName); } From 4d32939b58c6546bad015936df226037007b9fd7 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 18:37:31 +0000 Subject: [PATCH 18/28] format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/llm_request_source_plugin_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index b4ad5953d..c333bc32d 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -54,7 +54,7 @@ std::string GenerateRandomPrompt(int num_tokens) { // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. if (i < num_tokens - 1) { - result_string += ' '; + result_string += ' '; } } From 6a5156d09893ac8bdff23f60cbb89b81d8907ca2 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:00:27 +0000 Subject: [PATCH 19/28] fix nit and add strappend support Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/BUILD | 1 + source/request_source/llm_request_source_plugin_impl.cc | 5 +++-- test/request_source/BUILD | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/request_source/BUILD b/source/request_source/BUILD index cb7f4b95b..5c0a312a2 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -65,6 +65,7 @@ envoy_cc_library( "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:string_view", "@envoy//source/common/common:thread_lib_with_external_headers", diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index c333bc32d..4e6a1a567 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -8,6 +8,7 @@ #include "absl/random/random.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "source/request_source/llm_request_source_plugin.pb.h" @@ -49,12 +50,12 @@ std::string GenerateRandomPrompt(int num_tokens) { for (int i = 0; i < num_tokens; ++i) { // Append a random character from the charset. - result_string += kCharset[absl::Uniform(bitgen, 0, kCharset.length())]; + absl::StrAppend(&result_string, std::string(1, kCharset[absl::Uniform(bitgen, 0, kCharset.length())])); // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. if (i < num_tokens - 1) { - result_string += ' '; + absl::StrAppend(&result_string, " "); } } diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 1c4c896c2..3d019f021 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -70,6 +70,7 @@ envoy_cc_test( "//source/request_source:llm_request_source_plugin_impl", "//test/test_common:proto_matchers", "@envoy//source/common/http:header_map_lib_with_external_headers", + "@envoy//source/common/protobuf:protobuf_with_external_headers", "@envoy//test/mocks/api:api_mocks", ], ) From 470cd4dde23941421bcb397e2f1cc4b6ff6bd557 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:11:49 +0000 Subject: [PATCH 20/28] tab format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/request_source/BUILD b/source/request_source/BUILD index 5c0a312a2..0b7761ea0 100644 --- a/source/request_source/BUILD +++ b/source/request_source/BUILD @@ -65,7 +65,7 @@ envoy_cc_library( "@com_google_absl//absl/random", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:string_view", "@envoy//source/common/common:thread_lib_with_external_headers", From f66f634c7e71aaa2c1238f925b3498d05345740f Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:19:53 +0000 Subject: [PATCH 21/28] tab format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 3d019f021..e7e5dec80 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -70,7 +70,7 @@ envoy_cc_test( "//source/request_source:llm_request_source_plugin_impl", "//test/test_common:proto_matchers", "@envoy//source/common/http:header_map_lib_with_external_headers", - "@envoy//source/common/protobuf:protobuf_with_external_headers", + "@envoy//source/common/protobuf:protobuf_with_external_headers", "@envoy//test/mocks/api:api_mocks", ], ) From c068d79c4e3484829db9a4ab0593956cec1f830c Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:30:01 +0000 Subject: [PATCH 22/28] format fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/llm_request_source_plugin_impl.cc | 3 ++- test/request_source:llm_request_source_plugin_test | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test/request_source:llm_request_source_plugin_test diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index 4e6a1a567..9c437758c 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -50,7 +50,8 @@ std::string GenerateRandomPrompt(int num_tokens) { for (int i = 0; i < num_tokens; ++i) { // Append a random character from the charset. - absl::StrAppend(&result_string, std::string(1, kCharset[absl::Uniform(bitgen, 0, kCharset.length())])); + absl::StrAppend(&result_string, + std::string(1, kCharset[absl::Uniform(bitgen, 0, kCharset.length())])); // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. diff --git a/test/request_source:llm_request_source_plugin_test b/test/request_source:llm_request_source_plugin_test new file mode 100644 index 000000000..e69de29bb From 52e164e13e60eaaf809f44ffa542fdd3b4436cf8 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Fri, 6 Mar 2026 20:55:32 +0000 Subject: [PATCH 23/28] format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- source/request_source/llm_request_source_plugin_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/request_source/llm_request_source_plugin_impl.cc b/source/request_source/llm_request_source_plugin_impl.cc index 9c437758c..45b88f300 100644 --- a/source/request_source/llm_request_source_plugin_impl.cc +++ b/source/request_source/llm_request_source_plugin_impl.cc @@ -51,7 +51,7 @@ std::string GenerateRandomPrompt(int num_tokens) { for (int i = 0; i < num_tokens; ++i) { // Append a random character from the charset. absl::StrAppend(&result_string, - std::string(1, kCharset[absl::Uniform(bitgen, 0, kCharset.length())])); + std::string(1, kCharset[absl::Uniform(bitgen, 0, kCharset.length())])); // Add a space between tokens. This is a naive way to calculate the number // of tokens in the string as generally spaces delineate tokens. From 39f7a1ccec4df2a27408c03ecbcd6d4b95b3f109 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:38:09 +0000 Subject: [PATCH 24/28] format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 2 +- test/request_source/llm_request_source_plugin_test.cc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index e7e5dec80..5e9ab2b05 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -68,9 +68,9 @@ envoy_cc_test( "//source/common:request_source_impl_lib", "//source/request_source:llm_request_source_plugin_cc_proto", "//source/request_source:llm_request_source_plugin_impl", - "//test/test_common:proto_matchers", "@envoy//source/common/http:header_map_lib_with_external_headers", "@envoy//source/common/protobuf:protobuf_with_external_headers", + "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//test/mocks/api:api_mocks", ], ) diff --git a/test/request_source/llm_request_source_plugin_test.cc b/test/request_source/llm_request_source_plugin_test.cc index 90c27dd27..244838e13 100644 --- a/test/request_source/llm_request_source_plugin_test.cc +++ b/test/request_source/llm_request_source_plugin_test.cc @@ -15,6 +15,7 @@ #include "external/envoy/source/common/http/header_map_impl.h" #include "external/envoy/source/common/protobuf/protobuf.h" +#include "external/envoy/source/common/protobuf/utility.h" #include "external/envoy/test/mocks/api/mocks.h" #include "gmock/gmock.h" From dc533750d3b0b2b0ff691c0f1bb4f64928808471 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:50:31 +0000 Subject: [PATCH 25/28] format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 5e9ab2b05..d739b3a52 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -70,7 +70,7 @@ envoy_cc_test( "//source/request_source:llm_request_source_plugin_impl", "@envoy//source/common/http:header_map_lib_with_external_headers", "@envoy//source/common/protobuf:protobuf_with_external_headers", - "@envoy//source/common/protobuf:utility_lib_with_external_headers", + "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//test/mocks/api:api_mocks", ], ) From bce99d6c50cbbcd3d441b33f838b13d87b0a6357 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:17:01 +0000 Subject: [PATCH 26/28] BUILD issue fix Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/llm_request_source_plugin_test.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/request_source/llm_request_source_plugin_test.cc b/test/request_source/llm_request_source_plugin_test.cc index 244838e13..f3e408ac2 100644 --- a/test/request_source/llm_request_source_plugin_test.cc +++ b/test/request_source/llm_request_source_plugin_test.cc @@ -6,8 +6,6 @@ #include "source/request_source/llm_request_source_plugin.pb.h" #include "source/request_source/llm_request_source_plugin_impl.h" -#include "test/test_common/proto_matchers.h" - #include "nighthawk/common/request.h" #include "nighthawk/common/request_source.h" #include "source/common/common/assert.h" @@ -43,7 +41,7 @@ TEST(LlmRequestSourcePluginTest, TestLlmRequestSourcePlugin) { Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); LlmRequestSourcePluginFactory factory; NiceMock mock_api; - google::protobuf::Any config_wrapper; + Envoy::Protobuf::Any config_wrapper; config_wrapper.PackFrom(config); Nighthawk::RequestSourcePtr llm_request_source = factory.createRequestSourcePlugin(config_wrapper, mock_api, std::move(headers)); @@ -83,7 +81,7 @@ TEST(LlmRequestSourcePluginTest, TestLlmRequestSourcePluginFactory) { Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create(); LlmRequestSourcePluginFactory factory; NiceMock mock_api; - google::protobuf::Any config_wrapper; + Envoy::Protobuf::Any config_wrapper; config_wrapper.PackFrom(config); Nighthawk::RequestSourcePtr llm_request_source = factory.createRequestSourcePlugin(config_wrapper, mock_api, std::move(headers)); From 1940ea4f3bc27893c33c0ebbe3881a2400d75328 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:31:07 +0000 Subject: [PATCH 27/28] fix include Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 2 ++ test/request_source/llm_request_source_plugin_test.cc | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index d739b3a52..9446be954 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -68,7 +68,9 @@ envoy_cc_test( "//source/common:request_source_impl_lib", "//source/request_source:llm_request_source_plugin_cc_proto", "//source/request_source:llm_request_source_plugin_impl", + "@envoy//source/common/common:assert_lib_with_external_headers", "@envoy//source/common/http:header_map_lib_with_external_headers", + "@envoy//source/common/json:json_loader_lib", "@envoy//source/common/protobuf:protobuf_with_external_headers", "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//test/mocks/api:api_mocks", diff --git a/test/request_source/llm_request_source_plugin_test.cc b/test/request_source/llm_request_source_plugin_test.cc index f3e408ac2..12d3b6310 100644 --- a/test/request_source/llm_request_source_plugin_test.cc +++ b/test/request_source/llm_request_source_plugin_test.cc @@ -8,10 +8,10 @@ #include "nighthawk/common/request.h" #include "nighthawk/common/request_source.h" -#include "source/common/common/assert.h" -#include "source/common/json/json_loader.h" +#include "external/envoy/source/common/common/assert.h" #include "external/envoy/source/common/http/header_map_impl.h" +#include "external/envoy/source/common/json/json_loader.h" #include "external/envoy/source/common/protobuf/protobuf.h" #include "external/envoy/source/common/protobuf/utility.h" #include "external/envoy/test/mocks/api/mocks.h" From 34c6a2834e8f253c32bf08817011549974a28956 Mon Sep 17 00:00:00 2001 From: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:37:08 +0000 Subject: [PATCH 28/28] fix format Signed-off-by: grayson-lafleur-google <193935438+Grayson-LaFleur-Google@users.noreply.github.com> --- test/request_source/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/request_source/BUILD b/test/request_source/BUILD index 9446be954..2968fb1c5 100644 --- a/test/request_source/BUILD +++ b/test/request_source/BUILD @@ -70,7 +70,7 @@ envoy_cc_test( "//source/request_source:llm_request_source_plugin_impl", "@envoy//source/common/common:assert_lib_with_external_headers", "@envoy//source/common/http:header_map_lib_with_external_headers", - "@envoy//source/common/json:json_loader_lib", + "@envoy//source/common/json:json_loader_lib", "@envoy//source/common/protobuf:protobuf_with_external_headers", "@envoy//source/common/protobuf:utility_lib_with_external_headers", "@envoy//test/mocks/api:api_mocks",