nighthawk/source/request_source/llm_request_source_plugin_impl.cc at main · envoyproxy/nighthawk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#include "source/request_source/llm_request_source_plugin_impl.h"

#include <memory>
#include <string>
#include <utility>

#include "absl/log/check.h"
#include "absl/random/random.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"

#include "source/request_source/llm_request_source_plugin.pb.h"

#include "envoy/api/api.h"
#include "envoy/config/core/v3/base.pb.h"
#include "envoy/config/core/v3/extension.pb.h"
#include "envoy/http/header_map.h"
#include "envoy/registry/registry.h"
#include "external/envoy/source/common/http/header_map_impl.h"
#include "external/envoy/source/common/protobuf/protobuf.h"
#include "external/envoy/source/common/protobuf/utility.h"

#include "api/client/options.pb.h"
#include "api/request_source/request_source_plugin.pb.h"
#include "nighthawk/common/request.h"
#include "nighthawk/common/request_source.h"
#include "nighthawk/request_source/request_source_plugin_config_factory.h"
#include "source/common/request_impl.h"

namespace Nighthawk {
namespace {

absl::Status ValidateConfig(const nighthawk::LlmRequestSourcePluginConfig& config) {
  if (config.model_name().empty()) {
    return absl::InvalidArgumentError("Model name is required.");
  }

  return absl::OkStatus();
}

constexpr absl::string_view kCharset = "0123456789"
                                       "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                       "abcdefghijklmnopqrstuvwxyz";

std::string GenerateRandomPrompt(int num_tokens) {
  std::string result_string;
  absl::BitGen bitgen;

  for (int i = 0; i < num_tokens; ++i) {
    // Append a random character from the charset.
    absl::StrAppend(&result_string,
                    std::string(1, kCharset[absl::Uniform<size_t>(bitgen, 0, kCharset.length())]));

    // Add a space between tokens. This is a naive way to calculate the number
    // of tokens in the string as generally spaces delineate tokens.
    if (i < num_tokens - 1) {
      absl::StrAppend(&result_string, " ");
    }
  }

  return result_string;
}

} // namespace

Nighthawk::RequestGenerator LlmRequestSourcePlugin::get() {
  return [this]() -> std::unique_ptr<Nighthawk::Request> {
    Envoy::Http::RequestHeaderMapPtr headers = Envoy::Http::RequestHeaderMapImpl::create();
    Envoy::Http::HeaderMapImpl::copyFrom(*headers, *header_);

    std::string body =
        absl::StrFormat(R"json(
      {
        "model": "%s",
        "max_tokens": %d,
        "messages": [
          {
            "role": "user",
            "content": "%s"
          }
        ]
      }
    )json",
                        model_name_, resp_max_tokens_, GenerateRandomPrompt(req_token_count_));

    headers->setMethod(
        envoy::config::core::v3::RequestMethod_Name(envoy::config::core::v3::RequestMethod::POST));
    headers->setContentType("application/json");
    headers->setContentLength(body.size());

    auto path_key = Envoy::Http::LowerCaseString(":path");
    headers->setCopy(path_key, "/v1/completions");

    return std::make_unique<Nighthawk::RequestImpl>(std::move(headers), body);
  };
}

Nighthawk::RequestSourcePtr
LlmRequestSourcePluginFactory::createRequestSourcePlugin(const Envoy::Protobuf::Message& message,
                                                         Envoy::Api::Api&,
                                                         Envoy::Http::RequestHeaderMapPtr header) {
  const auto* any = Envoy::Protobuf::DynamicCastToGenerated<const Envoy::Protobuf::Any>(&message);
  nighthawk::LlmRequestSourcePluginConfig llm_config;
  THROW_IF_NOT_OK(Envoy::MessageUtil::unpackTo(*any, llm_config));
  THROW_IF_NOT_OK(ValidateConfig(llm_config));

  for (const nighthawk::client::RequestOptions& request_option :
       llm_config.options_list().options()) {
    for (const envoy::config::core::v3::HeaderValueOption& option_header :
         request_option.request_headers()) {
      auto lower_case_key = Envoy::Http::LowerCaseString(option_header.header().key());
      header->setCopy(lower_case_key, option_header.header().value());
    }
  }

  return std::make_unique<LlmRequestSourcePlugin>(std::string(llm_config.model_name()),
                                                  llm_config.req_token_count(),
                                                  llm_config.resp_max_tokens(), std::move(header));
};

REGISTER_FACTORY(LlmRequestSourcePluginFactory, Nighthawk::RequestSourcePluginConfigFactory);

} // namespace Nighthawk