From 5e92ed26465056c00f339fac086281f15b54013b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 13 Oct 2023 17:55:27 +0530 Subject: [PATCH 01/46] Add JVM Docker image --- docker/jvm/Dockerfile | 89 ++++ .../jvm/__pycache__/constants.cpython-39.pyc | Bin 0 -> 951 bytes docker/jvm/bash-config | 26 + docker/jvm/constants.py | 19 + docker/jvm/docker-compose.yml | 25 + docker/jvm/docker_release.py | 15 + docker/jvm/docker_sanity_test.py | 312 +++++++++++ docker/jvm/fixtures/input.txt | 0 docker/jvm/fixtures/kraft/docker-compose.yml | 100 ++++ docker/jvm/fixtures/native/docker-compose.yml | 68 +++ docker/jvm/fixtures/output.txt | 0 docker/jvm/fixtures/schema.avro | 8 + .../secrets/broker_broker-ssl_cert-file | 17 + .../secrets/broker_broker-ssl_cert-signed | 20 + .../broker_broker-ssl_server.keystore.jks | Bin 0 -> 4750 bytes .../broker_broker-ssl_server.truststore.jks | Bin 0 -> 1238 bytes .../fixtures/secrets/broker_broker_cert-file | 17 + .../secrets/broker_broker_cert-signed | 20 + .../secrets/broker_broker_server.keystore.jks | Bin 0 -> 4750 bytes .../broker_broker_server.truststore.jks | Bin 0 -> 1238 bytes docker/jvm/fixtures/secrets/ca-cert | 20 + docker/jvm/fixtures/secrets/ca-cert.key | 30 ++ docker/jvm/fixtures/secrets/ca-cert.srl | 1 + .../fixtures/secrets/client_python_client.key | 30 ++ .../fixtures/secrets/client_python_client.pem | 20 + .../fixtures/secrets/client_python_client.req | 17 + .../fixtures/secrets/kafka-generate-ssl.sh | 164 ++++++ .../jvm/fixtures/secrets/kafka_keystore_creds | 1 + .../jvm/fixtures/secrets/kafka_ssl_key_creds | 1 + .../fixtures/secrets/kafka_truststore_creds | 1 + docker/jvm/fixtures/sink_connector.json | 11 + docker/jvm/fixtures/source_connector.json | 14 + .../jvm/fixtures/zookeeper/docker-compose.yml | 132 +++++ docker/jvm/include/etc/kafka/docker/configure | 152 ++++++ .../etc/kafka/docker/configureDefaults | 45 ++ docker/jvm/include/etc/kafka/docker/ensure | 35 ++ .../docker/kafka-log4j.properties.template | 11 + .../kafka/docker/kafka-propertiesSpec.json | 25 + .../kafka-tools-log4j.properties.template | 6 + docker/jvm/include/etc/kafka/docker/launch | 30 ++ docker/jvm/include/etc/kafka/docker/run | 50 ++ docker/jvm/requirements.txt | 6 + docker/jvm/ub/go.mod | 15 + docker/jvm/ub/go.sum | 14 + docker/jvm/ub/testResources/sampleFile | 0 docker/jvm/ub/testResources/sampleFile2 | 0 .../jvm/ub/testResources/sampleLog4j.template | 11 + docker/jvm/ub/ub.go | 496 ++++++++++++++++++ docker/jvm/ub/ub_test.go | 446 ++++++++++++++++ 49 files changed, 2520 insertions(+) create mode 100644 docker/jvm/Dockerfile create mode 100644 docker/jvm/__pycache__/constants.cpython-39.pyc create mode 100644 docker/jvm/bash-config create mode 100644 docker/jvm/constants.py create mode 100644 docker/jvm/docker-compose.yml create mode 100644 docker/jvm/docker_release.py create mode 100644 docker/jvm/docker_sanity_test.py create mode 100644 docker/jvm/fixtures/input.txt create mode 100644 docker/jvm/fixtures/kraft/docker-compose.yml create mode 100644 docker/jvm/fixtures/native/docker-compose.yml create mode 100644 docker/jvm/fixtures/output.txt create mode 100644 docker/jvm/fixtures/schema.avro create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker_cert-file create mode 100644 docker/jvm/fixtures/secrets/broker_broker_cert-signed create mode 100644 docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks create mode 100644 docker/jvm/fixtures/secrets/ca-cert create mode 100644 docker/jvm/fixtures/secrets/ca-cert.key create mode 100644 docker/jvm/fixtures/secrets/ca-cert.srl create mode 100644 docker/jvm/fixtures/secrets/client_python_client.key create mode 100644 docker/jvm/fixtures/secrets/client_python_client.pem create mode 100644 docker/jvm/fixtures/secrets/client_python_client.req create mode 100755 docker/jvm/fixtures/secrets/kafka-generate-ssl.sh create mode 100644 docker/jvm/fixtures/secrets/kafka_keystore_creds create mode 100644 docker/jvm/fixtures/secrets/kafka_ssl_key_creds create mode 100644 docker/jvm/fixtures/secrets/kafka_truststore_creds create mode 100644 docker/jvm/fixtures/sink_connector.json create mode 100644 docker/jvm/fixtures/source_connector.json create mode 100644 docker/jvm/fixtures/zookeeper/docker-compose.yml create mode 100755 docker/jvm/include/etc/kafka/docker/configure create mode 100755 docker/jvm/include/etc/kafka/docker/configureDefaults create mode 100755 docker/jvm/include/etc/kafka/docker/ensure create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template create mode 100755 docker/jvm/include/etc/kafka/docker/launch create mode 100755 docker/jvm/include/etc/kafka/docker/run create mode 100644 docker/jvm/requirements.txt create mode 100644 docker/jvm/ub/go.mod create mode 100644 docker/jvm/ub/go.sum create mode 100755 docker/jvm/ub/testResources/sampleFile create mode 100755 docker/jvm/ub/testResources/sampleFile2 create mode 100644 docker/jvm/ub/testResources/sampleLog4j.template create mode 100644 docker/jvm/ub/ub.go create mode 100644 docker/jvm/ub/ub_test.go diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile new file mode 100644 index 0000000000000..ff54a3853f667 --- /dev/null +++ b/docker/jvm/Dockerfile @@ -0,0 +1,89 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +ARG GOLANG_VERSION=1.21.1 + +FROM golang:${GOLANG_VERSION} AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM eclipse-temurin:17-jre + +COPY bash-config /etc/kafka/docker/bash-config + +# exposed ports +EXPOSE 9092 + +USER root + +ARG kafka_url=https://archive.apache.org/dist/kafka/3.5.1/kafka_2.13-3.5.1.tgz +ARG kafka_version=2.13-3.5.1 +ARG vcs_ref=unspecified +ARG build_date=unspecified + +LABEL org.label-schema.name="kafka" \ + org.label-schema.description="Apache Kafka" \ + org.label-schema.build-date="${build_date}" \ + org.label-schema.vcs-url="https://github.com/apache/kafka" \ + org.label-schema.vcs-ref="${vcs_ref}" \ + org.label-schema.version="${kafka_version}" \ + org.label-schema.schema-version="1.0" \ + maintainer="apache" + +ENV KAFKA_URL=$kafka_url + +# allow arg override of required env params +ARG KAFKA_ZOOKEEPER_CONNECT +ENV KAFKA_ZOOKEEPER_CONNECT=${KAFKA_ZOOKEEPER_CONNECT} +ARG KAFKA_ADVERTISED_LISTENERS +ENV KAFKA_ADVERTISED_LISTENERS=${KAFKA_ADVERTISED_LISTENERS} +ARG CLUSTER_ID +ENV CLUSTER_ID=${CLUSTER_ID} + +RUN set -eux ; \ + apt-get update ; \ + apt-get upgrade -y ; \ + apt-get install -y --no-install-recommends curl wget gpg dirmngr gpg-agent; \ + mkdir opt/kafka; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ + wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ + gpg --import KEYS; \ + gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ + mkdir -p /etc/kafka/docker /usr/logs; \ + useradd --no-log-init --create-home --shell /bin/bash appuser; \ + chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ + chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + rm kafka.tgz; + +COPY --from=build-ub /build/ub /usr/bin +COPY --chown=appuser:appuser include/etc/kafka/docker /etc/kafka/docker + +USER appuser + +VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data"] + +CMD ["/etc/kafka/docker/run"] diff --git a/docker/jvm/__pycache__/constants.cpython-39.pyc b/docker/jvm/__pycache__/constants.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fca3f425054fd2ab00a8e3af2eac9a849d6efef GIT binary patch literal 951 zcma)5-EPw`6i&Nt{ab%lKw<+iZjh)DH3$h!6GAJQJEC<><)ni7#>#A$bZwI=b|=~^ zUWZrV5pv5FufP>fXuGWR0!KMI-*)W$B|B=?TpAkr|lmaK}6{pHD z4QnigQ!I_sYzuF(49>7^yv=s-4$I;!%i$c$<2)VRBcD1W*BA2m=+)8d`O{7G8$f-}<5c>A`1M?AzlgmlH`sjOGUoaGSwtz@-4xc}aO7=Px$JMg#6i zGMP!&_Xv$qW5UP(zj?@KIOkkZ#`|Afyr0l>75GCQ5G7tZfyDPL{Uf|+5(Li-%SM+3 zB7}>&+37&tjzQM4+ObpnfXs7<)K<%GqBz*8SHpiQ^iVroRfs@YZyT^<6Wh3eX3q`} z)_dBjBC*Y`p@+3)>jZbWvQ-b9lLd={7CvlZj5l{k1bER6Sllh5B+zQWwFN@j1M|)O?4HTi#(Ftu;sn zziUBFUCY?~ehYen2Ljx1rsn1f*9pcB@Pz None: + subprocess.run(["docker", "stop", self.CONTAINER_NAME]) + + def startCompose(self, filename) -> None: + old_string="image: {$IMAGE}" + new_string=f"image: {self.IMAGE}" + + with open(filename) as f: + s = f.read() + if old_string not in s: + print('"{old_string}" not found in {filename}.'.format(**locals())) + + with open(filename, 'w') as f: + print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) + s = s.replace(old_string, new_string) + f.write(s) + + subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) + time.sleep(25) + + def destroyCompose(self, filename) -> None: + old_string=f"image: {self.IMAGE}" + new_string="image: {$IMAGE}" + + subprocess.run(["docker-compose", "-f", filename, "down"]) + time.sleep(10) + with open(filename) as f: + s = f.read() + if old_string not in s: + print('"{old_string}" not found in {filename}.'.format(**locals())) + + with open(filename, 'w') as f: + print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) + s = s.replace(old_string, new_string) + f.write(s) + + def create_topic(self, topic): + kafka_admin = confluent_kafka.admin.AdminClient({"bootstrap.servers": "localhost:9092"}) + new_topic = confluent_kafka.admin.NewTopic(topic, 1, 1) + kafka_admin.create_topics([new_topic,]) + timeout = constants.CLIENT_TIMEOUT + while timeout > 0: + timeout -= 1 + if topic not in kafka_admin.list_topics().topics: + time.sleep(1) + continue + return topic + return None + + def produce_message(self, topic, producer_config, key, value): + producer = Producer(producer_config) + producer.produce(topic, key=key, value=value) + producer.flush() + del producer + + def consume_message(self, topic, consumer_config): + consumer = Consumer(consumer_config) + consumer.subscribe([topic]) + timeout = constants.CLIENT_TIMEOUT + while timeout > 0: + message = consumer.poll(1) + if message is None: + time.sleep(1) + timeout -= 1 + continue + del consumer + return message + raise None + + def schema_registry_flow(self): + print("Running Schema Registry tests") + errors = [] + schema_registry_conf = {'url': constants.SCHEMA_REGISTRY_URL} + schema_registry_client = SchemaRegistryClient(schema_registry_conf) + avro_schema = "" + with open("fixtures/schema.avro") as f: + avro_schema = f.read() + avro_serializer = AvroSerializer(schema_registry_client=schema_registry_client, + schema_str=avro_schema) + producer_config = { + "bootstrap.servers": "localhost:9092", + } + + avro_deserializer = AvroDeserializer(schema_registry_client, avro_schema) + + key = {"key": "key", "value": ""} + value = {"value": "message", "key": ""} + self.produce_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, producer_config, key=avro_serializer(key, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.KEY)), value=avro_serializer(value, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.VALUE))) + time.sleep(3) + + consumer_config = { + "bootstrap.servers": "localhost:9092", + "group.id": "test-group", + 'auto.offset.reset': "earliest" + } + + message = self.consume_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, consumer_config) + + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + return + + deserialized_value = avro_deserializer(message.value(), SerializationContext(message.topic(), MessageField.VALUE)) + deserialized_key = avro_deserializer(message.key(), SerializationContext(message.topic(), MessageField.KEY)) + try: + self.assertEqual(deserialized_key, key) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + try: + self.assertEqual(deserialized_value, value) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + print("Errors in Schema Registry Test Flow:-", errors) + return errors + + def connect_flow(self): + print("Running Connect tests") + errors = [] + try: + self.assertEqual(self.create_topic(constants.CONNECT_TEST_TOPIC), constants.CONNECT_TEST_TOPIC) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + return errors + subprocess.run(["curl", "-X", "POST", "-H", "Content-Type:application/json", "--data", constants.CONNECT_SOURCE_CONNECTOR_CONFIG, constants.CONNECT_URL]) + consumer_config = { + "bootstrap.servers": "localhost:9092", + "group.id": "test-group", + 'auto.offset.reset': "earliest" + } + message = self.consume_message(constants.CONNECT_TEST_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + return errors + try: + self.assertIn('User', message.key().decode('ascii')) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + try: + self.assertIsNotNone(message.value()) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + print("Errors in Connect Test Flow:-", errors) + return errors + + def ssl_flow(self): + print("Running SSL flow tests") + errors = [] + producer_config = {"bootstrap.servers": "localhost:9093", + "security.protocol": "SSL", + "ssl.ca.location": constants.SSL_CA_LOCATION, + "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, + "ssl.key.location": constants.SSL_KEY_LOCATION, + "ssl.endpoint.identification.algorithm": "none", + "ssl.key.password": constants.SSL_KEY_PASSWORD, + 'client.id': socket.gethostname() + '2'} + + self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") + + consumer_config = { + "bootstrap.servers": "localhost:9093", + "group.id": "test-group-5", + 'auto.offset.reset': "earliest", + "security.protocol": "SSL", + "ssl.ca.location": constants.SSL_CA_LOCATION, + "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, + "ssl.key.location": constants.SSL_KEY_LOCATION, + "ssl.endpoint.identification.algorithm": "none", + "ssl.key.password": constants.SSL_KEY_PASSWORD + } + message = self.consume_message(constants.SSL_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.key(), b'key') + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.value(), b'message') + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + print("Errors in SSL Flow:-", errors) + return errors + + def broker_restart_flow(self): + print("Running broker restart tests") + errors = [] + try: + self.assertEqual(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC), constants.BROKER_RESTART_TEST_TOPIC) + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + return errors + + producer_config = {"bootstrap.servers": "localhost:9092", 'client.id': socket.gethostname()} + self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + + print("Stopping Image") + self.stopImage() + time.sleep(15) + + print("Resuming Image") + self.resumeImage() + time.sleep(15) + consumer_config = {"bootstrap.servers": "localhost:9092", 'group.id': 'test-group-1', 'auto.offset.reset': 'smallest'} + message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + return errors + try: + self.assertEqual(message.key(), b'key') + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.value(), b'message') + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + print("Errors in Broker Restart Flow:-", errors) + return errors + + def execute(self): + total_errors = [] + try: + total_errors.extend(self.schema_registry_flow()) + except Exception as e: + print("Schema registry error") + total_errors.append(str(e)) + try: + total_errors.extend(self.connect_flow()) + except Exception as e: + print("Connect flow error") + total_errors.append(str(e)) + try: + total_errors.extend(self.ssl_flow()) + except Exception as e: + print("SSL flow error") + total_errors.append(str(e)) + try: + total_errors.extend(self.broker_restart_flow()) + except Exception as e: + print("Broker restart flow error") + total_errors.append(str(e)) + + self.assertEqual(total_errors, []) + +class DockerSanityTestKraftMode(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/kraft/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/kraft/docker-compose.yml") + def test_bed(self): + self.execute() + +class DockerSanityTestZookeeper(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/zookeeper/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/zookeeper/docker-compose.yml") + def test_bed(self): + self.execute() + +class DockerSanityTestNative(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/native/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/native/docker-compose.yml") + def test_bed(self): + self.execute() + +if __name__ == "__main__": + if len(sys.argv) > 0: + DockerSanityTestCommon.IMAGE = sys.argv.pop() + test_classes_to_run = [DockerSanityTestKraftMode, DockerSanityTestZookeeper, DockerSanityTestNative] + loader = unittest.TestLoader() + suites_list = [] + for test_class in test_classes_to_run: + suite = loader.loadTestsFromTestCase(test_class) + suites_list.append(suite) + big_suite = unittest.TestSuite(suites_list) + outfile = open("report.html", "w") + runner = HTMLTestRunner.HTMLTestRunner( + stream=outfile, + title='Test Report', + description='This demonstrates the report output.' + ) + runner.run(big_suite) \ No newline at end of file diff --git a/docker/jvm/fixtures/input.txt b/docker/jvm/fixtures/input.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/fixtures/kraft/docker-compose.yml b/docker/jvm/fixtures/kraft/docker-compose.yml new file mode 100644 index 0000000000000..704280c290f0d --- /dev/null +++ b/docker/jvm/fixtures/kraft/docker-compose.yml @@ -0,0 +1,100 @@ +--- +version: '2' +services: + + broker: + image: {$IMAGE} + hostname: broker + container_name: broker + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + broker-ssl: + image: {$IMAGE} + hostname: broker-ssl + container_name: broker-ssl + ports: + - "9093:9093" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker-ssl:29093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl:29093" + KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER" + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/native/docker-compose.yml b/docker/jvm/fixtures/native/docker-compose.yml new file mode 100644 index 0000000000000..0d4a874ee712d --- /dev/null +++ b/docker/jvm/fixtures/native/docker-compose.yml @@ -0,0 +1,68 @@ +--- +version: '2' +services: + + broker: + image: kafka-poc:latest + hostname: broker + container_name: broker + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/output.txt b/docker/jvm/fixtures/output.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/fixtures/schema.avro b/docker/jvm/fixtures/schema.avro new file mode 100644 index 0000000000000..d85b0e1204773 --- /dev/null +++ b/docker/jvm/fixtures/schema.avro @@ -0,0 +1,8 @@ +{ + "type": "record", + "name": "Message", + "fields": [ + {"name": "key", "type": "string"}, + {"name": "value", "type": "string"} + ] +} \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file new file mode 100644 index 0000000000000..3a0c3c9ee9f72 --- /dev/null +++ b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file @@ -0,0 +1,17 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIICyzCCAbMCAQAwVjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH +EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMRMwEQYDVQQDEwpicm9rZXIt +c3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI ++RN1dlbWoFI94Tlj+1ReO62HrJZIO+UGDl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK +1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8z +tinteU+0wLu6lQXU2d0MHGY4CuBDp3VwtGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajr +PMZ8Mp4fpERdblrXFZNyUnycMOhchAoDMdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+ +55RCwD98uIs4C+ZNdUD91W2baXaYMXdUF7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2r +FnPYxwIDAQABoDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQUCe7i0TB0oEfd +DmuM4WWcWgCxV+8wDQYJKoZIhvcNAQELBQADggEBAHFcgQDrj7F0Oi3CannGvOB6 +XLTf6S5+f7fd9aIkq+cRIVV7aIacu8xXmTKyLgbuJMN/AhPqzZwt79jnIm54/mWh +mTBM3B9BRQT4GreJ2b1xgb543JB85LyCU2eMxx5UxOvUV/7VMxee2mRcWQUPw6Jo +0YCJqeNFZwsg80MzuQMOPA6wmGPNvgJ8LmcwMMfUnnaUlnvYL1cdw9n79Ddkuvm+ +8I63wrws9ejuO45i6o4uIL7sy9n2egwZ85oz/8hboUQgaOs+V8A2LE8xLnoLUHAV +p5pvjlB3alfhxRJEhKf4W16i0CXT3tMBl/v1o9o7NA/CllfZeb0ElboBfZA2GpI= +-----END NEW CERTIFICATE REQUEST----- diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed new file mode 100644 index 0000000000000..0a5ccf415e8f6 --- /dev/null +++ b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAikCCQDO815g0gGg1DANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO +TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE +CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA +LzAgFw0yMzEwMTIxNzM2MzBaGA8yMDUxMDIyNzE3MzYzMFowVjELMAkGA1UEBhMC +Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV +BAsTAk5OMRMwEQYDVQQDEwpicm9rZXItc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UG +Dl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8 +a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8ztinteU+0wLu6lQXU2d0MHGY4CuBDp3Vw +tGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajrPMZ8Mp4fpERdblrXFZNyUnycMOhchAoD +MdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+55RCwD98uIs4C+ZNdUD91W2baXaYMXdU +F7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2rFnPYxwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQCYERHx3CzqOixzxtWZSugqRFahFdxWSFiuTTIv/3JhSpLjiMZGQt2YqX85 +YLnSvW0luChw8IW5S0Mtkn/Mgpnt9hzPsr1vY1aQ5By8PUXIqCNMmnIY8yC+HYNs +7DzTbU5Lin/YwRzMLnqq/9zvh+YBVdPhBrFpSXRjEkjqTXfs4fzNm8m1MsJifz3f +Q4t0iPOPjrbXrq1CQ+MstcpMwTi3eHHxcvyNHHlFLs10GH74NIymYKYwDG8fsatl +jScfxkn2rLuMFWuo42QqdHsPgS7QyTrZjvCM+5w1aUo6a35NPEcOqFD311/PJh/Y +vlSocIMIFklDRWFVh1D946t2Z42/ +-----END CERTIFICATE----- diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..a823cf3b7ca0fd48292238f98cf99d893e6aa067 GIT binary patch literal 4750 zcma)AWmFUlv)*00yBnly7o@uz=_Le~mX4*HB^8j65Rhg;KoD3`LIeaU>5vXdK|s2% z_uPBl@7`bE_hZg8^E@;2^Eqb*j39jl0-}Ktq)#z%c;eLKE(w5Gz#;@GJ34}t$W+{8w(|JrC~R7cqg@3nOY(ktEN*K!K_n}PyfsR9A` z`;tZ#bH?Bq+(hL+Bay47(T0bCeZrZJwti&jUk;;w>`&UO4i2A^ANHX(2LEgckmZid zzU&v0aZz47Z2IhPveMZ_|4=*mCifI4DDg_tRO07t3VyOe5`TH;&QZ0gMUEkR>7cG~ zm4KeP%#s{kWU2J4MQof9d=hf<=Qjnm9NNj2Aoc}|lm2)e0CyfV+GHr8zBji;m#XKrTC7*4EkH0>#s~ zXilCT@C04lALcg`q4qAzFWxpZ-j*xIoxwEkDPTh9nF)e=a;wqJrZWri^*2j;$b$<5 zmHW^x;}r2wYeK4V(cNABGQ-SK*LL6OZCw>p$)Z#+T4|X4i zP210c(536xa;P+$NYRA6U~_T-BlQ*!m2{XBCvm5~8i|{h9x?4(Y@+Rq*@2i=r0EV8 z&RQM`S@umi9f~4O%lolqn=HNqQo9@b6GM~%4esT=rMQO|TQc@+NrDHPPyg5w-$w&{ zx3Jr3z6Lt(rFS%SsJzVN$Zhv#sy6f(_ z0ml+K(U&jP48n%I|eF;yOYCVS#$o}&yWjerzEx?&6` zag~U@6-S#CdDm7FY#t09nbhinARAXmvbwj%8?cvAJaJbs_GJ2>bS3i__J^XLof|U2 zo5b~X3Z%~jE<(5$fw=2Gc1JXVP-}WbLwSwACYe!@|JD>PIZ%t{aP2q>EsyL9Zbp9 z3}?m&=G5lawL3^rlfso-1tNT&4{_jMK@aNEtB8^C150h*swD#=ogazuK^9xa3K9us$K?D|fk$QIii~Jh+`5$kd0m-c1paNhplCX2y&S*`bUAj~?}s zu(+D4SQ$uF{e+-%t{g5(JUHI2+LfBP3r@&JH5Hv^EkWVXZhJ4@o zRuKMWtBtUM+tH{Sdl@F0YO1*!B{Vi>`Qu}YX4L|%Ae+=E`}FVmK=(&??qkI5CLRL<6kgoL&+cV5uu)sLrB2u_1WRBIbAJKMaW#aQ~8q_S)*#E zp>JddGevqstn6-pPFIS>J6~6Q+zqG)gQp_Jjl!F+tRGuDW|;O`y_$UsGX3>oaf8;4 zopRD&y2gs_L-#Ot;g5XK)ICLFeXLv&u#SM&bo4uuS?O4;HPp`jnXm@wN88zsp zo+1A3;!7fD;<;#|KT$3Acfh_4TChyn-dSGRrdg5yw`W1Gr)th!_bslWGu0%NfzN=I!5&ip(iAv4c?H{b0v-v`vJCoRHCi-yNToT+HTa`Hup!ADXzBt-OS zmc5IYxG99VxYXt<-2~ea$;=>NnJKIX&9SIsfP6b{L@NZ%ObcELm_qgwZVD^(goh}d zq@lIkyfT~x6p4me&stEUF9#;Jy7*7IV*Mx$~di= z6!p6aZ#EWCblm_*y5)`hr^QFw%1{Th0~JdBh-dSwS*(;Yw)P>qba3H!%o(+U=ZQ2i zl5jWm*X?mt`#W)+qj~~G1)qJC&YGWxQx@1bkV|v(l&~QWtqeAzQsiUvd3@hSYWOlI zSsVxw>3y!zWwNx%tn58aswz&Aor$~l4YS){LGZcwF3%jd%#VbL$gz;9d3S%KrDwH( zy*l}U7u?HkBURdE_T*#=lI6x?~_A^q1UPN{e6y_4H1 zTXLf96`MU4G!BIN18kq!6um`6Y#SwnEDr8Ud70HQpq7k}NFDFgto*6HuAuz+rXT0& z?D)}E_S$#c^vftIdn?hz7vLlQ%k=PKE7ees4+-Jv0Y5vl$MaWWwiGdNCnWLWE_o=4+i#Li9pK|}n)Unxq|lDe zrmb7$t}6*Hkww@=jgnCwdp@9GkGA44XjAvShQW#_2`$iMeuc-rmpM}!&?1B3W~xn=N~A?3!czwxm>T_cl~(e zB4~Ge{3l2Cou$4&uSFBTBeY`jII6O&iWS((2ycJn3n1!YxEU}A*xTRy?Qr018CrmE zIHjW(O7xqquuah0=|HZU)4voOY;yX0vT2bssUIO1Zf+tb2xCTxE#CK-`<&;IeVgtIoRSs*3kg`^J}E{+Yu(NWSDexvQMJr; ztFSp-9>d5cMs8FkwKGqp|LWCTtgJzlJD{r>^KfM5Pez5)(7P774h_hoJ=eE}>J`%n z(T5+51)tP@9+#L{@EQNYwIz92@)4-v<86U0d{OZGel01gEfBM5EcoVx16HvLp4X}tiScZKOF6_@%o?ADd@BN%PCo4HLvuN%4vL;McneKqY2olC)U7BM6ldn%(f-3)qP$10#&AH_IADQ8vTmSYK&`mu=zdGPPtEq-|^HEa}L zc*d)oCJyb)b+eP4R^1KW%H)RGKUtS5$UQ27FD98pxf6oOPi^D7KlD=SE%ez85+JC~ z;+OSeI;3pLEdHka zj?8Vt1ax-r0wj0A=|d4`ct{)2QJk^kv3K-Hv6ytYIN+tV-gnc_-q3CILOEZbLrh3B z;=^&!vXeFYBUh)Rk*tdxt%Gap#Va3umEbXEfZkP0p7KJWhXH_K%l>Wf7jr}EDFPC_ zF^mrjb4fK#xMw|hi227X&F$iC8A{O<_xKsn>i{^|Qn=LJsA|GG?@a17%Co-uH-RdY zVkBSNR?SzXT|`LerP2Z;qmaHRgYhj5sVH{3cWlJ#SJY#Zr4pA=p?gKXE%X@A| zXm+C+>#wAd_b1fR&N)aF$~X7<1C49)mhlOW1q}3!Qzz!G3Ov%Q;yYH&pSK#D+z)I# zm*`Ur+v;cAGP3%DgIN7TU(2H<8xX{R&sd|Jgz%me36s0GDHDrg6~-2ANTHr*>imT= zlKe`p4|8ZNyG-F9AXzL zq72pYB#=&68CBpdCveDWQSCUd73NsVTBEW6>wqC(f`5KAARq<+o!Rx-vx&AZ7mdTK zECEq&ubrRCwqIMx?+}kGE3ToLB^#rYd(20o+(x`q9_{K5ti_RK*Jja4|9w*Y7a0`6 A#Q*>R literal 0 HcmV?d00001 diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..b526dd149726d5b5f14d458b9855af077c197cd2 GIT binary patch literal 1238 zcmV;{1S$J4f&|h60Ru3C1bhYwDuzgg_YDCD0ic2eZ3Kb@X)uBWWiWySVFn2*hDe6@ z4FLxRpn?QaFoFb50s#Opf&@nf2`Yw2hW8Bt2LUi<1_>&LNQU+thDZTr0|Wso1Q6)P*k=lo4{ribAXc|spM-#d1JG88u`XIkMH^{>lLExi+PFW^ zB?pbblmwhZl95UB&R59%mp?wOpl^w#7q&J%nb0|CPS8hqe$+89sZf3p`|@?MQggrY z6-TNTHFq}tQ4)^G7zuG;HGzvqC@!I48tfp4qd)UU(Kj6NScU}N?a>JNhW1c+A13Kl zQ;Gkn-)StuoBuk`kEidVAYIfab!|_{$!9`==yeKm56pr&aUOF59w1g@6CtZ+DLoPF zue;~{vjWt-o$4Z?bJHvPRbqi_>=-n=5#sPVYd%jPe$41HQsXhU+Sz21vR+BYQO*og z{6yh#t00ubb!pV@S`;iS(~urt!&#{-R) zY62Zj_OkN8`>Va1GLM*izh!bI!4gaCL_1FLPyQ*?6v9jUuFUufcLBhlS}(A@IlKKu zxMC+TOY`5ZsY9tu+anTqy{8^YXu^z+V$7P}i;;7K^m9cb@c>Ck;2sj0qlWW5CJTMM z)X3$Z@7WeGAxL$Q2f6>!EJlv>3mN6Af6F_-2QmB=+FecQu=LTp8ZHfZKUQWYn&MqLMb6rJ=V3r9OK=UbbQmJ{-3l~6Cid$41Y%|!$t z>wLkiAQWfy;H)DG$1dv?2aHy|pZDjDvTa-LS`@X?N0$H7Ic%{;1PWaE^gcvROeZtW zg#_N~lac)VV99}H)q52!ikbFeIFi^-g`H$?$5hFDX6FYs(*p+)ZPdjrK^#l>yt8%| zP@5^P#;gUlV-@??hv}0MAKcMiwYbt`1+OL^`n`@aJZHb1n> zCeuxysJaF7)iW};l{)45V;DO9Wx(`wo;=YX!8E~;v4pWK4GjEI-@(LBYo_2SD9Nb7 z4BU-#b_sGTJarFflL<1_@w>NC9O71OfpC00bb=VYHg|wC=Y_ z>K$^fKz=nohNEb?2Wq)Xg7;r0_z`ae6bHMGdOoJ>5YY_$M+66>H8dkc51?leY6p)gJl@gF{L6Me@ z>pl0L_q+Gk_x+gj%skJ`{Cv)tfuhJGfmqm36!{|@e7;cS&~sueT&x@vITr{;&i0pP zf}+3;|4#xJf>7YxzjWr`NW~%i-zg#jEMN`_Ec=&~g4+Bu0VakTL&g795<&Svly0Zr zvpr2A6+;g#xs5AXJKP1FP(UEh10WVFln4j!e>MW~!2l=|4*u&8WP>$fB?tu?m+m8Nn=89iSLpKUz&~ij?5t76%_(NzbzwB%YQVh zwdy;plk(XZ3hlNz4nt;Ffg&GXnr+t^nE0v7Y#f=U+1}1)1q1}CzM>PRuk#fiBV{e# zhUag3xxA%cEFH`0@Hraj==T0WH_J)mNF*;%5$j4z*(`c}UrdgWaJ?&0DcC7#0P!HH zWRJ)3PB9dGi-Z{-HuMCfXDZ>;p2P{G?-jj*y^ZcszuU^ge=(1Ifsh(6y-XFPqnO9) z(@8fwqwgu-S6n|EZYaYl#{8oG`07aXeD<@~S=^cR4kdxWO4LK?$HgR`W~7Ez5tt<9 zM12j%2o3Mo7RL%t^!8f|3x_fUqpN}|9X_S696t4`o|;rXXv+++RH`y4q0j>}$XPPq zFv-rLcGg+fTN_UeONYgD%g<8Rt~_MY1CBZ^hIYO7-iZrQNEgdGr^@tDslHm~>h}y+ zS#o3vY(cML=JaS5)Uv4BoL05WMKe1>;R{@biNRR75h*7C>FX>$c!^6ma2&~hWm_$( z$q+Th#(1f`+2bpWEO=b7ZZ%9O=ppW8KCwn1!Pm#$5^d-SN{}tHu8r%%V;r!^U#*~p zG=0nP6*P4$)vJgHj*jgJldsVEx;sAne0_B)gMQK&ZUx{QTvhV=VC`grNT4kkdPesy z_GJannJ59tL2$m=#5@F7!dxQTrln60DJZz(j7J5ZY6ko5U?KVHRim*^+KUpj_5 zDf@B#W@M1~8Ct$jjp!zqXz;*Z?$34_$BLcoH+9`L+et>9G?mMl_=Mhc_s&-GeBeio zO)jO!y#ZaZi7lENy=mQt>7TEy37$my%$)nSaY0aQSj6b|6`^iuW7xb|N`nai#*urn z*PX#^zq4meWwd1>WD|^gom7RuB$EXG&h2G%waS30{O%5_U3+x>ZdCJdFuOllU%gd| zsG39SY%rDy5w(YW?wnwSRGR$B<8cJH17Q}X-&fOq&MS9R$9cisk0+3k)*T&JUs0FY zhlgA^<+-;OYh2+E=^$l(swe&Y&uuSxs`nkH>!eijTJ?|l_n|Bvc5gBKHDIjqWT&_= zTRtI22#!&df#WOPCx@90{aFl4Xqq?c2X(ixQNXRa5t5tb_Y~JC1^3w5^Zp{!#b}Cc z`2-s-FjEcDj6gFbgl;vvtb5d4z*(1}IIHY#fPek_+;w4!Ws<(EPg^Du=dYUaGos=`nap8z#sUQs z-vdMzFGC2P<~ZYsarx&SI1oIilKgTLA8FEi32xXt<~H3I!~3Pw$cH1Iry&yw z4t{bqEye?yeKpfLy<#k-`i*v*$rh@TD6c&9rm>#(j%WJ*q4_txsjP1M6=`(dB?b0} zS{vANDBC}FiBAcmWF`k70L}nQfcamv`UJUij?gZWaLOV1S z;r9JtZn4=%_vw9R#!W5sfhfZ9G77)pGTWI?eO#qj0#WJ%f|8Hv9X+_{Dc`D$zlb6+ zu@a6oiI6C$Nd)p=LK1tKaK)dxw(340C!{AFRG@<(u~$z3&P2WxMg?k;l&~?=d?U$V zUwFyplyZ9RqwlLL8#M!G{ey_K3!lhycg4n$4GpYBeiECdRggWkKQ8xjRf;PQG`*z_ zwkR$2aSn$P2YB*a;Lt9-rXx5RPDUuhiJtnpX=|-0A`h7`HSEU|Txm<9J4^QBf^&xY za-w?&j%KLrPkSkIQg{eO-CB^R+q5#b=hF!p>GpWyb3!}*Q=Vh+pB8wpxf7{Wu}W_C z@m45Gybfy932i_7$)nQEd|rSa>DY8qrfn<8N5qoqCwOM%YdrvqCE1>CU}22E#nq2| zoW=8qhu56F-efSeNE2*v^B2FdSicTah&U?uTAZkp`N<7@iV+wH@=uAJAEE1w)!bU0i zelT!Xz2LsP5LXL^o2(@`H)*A*@ny=5thuko7d5ZxWLhLTl!V_Dzf9H~P7n}9ZyaG{ zQzyjqlqmCs{xOkSE6mX6fJs+6OL>2RqE9ea@ny=LtDcxnkqj6RHg3m7coY${z9ypvVN^eu_(b=_R6-cq$BA7;PbmY7$EISZ{25J2$^_3J?F!4neEZNLsgR^YjFV4@hSzP8Ka z8#rKBHkPWsd8xJ_4a0x8O~>(P%)=;GWh`${dylKAxV0I=-4GpW$*glhKYgYs<^AkJ z&nD{&%C3`OAX5I<6K6onx#;E_iHB_iJRjHdn?vQRR?2!FFG>q67S(jUlZ=pQ&2{h} zp00_GHl@C;Y+YCZ&_1%BGBi4Nk<3;$7#c^Ue&KXY*shLoxCT z!u1-o65PoQIA>smfmX@Y)I%?m%_+I?5L9V00O9h;-quPg)3h+p78 zV)fje#nw_ts3a|1Dj38B7+LYb7_r}kc4F^H7(J~V3CT#RIGuhvCZsX8nY^L2YJNZ# zN%VXf%FRWrYn|E;_5O3OuvsIdP|P^ehfI2r-pS7LRr`QMvqJ!sY_4hBh&_=O)Ke6^ zR<#?l=pmsn-<8KkU}#;({l0-65fyh3Z`+#UpqQB*CE=r$(v5d1hP)ty4T_qfl>uLv zs$(jJEY{MZsvL$9NaDsuPKU#nr=wB`a}57I`a^)5`bO|TgmT0UN@eSaTg)`nvG>Va zgd-9u8Q^>QX0?QOoQJYVt_t?d!8a@yu{_)JgZC#xAC0{%+qmw=sjS=>KRY`sOJOnM`Z=zwM2vrm8u(#)X zFQT71^M|CCP*x)WyLqPs>;AhG|ETsWD{x~iEW3>`YmEB z2Nm3oG8XZN+m1JGf^35JhDxdL>}hH)a+KcbGS_GLGT@`X`LKj7^`JYgv_k~flfK}e zI8`b>DEri({^2m2RzR{{x2-)Vpm8mz1wAg0g#)iCT*f6TI=EGKT1ak7SN2@}u3p(0 zgVx|>T{9t7zk47I+cuOIw06Y@Xob*A+J3_4C@JoryeUAObGx@Sa0`jrkPg4B2FJ3l zAjZ@`RXIs=cp6(sKJoVN8)%rR2qQ>6EpC##m|$K8ZvDBy@eQ1(cOna!wjcf93|6@b@TisdL^kpAU&r?6!s zay_TZX;JH?z?!jaZ06hDn&%Ue@U0XELisIxhH^5SYqJl8w$~<*X%&nMZ8PXf&BeWoB_|&m&B$+&k#Jc2akQp^Q2My+HP-v?ro~s2 zEl*}inG}~qzW$n>VPi-axYbrFoWj7RPxgLLrk<>(Sa(i4Ui5IC1SJIK{p1m13BR2# z$2(wd&*C6)-sPYOC5>qs@f~dPiR9>yfgUpW&D^CgC!&cvQ6@zxk`EX3$z#ta_Qy@OjEUf!g^w_!gS!n79dosCH%gGeE%PA#`PzJRXWv}|SuPcW)Asq2wIumCa z_;z^M^;_cZqOc9#*Jf?GaFgVV!%eD+lb9_w2)?J9Xqp5fmNjdPxM@P)iG^lVZ?kjO zqo*gA4dFHR0%5(z1&!3}Y?evg$CTzcO z%Gz95Ig@UG1o@*@#NmXf!9HQH0KgMjH z;FC~FRbGg8pBY%u=3`k%&*xL;AvLL5qFLK3+)~l3Jq$Gb3VWOsb5*) znG`sC+SiP;$18c~D2^*;&1a0V$n?B@4&slq*#2VUB7}rfPuSoV`)Ba#z;0pzwgpZ^ zFtmS3DJfC#@uD~;-z3~L1`m63+9>()@bhitI%Tvzf#Y>maXjkeL6tD}q(HHN%*UXp z*{#ek4KA)_E?4r!s{v<*&rj%|Fk(MsAT}(;GN9MleA6Xv!1d6mrXz3NxuP)L6b=Du z?x5M@ao zn4;J`%P-xiY-`WXde5@&zGRcsoo0oz6@?)Yq3+?{ot_pq@^aM??=tp47moNoNQzC- zBcXeH$OI`k$lC93O<8#8dDQ735PDy3PDUN-DyN(gb?3+J#f4R3YQ}$&P}nA#T$?FJ zR=OuBbqJrx;hZkic$fwt(@#{Hs><#-2YhS@r89hKDf3+h)$-P=yEMZ>K#G8kzvt&w z{O&4Kys1{J`Y^6jsFW{u{cjG{Cdl>0%E&950Qopr-{ec6z7=Rx!($!#731jI37;f& z90+=yvy~oh^D%kineRO&LNQUCc5!rUdCMV@CO0{ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5jYAgfh1*n&#?ebGsJHYtFD1JGFzj_}5R)2_-ag-=_~ny9w}M8;}F zDKA=%k;NKXcVz33DvTJqV2-?frBKif~zW%qBuAh6lp?@njnlUH;-<%m@7iIV63WOQ8hiY zZ0#38umsN<)9g#{C(4<@`8M^hLu6q)QIoR{FrtvATf+#49?+MB|Lwmu!$nvjMyY-O zk@sgxe1d&w`Xi-O=77DD)!LT9{svF#!e`tue(yujIOBOHJYrxbEJsQ?D}_9FlI0iIeIJOSKK;Twz2W8 zg5S85RYijnsXD?@af8pubU+qrw+anAL~axgG*T$8RaCYFe^} z%o)5U$KOAGzYU@`QbY)*TCCIvw6-qb9bA`Ky))GpTgLi!G$bcLe?B+*#YEID~)LQ4I{G;o7 zs}A$G>35i-vq1j_why`uBe$NsrxA3JoZZnJ(CmE4qHc~6cnGUeJ zJfYcIc!#@{pT;)dzIp(F?mXWEC8sjoGb>h^s63!~lNkDWtWFg%C(w}Rk|BW@-pbTN zI9E~88JAM|@2N$p3GY;2whx-8Y!!E0q_b${Y-B;}=yAKF^&$#L;oY7dg9Js-jrx#) zDxUBOGN_{BkV1JYZIHpk@yS&ooA^_*k&MKF=r!=Tv-wCn0!$FupntmE;oN4cMe2L- z?hnOTGrapoa+p(Yhb`<-kR=^*4}^HUA;z}hT;BDVcnqlK*)%{B%E+9r`#esnNRFflL<1_@w>NC9O71OfpC00baQabg^ag;b " + echo " $0 [-k] server|client " + echo "" + echo " -k = Use keytool/Java Keystore, else standard SSL keys" + exit 1 +fi \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_keystore_creds b/docker/jvm/fixtures/secrets/kafka_keystore_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_keystore_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_ssl_key_creds b/docker/jvm/fixtures/secrets/kafka_ssl_key_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_ssl_key_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_truststore_creds b/docker/jvm/fixtures/secrets/kafka_truststore_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_truststore_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/sink_connector.json b/docker/jvm/fixtures/sink_connector.json new file mode 100644 index 0000000000000..f22ef5495f0af --- /dev/null +++ b/docker/jvm/fixtures/sink_connector.json @@ -0,0 +1,11 @@ +{ + "name": "kafka-sink", + "config": { + "connector.class": "org.apache.kafka.connect.file.FileStreamSinkConnector", + "tasks.max": "1", + "file": "/tmp/output.txt", + "topics": "test_topic_connect", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} \ No newline at end of file diff --git a/docker/jvm/fixtures/source_connector.json b/docker/jvm/fixtures/source_connector.json new file mode 100644 index 0000000000000..495bed2db8f1b --- /dev/null +++ b/docker/jvm/fixtures/source_connector.json @@ -0,0 +1,14 @@ +{ + "name": "datagen-users", + "config": { + "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", + "kafka.topic": "test_topic_connect", + "quickstart": "users", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "max.interval": 1000, + "iterations": 10, + "tasks.max": "1" + } +} \ No newline at end of file diff --git a/docker/jvm/fixtures/zookeeper/docker-compose.yml b/docker/jvm/fixtures/zookeeper/docker-compose.yml new file mode 100644 index 0000000000000..1057375bf7b2c --- /dev/null +++ b/docker/jvm/fixtures/zookeeper/docker-compose.yml @@ -0,0 +1,132 @@ +--- +version: '2' +services: + + zookeeper-1: + image: confluentinc/cp-zookeeper:latest + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + zookeeper-2: + image: confluentinc/cp-zookeeper:latest + ports: + - "2182:2182" + environment: + ZOOKEEPER_CLIENT_PORT: 2182 + ZOOKEEPER_SERVER_ID: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + broker: + image: {$IMAGE} + hostname: broker + container_name: broker + ports: + - "9092:9092" + - "29092:29092" + - "39092:39092" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:19092,EXTERNAL://localhost:9092,SSL://localhost:39092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT,SSL:SSL + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + depends_on: + - zookeeper-1 + - zookeeper-2 + + broker-ssl: + image: {$IMAGE} + hostname: broker-ssl + container_name: broker-ssl + ports: + - "29093:29093" + - "9093:9093" + - "39093:39093" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + depends_on: + - zookeeper-1 + - zookeeper-2 + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - zookeeper-1 + - zookeeper-2 + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - zookeeper-1 + - zookeeper-2 + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/include/etc/kafka/docker/configure b/docker/jvm/include/etc/kafka/docker/configure new file mode 100755 index 0000000000000..059d03c0819ab --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/configure @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +# --- for broker + +# If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode + +# unset KAFKA_ADVERTISED_LISTENERS from ENV in KRaft mode when running as controller only +if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] +then + echo "Running in KRaft mode..." + ub ensure CLUSTER_ID + if [[ $KAFKA_PROCESS_ROLES == "controller" ]] + then + if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] + then + echo "KAFKA_ADVERTISED_LISTENERS is not supported on a KRaft controller." + exit 1 + else + unset KAFKA_ADVERTISED_LISTENERS + fi + else + ub ensure KAFKA_ADVERTISED_LISTENERS + fi +else + echo "Running in Zookeeper mode..." + ub ensure KAFKA_ZOOKEEPER_CONNECT + ub ensure KAFKA_ADVERTISED_LISTENERS +fi + +# By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing +# hosts with 0.0.0.0. This is good default as it ensures that the broker +# process listens on all ports. +if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) +then + export KAFKA_LISTENERS + KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') +fi + +ub path /etc/kafka/ writable + +if [[ -z "${KAFKA_LOG_DIRS-}" ]] +then + export KAFKA_LOG_DIRS + KAFKA_LOG_DIRS="/var/lib/kafka/data" +fi + +# advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. +if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] +then + echo "advertised.port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_ADVERTISED_HOST-}" ]] +then + echo "advertised.host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_HOST-}" ]] +then + echo "host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_PORT-}" ]] +then + echo "port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +# Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. +if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] +then + echo "SSL is enabled." + + ub ensure KAFKA_SSL_KEYSTORE_FILENAME + export KAFKA_SSL_KEYSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_FILENAME" + ub path "$KAFKA_SSL_KEYSTORE_LOCATION" existence + + ub ensure KAFKA_SSL_KEY_CREDENTIALS + KAFKA_SSL_KEY_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEY_CREDENTIALS" + ub path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_KEY_PASSWORD + KAFKA_SSL_KEY_PASSWORD=$(cat "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION") + + ub ensure KAFKA_SSL_KEYSTORE_CREDENTIALS + KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_CREDENTIALS" + ub path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_KEYSTORE_PASSWORD + KAFKA_SSL_KEYSTORE_PASSWORD=$(cat "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION") + + if [[ -n "${KAFKA_SSL_CLIENT_AUTH-}" ]] && ( [[ $KAFKA_SSL_CLIENT_AUTH == *"required"* ]] || [[ $KAFKA_SSL_CLIENT_AUTH == *"requested"* ]] ) + then + ub ensure KAFKA_SSL_TRUSTSTORE_FILENAME + export KAFKA_SSL_TRUSTSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_FILENAME" + ub path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence + + ub ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS + KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_CREDENTIALS" + ub path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_TRUSTSTORE_PASSWORD + KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") + fi + +fi + +# Set if KAFKA_ADVERTISED_LISTENERS has SASL_PLAINTEXT:// or SASL_SSL:// endpoints. +if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS =~ .*SASL_.*://.* ]] +then + echo "SASL" is enabled. + + ub ensure KAFKA_OPTS + + if [[ ! $KAFKA_OPTS == *"java.security.auth.login.config"* ]] + then + echo "KAFKA_OPTS should contain 'java.security.auth.login.config' property." + fi +fi + +if [[ -n "${KAFKA_JMX_OPTS-}" ]] +then + if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"* ]] + then + echo "KAFKA_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally." + fi +fi + + +# --- for broker +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/tools-log4j.properties diff --git a/docker/jvm/include/etc/kafka/docker/configureDefaults b/docker/jvm/include/etc/kafka/docker/configureDefaults new file mode 100755 index 0000000000000..6868f14d220a1 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/configureDefaults @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +declare -A env_defaults +env_defaults=( +# Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" +# See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh + ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" + ["KAFKA_NODE_ID"]=1 + ["KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"]="CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" + ["KAFKA_LISTENERS"]="PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092" + ["KAFKA_ADVERTISED_LISTENERS"]="PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092" + ["KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"]=1 + ["KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS"]=0 + ["KAFKA_TRANSACTION_STATE_LOG_MIN_ISR"]=1 + ["KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR"]=1 + ["KAFKA_PROCESS_ROLES"]="broker,controller" + ["KAFKA_CONTROLLER_QUORUM_VOTERS"]="1@localhost:29093" + ["KAFKA_INTER_BROKER_LISTENER_NAME"]="PLAINTEXT" + ["KAFKA_CONTROLLER_LISTENER_NAMES"]="CONTROLLER" + ["KAFKA_LOG_DIRS"]="/tmp/kraft-combined-logs" +) + +for key in "${!env_defaults[@]}"; do + if [[ -z "${!key:-}" ]]; then + echo ${key} not set. Setting it to default value: \"${env_defaults[$key]}\" + export "$key"="${env_defaults[$key]}" + fi +done diff --git a/docker/jvm/include/etc/kafka/docker/ensure b/docker/jvm/include/etc/kafka/docker/ensure new file mode 100755 index 0000000000000..a50f7812ca863 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/ensure @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +export KAFKA_DATA_DIRS=${KAFKA_DATA_DIRS:-"/var/lib/kafka/data"} +echo "===> Check if $KAFKA_DATA_DIRS is writable ..." +ub path "$KAFKA_DATA_DIRS" writable + +# KRaft required step: Format the storage directory with provided cluster ID unless it already exists. +if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] +then + echo "===> Using provided cluster id $CLUSTER_ID ..." + + # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this + result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /etc/kafka/kafka.properties 2>&1) || \ + echo $result | grep -i "already formatted" || \ + { echo $result && (exit 1) } +fi diff --git a/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template new file mode 100644 index 0000000000000..3a7b4744e34c4 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template @@ -0,0 +1,11 @@ +log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} +{{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} +{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +log4j.logger.{{ $k }}={{ $v -}} +{{ end }} diff --git a/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json new file mode 100644 index 0000000000000..0e10d92d6473e --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json @@ -0,0 +1,25 @@ +{ + "prefixes": { + "KAFKA": false, + "CONFLUENT": true + }, + "renamed": { + "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET": "zookeeper.clientCnxnSocket" + }, + "excludes": [ + "KAFKA_VERSION", + "KAFKA_HEAP_OPT", + "KAFKA_LOG4J_OPTS", + "KAFKA_OPTS", + "KAFKA_JMX_OPTS", + "KAFKA_JVM_PERFORMANCE_OPTS", + "KAFKA_GC_LOG_OPTS", + "KAFKA_LOG4J_ROOT_LOGLEVEL", + "KAFKA_LOG4J_LOGGERS", + "KAFKA_TOOLS_LOG4J_LOGLEVEL", + "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET" + ], + "defaults": { + }, + "excludeWithPrefix": "" +} diff --git a/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template new file mode 100644 index 0000000000000..c2df5bcf064a4 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template @@ -0,0 +1,6 @@ +log4j.rootLogger={{ getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN" }}, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err diff --git a/docker/jvm/include/etc/kafka/docker/launch b/docker/jvm/include/etc/kafka/docker/launch new file mode 100755 index 0000000000000..b29294011c780 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/launch @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + + +# Start kafka broker +# TODO: REMOVE THIS +echo "$(date +"%H:%M:%S.%3N") ===> Launching kafka ... " +/opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties & # your first application +P1=$! # capture PID of the process + +# Wait for process to exit +wait -n $P1 +# Exit with status of process that exited first +exit $? diff --git a/docker/jvm/include/etc/kafka/docker/run b/docker/jvm/include/etc/kafka/docker/run new file mode 100755 index 0000000000000..d32e8e59f4441 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/run @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +#TODO: REMOVE THIS +echo $(date +"%H:%M:%S.%3N") + +# Set environment values if they exist as arguments +if [ $# -ne 0 ]; then + echo "===> Overriding env params with args ..." + for var in "$@" + do + export "$var" + done +fi + +echo "===> User" +id + +if [[ -z "${KAFKA_ZOOKEEPER_CONNECT}" ]] +then +echo "===> Setting default values of environment variables if not already set." +. /etc/kafka/docker/configureDefaults +fi + +echo "===> Configuring ..." +/etc/kafka/docker/configure + +echo "===> Running preflight checks ... " +/etc/kafka/docker/ensure + +echo "===> Launching ... " +exec /etc/kafka/docker/launch diff --git a/docker/jvm/requirements.txt b/docker/jvm/requirements.txt new file mode 100644 index 0000000000000..011440694c092 --- /dev/null +++ b/docker/jvm/requirements.txt @@ -0,0 +1,6 @@ +confluent_kafka +urllib3 +requests +fastavro +jsonschema +HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/jvm/ub/go.mod b/docker/jvm/ub/go.mod new file mode 100644 index 0000000000000..1723614f1826f --- /dev/null +++ b/docker/jvm/ub/go.mod @@ -0,0 +1,15 @@ +//module base-lite +module ub + +go 1.19 + +require ( + github.com/spf13/cobra v1.7.0 + golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c + golang.org/x/sys v0.7.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/docker/jvm/ub/go.sum b/docker/jvm/ub/go.sum new file mode 100644 index 0000000000000..5f20e19b6de1e --- /dev/null +++ b/docker/jvm/ub/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c h1:HDdYQYKOkvJT/Plb5HwJJywTVyUnIctjQm6XSnZ/0CY= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docker/jvm/ub/testResources/sampleFile b/docker/jvm/ub/testResources/sampleFile new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/ub/testResources/sampleFile2 b/docker/jvm/ub/testResources/sampleFile2 new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/ub/testResources/sampleLog4j.template b/docker/jvm/ub/testResources/sampleLog4j.template new file mode 100644 index 0000000000000..3aace55b81aba --- /dev/null +++ b/docker/jvm/ub/testResources/sampleLog4j.template @@ -0,0 +1,11 @@ +log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} +{{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} +{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +log4j.logger.{{ $k }}={{ $v -}} +{{ end }} \ No newline at end of file diff --git a/docker/jvm/ub/ub.go b/docker/jvm/ub/ub.go new file mode 100644 index 0000000000000..47dec76966475 --- /dev/null +++ b/docker/jvm/ub/ub.go @@ -0,0 +1,496 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "regexp" + "sort" + "strconv" + "strings" + "text/template" + "time" + + pt "path" + + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "golang.org/x/sys/unix" +) + +type ConfigSpec struct { + Prefixes map[string]bool `json:"prefixes"` + Excludes []string `json:"excludes"` + Renamed map[string]string `json:"renamed"` + Defaults map[string]string `json:"defaults"` + ExcludeWithPrefix string `json:"excludeWithPrefix"` +} + +var ( + bootstrapServers string + configFile string + zookeeperConnect string + security string + + re = regexp.MustCompile("[^_]_[^_]") + + ensureCmd = &cobra.Command{ + Use: "ensure ", + Short: "checks if environment variable is set or not", + Args: cobra.ExactArgs(1), + RunE: runEnsureCmd, + } + + pathCmd = &cobra.Command{ + Use: "path ", + Short: "checks if an operation is permitted on a file", + Args: cobra.ExactArgs(2), + RunE: runPathCmd, + } + + renderTemplateCmd = &cobra.Command{ + Use: "render-template ", + Short: "renders template to stdout", + Args: cobra.ExactArgs(1), + RunE: runRenderTemplateCmd, + } + + renderPropertiesCmd = &cobra.Command{ + Use: "render-properties ", + Short: "creates and renders properties to stdout using the json config spec.", + Args: cobra.ExactArgs(1), + RunE: runRenderPropertiesCmd, + } + + waitCmd = &cobra.Command{ + Use: "wait ", + Short: "waits for a service to start listening on a port", + Args: cobra.ExactArgs(3), + RunE: runWaitCmd, + } + + httpReadyCmd = &cobra.Command{ + Use: "http-ready ", + Short: "waits for an HTTP/HTTPS URL to be retrievable", + Args: cobra.ExactArgs(2), + RunE: runHttpReadyCmd, + } + + kafkaReadyCmd = &cobra.Command{ + Use: "kafka-ready ", + Short: "checks if kafka brokers are up and running", + Args: cobra.ExactArgs(2), + RunE: runKafkaReadyCmd, + } +) + +func ensure(envVar string) bool { + _, found := os.LookupEnv(envVar) + return found +} + +func path(filePath string, operation string) (bool, error) { + switch operation { + + case "readable": + err := unix.Access(filePath, unix.R_OK) + if err != nil { + return false, err + } + return true, nil + case "executable": + info, err := os.Stat(filePath) + if err != nil { + err = fmt.Errorf("error checking executable status of file %q: %w", filePath, err) + return false, err + } + return info.Mode()&0111 != 0, nil //check whether file is executable by anyone, use 0100 to check for execution rights for owner + case "existence": + if _, err := os.Stat(filePath); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil + case "writable": + err := unix.Access(filePath, unix.W_OK) + if err != nil { + return false, err + } + return true, nil + default: + err := fmt.Errorf("unknown operation %q", operation) + return false, err + } +} + +func renderTemplate(templateFilePath string) error { + funcs := template.FuncMap{ + "getEnv": getEnvOrDefault, + "splitToMapDefaults": splitToMapDefaults, + } + t, err := template.New(pt.Base(templateFilePath)).Funcs(funcs).ParseFiles(templateFilePath) + if err != nil { + err = fmt.Errorf("error %q: %w", templateFilePath, err) + return err + } + return buildTemplate(os.Stdout, *t) +} + +func buildTemplate(writer io.Writer, template template.Template) error { + err := template.Execute(writer, GetEnvironment()) + if err != nil { + err = fmt.Errorf("error building template file : %w", err) + return err + } + return nil +} + +func renderConfig(writer io.Writer, configSpec ConfigSpec) error { + return writeConfig(writer, buildProperties(configSpec, GetEnvironment())) +} + +// ConvertKey Converts an environment variable name to a property-name according to the following rules: +// - a single underscore (_) is replaced with a . +// - a double underscore (__) is replaced with a single underscore +// - a triple underscore (___) is replaced with a dash +// Moreover, the whole string is converted to lower-case. +// The behavior of sequences of four or more underscores is undefined. +func ConvertKey(key string) string { + singleReplaced := re.ReplaceAllStringFunc(key, replaceUnderscores) + singleTripleReplaced := strings.ReplaceAll(singleReplaced, "___", "-") + return strings.ToLower(strings.ReplaceAll(singleTripleReplaced, "__", "_")) +} + +// replaceUnderscores replaces every underscore '_' by a dot '.' +func replaceUnderscores(s string) string { + return strings.ReplaceAll(s, "_", ".") +} + +// ListToMap splits each and entry of the kvList argument at '=' into a key/value pair and returns a map of all the k/v pair thus obtained. +// this method will only consider values in the list formatted as key=value +func ListToMap(kvList []string) map[string]string { + m := make(map[string]string, len(kvList)) + for _, l := range kvList { + parts := strings.Split(l, "=") + if len(parts) == 2 { + m[parts[0]] = parts[1] + } + } + return m +} + +func splitToMapDefaults(separator string, defaultValues string, value string) map[string]string { + values := KvStringToMap(defaultValues, separator) + for k, v := range KvStringToMap(value, separator) { + values[k] = v + } + return values +} + +func KvStringToMap(kvString string, sep string) map[string]string { + return ListToMap(strings.Split(kvString, sep)) +} + +// GetEnvironment returns the current environment as a map. +func GetEnvironment() map[string]string { + return ListToMap(os.Environ()) +} + +// buildProperties creates a map suitable to be output as Java properties from a ConfigSpec and a map representing an environment. +func buildProperties(spec ConfigSpec, environment map[string]string) map[string]string { + config := make(map[string]string) + for key, value := range spec.Defaults { + config[key] = value + } + + for envKey, envValue := range environment { + if newKey, found := spec.Renamed[envKey]; found { + config[newKey] = envValue + } else { + if !slices.Contains(spec.Excludes, envKey) && !(len(spec.ExcludeWithPrefix) > 0 && strings.HasPrefix(envKey, spec.ExcludeWithPrefix)) { + for prefix, keep := range spec.Prefixes { + if strings.HasPrefix(envKey, prefix) { + var effectiveKey string + if keep { + effectiveKey = envKey + } else { + effectiveKey = envKey[len(prefix)+1:] + } + config[ConvertKey(effectiveKey)] = envValue + } + } + } + } + } + return config +} + +func writeConfig(writer io.Writer, config map[string]string) error { + // Go randomizes iterations over map by design. We sort properties by name to ease debugging: + sortedNames := make([]string, 0, len(config)) + for name := range config { + sortedNames = append(sortedNames, name) + } + sort.Strings(sortedNames) + for _, n := range sortedNames { + _, err := fmt.Fprintf(writer, "%s=%s\n", n, config[n]) + if err != nil { + err = fmt.Errorf("error printing configs: %w", err) + return err + } + } + return nil +} + +func loadConfigSpec(path string) (ConfigSpec, error) { + var spec ConfigSpec + bytes, err := os.ReadFile(path) + if err != nil { + err = fmt.Errorf("error reading from json file %q : %w", path, err) + return spec, err + } + + errParse := json.Unmarshal(bytes, &spec) + if errParse != nil { + err = fmt.Errorf("error parsing json file %q : %w", path, errParse) + return spec, err + } + return spec, nil +} + +func invokeJavaCommand(className string, jvmOpts string, args []string) bool { + classPath := getEnvOrDefault("UB_CLASSPATH", "/usr/share/java/cp-base-lite/*") + + opts := []string{} + if jvmOpts != "" { + opts = append(opts, jvmOpts) + } + opts = append(opts, "-cp", classPath, className) + cmd := exec.Command("java", append(opts[:], args...)...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return exitError.ExitCode() == 0 + } + return false + } + return true +} + +func getEnvOrDefault(envVar string, defaultValue string) string { + val := os.Getenv(envVar) + if len(val) == 0 { + return defaultValue + } + return val +} + +func checkKafkaReady(minNumBroker string, timeout string, bootstrapServers string, zookeeperConnect string, configFile string, security string) bool { + + opts := []string{minNumBroker, timeout + "000"} + if bootstrapServers != "" { + opts = append(opts, "-b", bootstrapServers) + } + if zookeeperConnect != "" { + opts = append(opts, "-z", zookeeperConnect) + } + if configFile != "" { + opts = append(opts, "-c", configFile) + } + if security != "" { + opts = append(opts, "-s", security) + } + jvmOpts := os.Getenv("KAFKA_OPTS") + return invokeJavaCommand("io.confluent.admin.utils.cli.KafkaReadyCommand", jvmOpts, opts) +} + +func waitForServer(host string, port int, timeout time.Duration) bool { + address := fmt.Sprintf("%s:%d", host, port) + startTime := time.Now() + connectTimeout := 5 * time.Second + + for { + conn, err := net.DialTimeout("tcp", address, connectTimeout) + if err == nil { + _ = conn.Close() + return true + } + if time.Since(startTime) >= timeout { + return false + } + time.Sleep(1 * time.Second) + } +} + +func waitForHttp(URL string, timeout time.Duration) error { + parsedURL, err := url.Parse(URL) + if err != nil { + return fmt.Errorf("error in parsing url %q: %w", URL, err) + } + + host := parsedURL.Hostname() + portStr := parsedURL.Port() + + if len(host) == 0 { + host = "localhost" + } + + if len(portStr) == 0 { + switch parsedURL.Scheme { + case "http": + portStr = "80" + case "https": + portStr = "443" + default: + return fmt.Errorf("no port specified and cannot infer port based on protocol (only http(s) supported)") + } + } + port, err := strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("error in parsing port %q: %w", portStr, err) + } + + if !waitForServer(host, port, timeout) { + return fmt.Errorf("service is unreachable on host = %q, port = %q", host, portStr) + } + + httpClient := &http.Client{ + Timeout: timeout * time.Second, + } + resp, err := httpClient.Get(URL) + if err != nil { + return fmt.Errorf("error retrieving url") + } + statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 + if !statusOK { + return fmt.Errorf("unexpected response for %q with code %d", URL, resp.StatusCode) + } + return nil +} + +func runEnsureCmd(_ *cobra.Command, args []string) error { + success := ensure(args[0]) + if !success { + err := fmt.Errorf("environment variable %q is not set", args[0]) + return err + } + return nil +} + +func runPathCmd(_ *cobra.Command, args []string) error { + success, err := path(args[0], args[1]) + if err != nil { + err = fmt.Errorf("error in checking operation %q on file %q: %w", args[1], args[0], err) + return err + } + if !success { + err = fmt.Errorf("operation %q on file %q is unsuccessful", args[1], args[0]) + return err + } + return nil +} + +func runRenderTemplateCmd(_ *cobra.Command, args []string) error { + err := renderTemplate(args[0]) + if err != nil { + err = fmt.Errorf("error in rendering template %q: %w", args[0], err) + return err + } + return nil +} + +func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { + configSpec, err := loadConfigSpec(args[0]) + if err != nil { + err = fmt.Errorf("error in loading config from file %q: %w", args[0], err) + return err + } + err = renderConfig(os.Stdout, configSpec) + if err != nil { + err = fmt.Errorf("error in building properties from file %q: %w", args[0], err) + return err + } + return nil +} + +func runWaitCmd(_ *cobra.Command, args []string) error { + port, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("error in parsing port %q: %w", args[1], err) + } + + secs, err := strconv.Atoi(args[2]) + if err != nil { + return fmt.Errorf("error in parsing timeout seconds %q: %w", args[2], err) + } + timeout := time.Duration(secs) * time.Second + + success := waitForServer(args[0], port, timeout) + if !success { + return fmt.Errorf("service is unreachable for host %q and port %q", args[0], args[1]) + } + return nil +} + +func runHttpReadyCmd(_ *cobra.Command, args []string) error { + secs, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("error in parsing timeout seconds %q: %w", args[1], err) + } + timeout := time.Duration(secs) * time.Second + + success := waitForHttp(args[0], timeout) + if success != nil { + return fmt.Errorf("error in http-ready check for url %q: %w", args[0], success) + } + return nil +} + +func runKafkaReadyCmd(_ *cobra.Command, args []string) error { + success := checkKafkaReady(args[0], args[1], bootstrapServers, zookeeperConnect, configFile, security) + if !success { + err := fmt.Errorf("kafka-ready check failed") + return err + } + return nil +} + +func main() { + rootCmd := &cobra.Command{ + Use: "ub", + Short: "utility commands for cp docker images", + Run: func(cmd *cobra.Command, args []string) {}, + } + + kafkaReadyCmd.PersistentFlags().StringVarP(&bootstrapServers, "bootstrap-servers", "b", "", "comma-separated list of kafka brokers") + kafkaReadyCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "path to the config file") + kafkaReadyCmd.PersistentFlags().StringVarP(&zookeeperConnect, "zookeeper-connect", "z", "", "zookeeper connect string") + kafkaReadyCmd.PersistentFlags().StringVarP(&security, "security", "s", "", "security protocol to use when multiple listeners are enabled.") + + rootCmd.AddCommand(pathCmd) + rootCmd.AddCommand(ensureCmd) + rootCmd.AddCommand(renderTemplateCmd) + rootCmd.AddCommand(renderPropertiesCmd) + rootCmd.AddCommand(waitCmd) + rootCmd.AddCommand(httpReadyCmd) + rootCmd.AddCommand(kafkaReadyCmd) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if err := rootCmd.ExecuteContext(ctx); err != nil { + fmt.Fprintf(os.Stderr, "error in executing the command: %s", err) + os.Exit(1) + } +} diff --git a/docker/jvm/ub/ub_test.go b/docker/jvm/ub/ub_test.go new file mode 100644 index 0000000000000..8ec46258597cb --- /dev/null +++ b/docker/jvm/ub/ub_test.go @@ -0,0 +1,446 @@ +package main + +import ( + "net" + "net/http" + "net/http/httptest" + "os" + "reflect" + "testing" + "time" +) + +func assertEqual(a string, b string, t *testing.T) { + if a != b { + t.Error(a + " != " + b) + } +} + +func Test_ensure(t *testing.T) { + type args struct { + envVar string + } + err := os.Setenv("ENV_VAR", "value") + if err != nil { + t.Fatal("Unable to set ENV_VAR for the test") + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "should exist", + args: args{ + envVar: "ENV_VAR", + }, + want: true, + }, + { + name: "should not exist", + args: args{ + envVar: "RANDOM_ENV_VAR", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ensure(tt.args.envVar); got != tt.want { + t.Errorf("ensure() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_path(t *testing.T) { + type args struct { + filePath string + operation string + } + const ( + sampleFile = "testResources/sampleFile" + sampleFile2 = "testResources/sampleFile2" + fileDoesNotExist = "testResources/sampleFile3" + ) + err := os.Chmod(sampleFile, 0777) + if err != nil { + t.Error("Unable to set permissions for the file") + } + err = os.Chmod(sampleFile2, 0000) + if err != nil { + t.Error("Unable to set permissions for the file") + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "file readable", + args: args{filePath: sampleFile, + operation: "readable"}, + want: true, + wantErr: false, + }, + { + name: "file writable", + args: args{filePath: sampleFile, + operation: "writable"}, + want: true, + wantErr: false, + }, + { + name: "file executable", + args: args{filePath: sampleFile, + operation: "executable"}, + want: true, + wantErr: false, + }, + { + name: "file existence", + args: args{filePath: sampleFile, + operation: "existence"}, + want: true, + wantErr: false, + }, + { + name: "file not readable", + args: args{filePath: sampleFile2, + operation: "readable"}, + want: false, + wantErr: true, + }, + { + name: "file not writable", + args: args{filePath: sampleFile2, + operation: "writable"}, + want: false, + wantErr: true, + }, + { + name: "file not executable", + args: args{filePath: sampleFile2, + operation: "executable"}, + want: false, + wantErr: false, + }, + { + name: "file does not exist", + args: args{filePath: fileDoesNotExist, + operation: "existence"}, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := path(tt.args.filePath, tt.args.operation) + if (err != nil) != tt.wantErr { + t.Errorf("path() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("path() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_renderTemplate(t *testing.T) { + type args struct { + templateFilePath string + } + const ( + fileExistsAndRenderable = "testResources/sampleLog4j.template" + fileDoesNotExist = "testResources/RandomFileName" + ) + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "render template success", + args: args{templateFilePath: fileExistsAndRenderable}, + wantErr: false, + }, + { + name: "render template failure ", + args: args{templateFilePath: fileDoesNotExist}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := renderTemplate(tt.args.templateFilePath); (err != nil) != tt.wantErr { + t.Errorf("renderTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} +func Test_convertKey(t *testing.T) { + type args struct { + key string + } + tests := []struct { + name string + args args + wantString string + }{ + { + name: "Capitals", + args: args{key: "KEY"}, + wantString: "key", + }, + { + name: "Capitals with underscore", + args: args{key: "KEY_FOO"}, + wantString: "key.foo", + }, + { + name: "Capitals with double underscore", + args: args{key: "KEY__UNDERSCORE"}, + wantString: "key_underscore", + }, + { + name: "Capitals with double and single underscore", + args: args{key: "KEY_WITH__UNDERSCORE_AND__MORE"}, + wantString: "key.with_underscore.and_more", + }, + { + name: "Capitals with triple underscore", + args: args{key: "KEY___DASH"}, + wantString: "key-dash", + }, + { + name: "capitals with double,triple and single underscore", + args: args{key: "KEY_WITH___DASH_AND___MORE__UNDERSCORE"}, + wantString: "key.with-dash.and-more_underscore", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := ConvertKey(tt.args.key); result != tt.wantString { + t.Errorf("ConvertKey() result = %v, wantStr %v", result, tt.wantString) + } + }) + } +} + +func Test_buildProperties(t *testing.T) { + type args struct { + spec ConfigSpec + environment map[string]string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "only defaults", + args: args{ + spec: ConfigSpec{ + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "CONFLUENT_METRICS": "metricsValue", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value"}, + }, + { + name: "server properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "CONFLUENT_METRICS": "metricsValue", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "localhost:9092", "confluent.metrics": "metricsValue", "default.property.key": "default.property.value"}, + }, + { + name: "kafka properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "KAFKA_FOO": "foo", + "KAFKA_FOO_BAR": "bar", + "KAFKA_IGNORED": "ignored", + "KAFKA_WITH__UNDERSCORE": "with underscore", + "KAFKA_WITH__UNDERSCORE_AND_MORE": "with underscore and more", + "KAFKA_WITH___DASH": "with dash", + "KAFKA_WITH___DASH_AND_MORE": "with dash and more", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value", "foo": "foo", "foo.bar": "bar", "with-dash": "with dash", "with-dash.and.more": "with dash and more", "with_underscore": "with underscore", "with_underscore.and.more": "with underscore and more"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildProperties(tt.args.spec, tt.args.environment); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildProperties() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_splitToMapDefaults(t *testing.T) { + type args struct { + separator string + defaultValues string + value string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "split to default", + args: args{ + separator: ",", + defaultValues: "kafka=INFO,kafka.producer.async.DefaultEventHandler=DEBUG,state.change.logger=TRACE", + value: "kafka.producer.async.DefaultEventHandler=ERROR,kafka.request.logger=WARN", + }, + want: map[string]string{"kafka": "INFO", "kafka.producer.async.DefaultEventHandler": "ERROR", "kafka.request.logger": "WARN", "state.change.logger": "TRACE"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := splitToMapDefaults(tt.args.separator, tt.args.defaultValues, tt.args.value); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitToMapDefaults() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_waitForServer(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockServer.Close() + port := mockServer.Listener.Addr().(*net.TCPAddr).Port + + type args struct { + host string + port int + timeout time.Duration + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "invalid server address", + args: args{ + host: "localhost", + port: port + 1, + timeout: time.Duration(5) * time.Second, + }, + want: false, + }, + { + name: "valid server address", + args: args{ + host: "localhost", + port: port, + timeout: time.Duration(5) * time.Second, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := waitForServer(tt.args.host, tt.args.port, tt.args.timeout); !reflect.DeepEqual(got, tt.want) { + t.Errorf("waitForServer() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_waitForHttp(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/names" { + w.WriteHeader(http.StatusOK) + } else { + http.NotFound(w, r) + } + })) + defer mockServer.Close() + + serverURL := mockServer.URL + + type args struct { + URL string + timeout time.Duration + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "valid server address, valid url", + args: args{ + URL: serverURL + "/names", + timeout: time.Duration(5) * time.Second, + }, + wantErr: false, + }, + { + name: "valid server address, invalid url", + args: args{ + URL: serverURL, + timeout: time.Duration(5) * time.Second, + }, + wantErr: true, + }, + { + name: "invalid server address", + args: args{ + URL: "http://invalidAddress:50111/names", + timeout: time.Duration(5) * time.Second, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := waitForHttp(tt.args.URL, tt.args.timeout); (err != nil) != tt.wantErr { + t.Errorf("waitForHttp() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 7e7f8e0bdc4f27ab2f8f8fa14e1b01a11e1cb5ae Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 12:37:50 +0530 Subject: [PATCH 02/46] Refactor build test and setup proper directory structure --- docker/docker_build_test.py | 37 ++++ docker/jvm/Dockerfile | 14 +- docker/jvm/docker_release.py | 15 -- docker/jvm/fixtures/native/docker-compose.yml | 68 -------- .../fixtures/secrets/kafka-generate-ssl.sh | 164 ------------------ docker/jvm/fixtures/sink_connector.json | 11 -- .../docker_scripts}/bash-config | 0 .../include/etc/kafka/docker/configure | 0 .../etc/kafka/docker/configureDefaults | 0 .../include/etc/kafka/docker/ensure | 0 .../docker/kafka-log4j.properties.template | 0 .../kafka/docker/kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../include/etc/kafka/docker/launch | 0 .../include/etc/kafka/docker/run | 0 docker/{jvm => resources}/ub/go.mod | 0 docker/{jvm => resources}/ub/go.sum | 0 .../ub/testResources/sampleFile | 0 .../ub/testResources/sampleFile2 | 0 .../ub/testResources/sampleLog4j.template | 0 docker/{jvm => resources}/ub/ub.go | 0 docker/{jvm => resources}/ub/ub_test.go | 0 .../__pycache__/constants.cpython-39.pyc | Bin 951 -> 952 bytes docker/{jvm => test}/constants.py | 0 docker/{jvm => test}/docker_sanity_test.py | 25 +-- docker/{jvm => test}/fixtures/input.txt | 0 .../fixtures/kraft/docker-compose.yml | 0 docker/{jvm => test}/fixtures/output.txt | 0 docker/{jvm => test}/fixtures/schema.avro | 0 .../secrets/broker_broker-ssl_cert-file | 0 .../secrets/broker_broker-ssl_cert-signed | 0 .../broker_broker-ssl_server.keystore.jks | Bin .../broker_broker-ssl_server.truststore.jks | Bin .../fixtures/secrets/broker_broker_cert-file | 0 .../secrets/broker_broker_cert-signed | 0 .../secrets/broker_broker_server.keystore.jks | Bin .../broker_broker_server.truststore.jks | Bin docker/{jvm => test}/fixtures/secrets/ca-cert | 0 .../fixtures/secrets/ca-cert.key | 0 .../fixtures/secrets/ca-cert.srl | 0 .../fixtures/secrets/client_python_client.key | 0 .../fixtures/secrets/client_python_client.pem | 0 .../fixtures/secrets/client_python_client.req | 0 .../fixtures/secrets/kafka_keystore_creds | 0 .../fixtures/secrets/kafka_ssl_key_creds | 0 .../fixtures/secrets/kafka_truststore_creds | 0 .../fixtures/source_connector.json | 0 .../fixtures/zookeeper/docker-compose.yml | 0 docker/{jvm => test}/requirements.txt | 0 49 files changed, 56 insertions(+), 278 deletions(-) create mode 100644 docker/docker_build_test.py delete mode 100644 docker/jvm/docker_release.py delete mode 100644 docker/jvm/fixtures/native/docker-compose.yml delete mode 100755 docker/jvm/fixtures/secrets/kafka-generate-ssl.sh delete mode 100644 docker/jvm/fixtures/sink_connector.json rename docker/{jvm => resources/docker_scripts}/bash-config (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/configure (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/configureDefaults (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/ensure (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-log4j.properties.template (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-propertiesSpec.json (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-tools-log4j.properties.template (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/launch (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/run (100%) rename docker/{jvm => resources}/ub/go.mod (100%) rename docker/{jvm => resources}/ub/go.sum (100%) rename docker/{jvm => resources}/ub/testResources/sampleFile (100%) rename docker/{jvm => resources}/ub/testResources/sampleFile2 (100%) rename docker/{jvm => resources}/ub/testResources/sampleLog4j.template (100%) rename docker/{jvm => resources}/ub/ub.go (100%) rename docker/{jvm => resources}/ub/ub_test.go (100%) rename docker/{jvm => test}/__pycache__/constants.cpython-39.pyc (83%) rename docker/{jvm => test}/constants.py (100%) rename docker/{jvm => test}/docker_sanity_test.py (95%) rename docker/{jvm => test}/fixtures/input.txt (100%) rename docker/{jvm => test}/fixtures/kraft/docker-compose.yml (100%) rename docker/{jvm => test}/fixtures/output.txt (100%) rename docker/{jvm => test}/fixtures/schema.avro (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_cert-file (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_cert-signed (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_server.keystore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_server.truststore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_cert-file (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_cert-signed (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_server.keystore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_server.truststore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert.key (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert.srl (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.key (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.pem (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.req (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_keystore_creds (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_ssl_key_creds (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_truststore_creds (100%) rename docker/{jvm => test}/fixtures/source_connector.json (100%) rename docker/{jvm => test}/fixtures/zookeeper/docker-compose.yml (100%) rename docker/{jvm => test}/requirements.txt (100%) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py new file mode 100644 index 0000000000000..1e9d5d9862b02 --- /dev/null +++ b/docker/docker_build_test.py @@ -0,0 +1,37 @@ +import subprocess +from datetime import date +import argparse +from distutils.dir_util import copy_tree +import shutil + +def build_jvm(image, tag, kafka_url): + image = f'{image}:{tag}' + copy_tree("resources", "jvm/resources") + result = subprocess.run(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + "--build-arg", f'build_date={date.today()}', "jvm"]) + if result.stderr: + print(result.stdout) + return + shutil.rmtree("jvm/resources") + +def run_jvm_tests(image, tag): + subprocess.Popen(["python", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("image") + parser.add_argument("tag") + parser.add_argument("image_type", default="all") + parser.add_argument("-ku", "--kafka-url", dest="kafka_url") + parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only builds the image, don't run tests") + parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") + args = parser.parse_args() + + if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)): + if args.kafka_url: + build_jvm(args.image, args.tag, args.kafka_url) + else: + raise ValueError("--kafka-url is a required argument for jvm image") + + if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): + run_jvm_tests(args.image, args.tag) \ No newline at end of file diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index ff54a3853f667..738dd6e62e766 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -21,7 +21,7 @@ ARG GOLANG_VERSION=1.21.1 FROM golang:${GOLANG_VERSION} AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser ub/ ./ +COPY --chown=appuser:appuser resources/ub/ ./ RUN go build -ldflags="-w -s" ./ub.go USER appuser RUN go test ./... @@ -29,24 +29,22 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY bash-config /etc/kafka/docker/bash-config +COPY resources/docker_scripts/bash-config /etc/kafka/docker/bash-config # exposed ports EXPOSE 9092 USER root -ARG kafka_url=https://archive.apache.org/dist/kafka/3.5.1/kafka_2.13-3.5.1.tgz -ARG kafka_version=2.13-3.5.1 -ARG vcs_ref=unspecified +# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments +ARG kafka_url=unspecified ARG build_date=unspecified + LABEL org.label-schema.name="kafka" \ org.label-schema.description="Apache Kafka" \ org.label-schema.build-date="${build_date}" \ org.label-schema.vcs-url="https://github.com/apache/kafka" \ - org.label-schema.vcs-ref="${vcs_ref}" \ - org.label-schema.version="${kafka_version}" \ org.label-schema.schema-version="1.0" \ maintainer="apache" @@ -80,7 +78,7 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser include/etc/kafka/docker /etc/kafka/docker +COPY --chown=appuser:appuser resources/docker_scripts/include/etc/kafka/docker /etc/kafka/docker USER appuser diff --git a/docker/jvm/docker_release.py b/docker/jvm/docker_release.py deleted file mode 100644 index 3f5ee911d7f99..0000000000000 --- a/docker/jvm/docker_release.py +++ /dev/null @@ -1,15 +0,0 @@ -import subprocess -from datetime import date - -def main(): - image = "apache/kafka:3.5.1" - # Refactor this and extract to constant - result = subprocess.run(["docker", "build", "-t", image, "--build-arg", "kafka_url=https://downloads.apache.org/kafka/3.5.1/kafka_2.13-3.5.1.tgz", - "--build-arg", f'build_date={date.today()}', "--build-arg", "kafka_version=2.13-3.5.1", "."]) - if result.stderr: - print(result.stdout) - return - subprocess.run(["python", "docker_sanity_test.py", image]) - -if __name__ == '__main__': - main() diff --git a/docker/jvm/fixtures/native/docker-compose.yml b/docker/jvm/fixtures/native/docker-compose.yml deleted file mode 100644 index 0d4a874ee712d..0000000000000 --- a/docker/jvm/fixtures/native/docker-compose.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -version: '2' -services: - - broker: - image: kafka-poc:latest - hostname: broker - container_name: broker - ports: - - "9092:9092" - environment: - KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh b/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh deleted file mode 100755 index 3cf17e305ab10..0000000000000 --- a/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash -# -# -# This scripts generates: -# - root CA certificate -# - server certificate and keystore -# - client keys -# -# https://cwiki.apache.org/confluence/display/KAFKA/Deploying+SSL+for+Kafka -# - - -if [[ "$1" == "-k" ]]; then - USE_KEYTOOL=1 - shift -else - USE_KEYTOOL=0 -fi - -OP="$1" -CA_CERT="$2" -PFX="$3" -HOST="$4" - -C=NN -ST=NN -L=NN -O=NN -OU=NN -CN="$HOST" - - -# Password -PASS="abcdefgh" - -# Cert validity, in days -VALIDITY=10000 - -set -e - -export LC_ALL=C - -if [[ $OP == "ca" && ! -z "$CA_CERT" && ! -z "$3" ]]; then - CN="$3" - openssl req -new -x509 -keyout ${CA_CERT}.key -out $CA_CERT -days $VALIDITY -passin "pass:$PASS" -passout "pass:$PASS" < " - echo " $0 [-k] server|client " - echo "" - echo " -k = Use keytool/Java Keystore, else standard SSL keys" - exit 1 -fi \ No newline at end of file diff --git a/docker/jvm/fixtures/sink_connector.json b/docker/jvm/fixtures/sink_connector.json deleted file mode 100644 index f22ef5495f0af..0000000000000 --- a/docker/jvm/fixtures/sink_connector.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "kafka-sink", - "config": { - "connector.class": "org.apache.kafka.connect.file.FileStreamSinkConnector", - "tasks.max": "1", - "file": "/tmp/output.txt", - "topics": "test_topic_connect", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter" - } -} \ No newline at end of file diff --git a/docker/jvm/bash-config b/docker/resources/docker_scripts/bash-config similarity index 100% rename from docker/jvm/bash-config rename to docker/resources/docker_scripts/bash-config diff --git a/docker/jvm/include/etc/kafka/docker/configure b/docker/resources/docker_scripts/include/etc/kafka/docker/configure similarity index 100% rename from docker/jvm/include/etc/kafka/docker/configure rename to docker/resources/docker_scripts/include/etc/kafka/docker/configure diff --git a/docker/jvm/include/etc/kafka/docker/configureDefaults b/docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults similarity index 100% rename from docker/jvm/include/etc/kafka/docker/configureDefaults rename to docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults diff --git a/docker/jvm/include/etc/kafka/docker/ensure b/docker/resources/docker_scripts/include/etc/kafka/docker/ensure similarity index 100% rename from docker/jvm/include/etc/kafka/docker/ensure rename to docker/resources/docker_scripts/include/etc/kafka/docker/ensure diff --git a/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template diff --git a/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json diff --git a/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template diff --git a/docker/jvm/include/etc/kafka/docker/launch b/docker/resources/docker_scripts/include/etc/kafka/docker/launch similarity index 100% rename from docker/jvm/include/etc/kafka/docker/launch rename to docker/resources/docker_scripts/include/etc/kafka/docker/launch diff --git a/docker/jvm/include/etc/kafka/docker/run b/docker/resources/docker_scripts/include/etc/kafka/docker/run similarity index 100% rename from docker/jvm/include/etc/kafka/docker/run rename to docker/resources/docker_scripts/include/etc/kafka/docker/run diff --git a/docker/jvm/ub/go.mod b/docker/resources/ub/go.mod similarity index 100% rename from docker/jvm/ub/go.mod rename to docker/resources/ub/go.mod diff --git a/docker/jvm/ub/go.sum b/docker/resources/ub/go.sum similarity index 100% rename from docker/jvm/ub/go.sum rename to docker/resources/ub/go.sum diff --git a/docker/jvm/ub/testResources/sampleFile b/docker/resources/ub/testResources/sampleFile similarity index 100% rename from docker/jvm/ub/testResources/sampleFile rename to docker/resources/ub/testResources/sampleFile diff --git a/docker/jvm/ub/testResources/sampleFile2 b/docker/resources/ub/testResources/sampleFile2 similarity index 100% rename from docker/jvm/ub/testResources/sampleFile2 rename to docker/resources/ub/testResources/sampleFile2 diff --git a/docker/jvm/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template similarity index 100% rename from docker/jvm/ub/testResources/sampleLog4j.template rename to docker/resources/ub/testResources/sampleLog4j.template diff --git a/docker/jvm/ub/ub.go b/docker/resources/ub/ub.go similarity index 100% rename from docker/jvm/ub/ub.go rename to docker/resources/ub/ub.go diff --git a/docker/jvm/ub/ub_test.go b/docker/resources/ub/ub_test.go similarity index 100% rename from docker/jvm/ub/ub_test.go rename to docker/resources/ub/ub_test.go diff --git a/docker/jvm/__pycache__/constants.cpython-39.pyc b/docker/test/__pycache__/constants.cpython-39.pyc similarity index 83% rename from docker/jvm/__pycache__/constants.cpython-39.pyc rename to docker/test/__pycache__/constants.cpython-39.pyc index 1fca3f425054fd2ab00a8e3af2eac9a849d6efef..75258fcc2b41dfd28c547d2ed05b9959b5c5ec66 100644 GIT binary patch delta 22 dcmdnazJq;(KQp8GZ$yLmD07(u8VE_OC diff --git a/docker/jvm/constants.py b/docker/test/constants.py similarity index 100% rename from docker/jvm/constants.py rename to docker/test/constants.py diff --git a/docker/jvm/docker_sanity_test.py b/docker/test/docker_sanity_test.py similarity index 95% rename from docker/jvm/docker_sanity_test.py rename to docker/test/docker_sanity_test.py index b1f4f76fe988c..7b9378299f9c5 100644 --- a/docker/jvm/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -10,6 +10,7 @@ from confluent_kafka.schema_registry import SchemaRegistryClient from HTMLTestRunner import HTMLTestRunner import constants +import argparse class DockerSanityTestCommon(unittest.TestCase): CONTAINER_NAME="broker" @@ -285,25 +286,25 @@ def tearDown(self) -> None: def test_bed(self): self.execute() -class DockerSanityTestNative(DockerSanityTestCommon): - def setUp(self) -> None: - self.startCompose("fixtures/native/docker-compose.yml") - def tearDown(self) -> None: - self.destroyCompose("fixtures/native/docker-compose.yml") - def test_bed(self): - self.execute() - if __name__ == "__main__": - if len(sys.argv) > 0: - DockerSanityTestCommon.IMAGE = sys.argv.pop() - test_classes_to_run = [DockerSanityTestKraftMode, DockerSanityTestZookeeper, DockerSanityTestNative] + parser = argparse.ArgumentParser() + parser.add_argument("image") + parser.add_argument("mode", default="all") + args = parser.parse_args() + + DockerSanityTestCommon.IMAGE = args.image + + test_classes_to_run = [] + if args.mode in ("all", "jvm"): + test_classes_to_run.extend([DockerSanityTestKraftMode, DockerSanityTestZookeeper]) + loader = unittest.TestLoader() suites_list = [] for test_class in test_classes_to_run: suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open("report.html", "w") + outfile = open(f"report.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', diff --git a/docker/jvm/fixtures/input.txt b/docker/test/fixtures/input.txt similarity index 100% rename from docker/jvm/fixtures/input.txt rename to docker/test/fixtures/input.txt diff --git a/docker/jvm/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml similarity index 100% rename from docker/jvm/fixtures/kraft/docker-compose.yml rename to docker/test/fixtures/kraft/docker-compose.yml diff --git a/docker/jvm/fixtures/output.txt b/docker/test/fixtures/output.txt similarity index 100% rename from docker/jvm/fixtures/output.txt rename to docker/test/fixtures/output.txt diff --git a/docker/jvm/fixtures/schema.avro b/docker/test/fixtures/schema.avro similarity index 100% rename from docker/jvm/fixtures/schema.avro rename to docker/test/fixtures/schema.avro diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file b/docker/test/fixtures/secrets/broker_broker-ssl_cert-file similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file rename to docker/test/fixtures/secrets/broker_broker-ssl_cert-file diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed rename to docker/test/fixtures/secrets/broker_broker-ssl_cert-signed diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks rename to docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks rename to docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker_cert-file b/docker/test/fixtures/secrets/broker_broker_cert-file similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_cert-file rename to docker/test/fixtures/secrets/broker_broker_cert-file diff --git a/docker/jvm/fixtures/secrets/broker_broker_cert-signed b/docker/test/fixtures/secrets/broker_broker_cert-signed similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_cert-signed rename to docker/test/fixtures/secrets/broker_broker_cert-signed diff --git a/docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker_server.keystore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks rename to docker/test/fixtures/secrets/broker_broker_server.keystore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker_server.truststore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks rename to docker/test/fixtures/secrets/broker_broker_server.truststore.jks diff --git a/docker/jvm/fixtures/secrets/ca-cert b/docker/test/fixtures/secrets/ca-cert similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert rename to docker/test/fixtures/secrets/ca-cert diff --git a/docker/jvm/fixtures/secrets/ca-cert.key b/docker/test/fixtures/secrets/ca-cert.key similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert.key rename to docker/test/fixtures/secrets/ca-cert.key diff --git a/docker/jvm/fixtures/secrets/ca-cert.srl b/docker/test/fixtures/secrets/ca-cert.srl similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert.srl rename to docker/test/fixtures/secrets/ca-cert.srl diff --git a/docker/jvm/fixtures/secrets/client_python_client.key b/docker/test/fixtures/secrets/client_python_client.key similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.key rename to docker/test/fixtures/secrets/client_python_client.key diff --git a/docker/jvm/fixtures/secrets/client_python_client.pem b/docker/test/fixtures/secrets/client_python_client.pem similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.pem rename to docker/test/fixtures/secrets/client_python_client.pem diff --git a/docker/jvm/fixtures/secrets/client_python_client.req b/docker/test/fixtures/secrets/client_python_client.req similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.req rename to docker/test/fixtures/secrets/client_python_client.req diff --git a/docker/jvm/fixtures/secrets/kafka_keystore_creds b/docker/test/fixtures/secrets/kafka_keystore_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_keystore_creds rename to docker/test/fixtures/secrets/kafka_keystore_creds diff --git a/docker/jvm/fixtures/secrets/kafka_ssl_key_creds b/docker/test/fixtures/secrets/kafka_ssl_key_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_ssl_key_creds rename to docker/test/fixtures/secrets/kafka_ssl_key_creds diff --git a/docker/jvm/fixtures/secrets/kafka_truststore_creds b/docker/test/fixtures/secrets/kafka_truststore_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_truststore_creds rename to docker/test/fixtures/secrets/kafka_truststore_creds diff --git a/docker/jvm/fixtures/source_connector.json b/docker/test/fixtures/source_connector.json similarity index 100% rename from docker/jvm/fixtures/source_connector.json rename to docker/test/fixtures/source_connector.json diff --git a/docker/jvm/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml similarity index 100% rename from docker/jvm/fixtures/zookeeper/docker-compose.yml rename to docker/test/fixtures/zookeeper/docker-compose.yml diff --git a/docker/jvm/requirements.txt b/docker/test/requirements.txt similarity index 100% rename from docker/jvm/requirements.txt rename to docker/test/requirements.txt From 68a5783a5b414e1db745d0cfe0785e47acb7b8c6 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:23:43 +0530 Subject: [PATCH 03/46] KAFKA-15444: Initial Changes for Native Docker Image --- core/src/main/scala/kafka/Kafka.scala | 4 + .../main/scala/kafka/KafkaNativeWrapper.scala | 21 + .../main/scala/kafka/tools/StorageTool.scala | 4 + docker/docker_build_test.py | 33 +- docker/native-image/Dockerfile | 48 ++ .../native-image-configs/jni-config.json | 35 + .../predefined-classes-config.json | 7 + .../native-image-configs/proxy-config.json | 5 + .../native-image-configs/reflect-config.json | 719 ++++++++++++++++++ .../native-image-configs/resource-config.json | 20 + .../serialization-config.json | 17 + docker/native-image/singleScript | 9 + 12 files changed, 916 insertions(+), 6 deletions(-) create mode 100644 core/src/main/scala/kafka/KafkaNativeWrapper.scala create mode 100644 docker/native-image/Dockerfile create mode 100644 docker/native-image/native-image-configs/jni-config.json create mode 100644 docker/native-image/native-image-configs/predefined-classes-config.json create mode 100644 docker/native-image/native-image-configs/proxy-config.json create mode 100644 docker/native-image/native-image-configs/reflect-config.json create mode 100644 docker/native-image/native-image-configs/resource-config.json create mode 100644 docker/native-image/native-image-configs/serialization-config.json create mode 100755 docker/native-image/singleScript diff --git a/core/src/main/scala/kafka/Kafka.scala b/core/src/main/scala/kafka/Kafka.scala index a1791ccbe0bf1..69c4906c8d5ab 100755 --- a/core/src/main/scala/kafka/Kafka.scala +++ b/core/src/main/scala/kafka/Kafka.scala @@ -86,6 +86,10 @@ object Kafka extends Logging { } def main(args: Array[String]): Unit = { + process(args) + } + + def process(args: Array[String]): Unit = { try { val serverProps = getPropsFromArgs(args) val server = buildServer(serverProps) diff --git a/core/src/main/scala/kafka/KafkaNativeWrapper.scala b/core/src/main/scala/kafka/KafkaNativeWrapper.scala new file mode 100644 index 0000000000000..898b4aac88e0f --- /dev/null +++ b/core/src/main/scala/kafka/KafkaNativeWrapper.scala @@ -0,0 +1,21 @@ +package kafka + +import kafka.tools.StorageTool +import kafka.utils.Logging + +object KafkaNativeWrapper extends Logging { + def main(args: Array[String]): Unit = { + if (args.length == 0) { + + } + val operation = args.head + val arguments = args.tail + operation match { + case "storage-tool" => StorageTool.process(arguments) + case "kafka" => Kafka.process(arguments) + case _ => + throw new RuntimeException(s"Unknown operation $operation. " + + s"Please provide a valid operation: 'storage-tool' or 'kafka'.") + } + } +} \ No newline at end of file diff --git a/core/src/main/scala/kafka/tools/StorageTool.scala b/core/src/main/scala/kafka/tools/StorageTool.scala index 2aa1e02853e1f..fb6fc40e0796b 100644 --- a/core/src/main/scala/kafka/tools/StorageTool.scala +++ b/core/src/main/scala/kafka/tools/StorageTool.scala @@ -42,6 +42,10 @@ import scala.collection.mutable.ArrayBuffer object StorageTool extends Logging { def main(args: Array[String]): Unit = { + process(args) + } + + def process(args: Array[String]): Unit = { try { val namespace = parseArguments(args) val command = namespace.getString("command") diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 1e9d5d9862b02..3986dc720da96 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -4,27 +4,45 @@ from distutils.dir_util import copy_tree import shutil + def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' copy_tree("resources", "jvm/resources") - result = subprocess.run(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", - "--build-arg", f'build_date={date.today()}', "jvm"]) + result = subprocess.run( + ["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + "--build-arg", f'build_date={date.today()}', "jvm"]) if result.stderr: print(result.stdout) return shutil.rmtree("jvm/resources") + +def build_native(image, tag): + image = f'{image}:{tag}' + copy_tree("resources", "native-image/resources") + result = subprocess.run( + ["docker", "build", "-f", "native-image/Dockerfile", "-t", image, + "--build-arg", f'build_date={date.today()}', "native-image"]) + if result.stderr: + print(result.stdout) + return + shutil.rmtree("native-image/resources") + + def run_jvm_tests(image, tag): subprocess.Popen(["python", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("image") parser.add_argument("tag") parser.add_argument("image_type", default="all") parser.add_argument("-ku", "--kafka-url", dest="kafka_url") - parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only builds the image, don't run tests") - parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") + parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, + help="Only builds the image, don't run tests") + parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, + help="Only run the tests, don't build the image") args = parser.parse_args() if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)): @@ -32,6 +50,9 @@ def run_jvm_tests(image, tag): build_jvm(args.image, args.tag, args.kafka_url) else: raise ValueError("--kafka-url is a required argument for jvm image") - + + if args.image_type in ("all", "native-image") and (args.build_only or not (args.build_only or args.test_only)): + build_native(args.image, args.tag) + if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): - run_jvm_tests(args.image, args.tag) \ No newline at end of file + run_jvm_tests(args.image, args.tag) diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile new file mode 100644 index 0000000000000..155f8418e911c --- /dev/null +++ b/docker/native-image/Dockerfile @@ -0,0 +1,48 @@ +FROM golang:1.21-bullseye AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser resources/ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image + +WORKDIR /app + +COPY kafka_2.13-3.7.0-SNAPSHOT.tgz kafka_2.13-3.7.0-SNAPSHOT.tgz +COPY native-image-configs native-image-configs + +RUN tar -xzf kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + rm kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + cd kafka_2.13-3.7.0-SNAPSHOT ; \ + native-image --no-fallback \ + --allow-incomplete-classpath \ + --report-unsupported-elements-at-runtime \ + --install-exit-handlers \ + -H:+ReportExceptionStackTraces \ + -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \ + -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \ + -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \ + -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \ + -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \ + -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \ + --verbose \ + -cp "libs/*" kafka.KafkaNativeWrapper + + +FROM alpine:latest +RUN apk update && \ + apk add --no-cache gcompat && \ + mkdir -p /etc/kafka/config +WORKDIR /app + +COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . +#COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/config/kraft/server.properties server.properties +#COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/config/log4j.properties log4j.properties +COPY resources/docker_scripts/include/etc/kafka/docker /etc/kafka/resources +COPY singleScript /etc/kafka/resources/ + +CMD ["/etc/kafka/resources/singleScript"] diff --git a/docker/native-image/native-image-configs/jni-config.json b/docker/native-image/native-image-configs/jni-config.json new file mode 100644 index 0000000000000..50b6bf1fd6dbb --- /dev/null +++ b/docker/native-image/native-image-configs/jni-config.json @@ -0,0 +1,35 @@ +[ +{ + "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;" +}, +{ + "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;" +}, +{ + "name":"com.github.luben.zstd.ZstdInputStreamNoFinalizer", + "fields":[{"name":"dstPos"}, {"name":"srcPos"}] +}, +{ + "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] +}, +{ + "name":"com.sun.management.internal.DiagnosticCommandInfo", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] +}, +{ + "name":"java.lang.Boolean", + "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.OutOfMemoryError" +}, +{ + "name":"java.util.Arrays", + "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] +}, +{ + "name":"sun.management.VMManagementImpl", + "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}] +} +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/predefined-classes-config.json b/docker/native-image/native-image-configs/predefined-classes-config.json new file mode 100644 index 0000000000000..847895071fbcb --- /dev/null +++ b/docker/native-image/native-image-configs/predefined-classes-config.json @@ -0,0 +1,7 @@ +[ + { + "type":"agent-extracted", + "classes":[ + ] + } +] diff --git a/docker/native-image/native-image-configs/proxy-config.json b/docker/native-image/native-image-configs/proxy-config.json new file mode 100644 index 0000000000000..2a8e3c8a13689 --- /dev/null +++ b/docker/native-image/native-image-configs/proxy-config.json @@ -0,0 +1,5 @@ +[ + { + "interfaces":["sun.misc.SignalHandler"] + } +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/reflect-config.json b/docker/native-image/native-image-configs/reflect-config.json new file mode 100644 index 0000000000000..30a90f34cea81 --- /dev/null +++ b/docker/native-image/native-image-configs/reflect-config.json @@ -0,0 +1,719 @@ +[ +{ + "name":"[B" +}, +{ + "name":"[C" +}, +{ + "name":"[D" +}, +{ + "name":"[F" +}, +{ + "name":"[I" +}, +{ + "name":"[J" +}, +{ + "name":"[Ljava.io.File;" +}, +{ + "name":"[Ljava.lang.Runnable;" +}, +{ + "name":"[Ljava.lang.String;" +}, +{ + "name":"[Ljava.util.concurrent.CompletableFuture;" +}, +{ + "name":"[Ljava.util.concurrent.Future;" +}, +{ + "name":"[Ljavax.management.openmbean.CompositeData;" +}, +{ + "name":"[Lkafka.server.DelayedOperationPurgatory$WatcherList;" +}, +{ + "name":"[Lorg.apache.kafka.common.resource.PatternType;" +}, +{ + "name":"[Lscala.Tuple2;" +}, +{ + "name":"[S" +}, +{ + "name":"[Z" +}, +{ + "name":"com.fasterxml.jackson.databind.ext.Java7SupportImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.management.GarbageCollectorMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.GcInfo", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.HotSpotDiagnosticMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.ThreadMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.UnixOperatingSystemMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.VMOption", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.internal.GarbageCollectorExtImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.HotSpotDiagnostic", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.HotSpotThreadImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.OperatingSystemImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Gauge", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$GaugeMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Histogram", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$HistogramMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Meter", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$MeterMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Timer", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$TimerMBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.beans.PropertyVetoException" +}, +{ + "name":"java.io.Serializable", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.lang.Boolean", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Byte", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Character", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.ClassValue" +}, +{ + "name":"java.lang.Deprecated", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.Double", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Float", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Integer", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Iterable", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.lang.Long", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Object", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.ObjectBeanInfo" +}, +{ + "name":"java.lang.ObjectCustomizer" +}, +{ + "name":"java.lang.Short", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.StackTraceElement", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.String", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Thread", + "fields":[{"name":"threadLocalRandomProbe"}] +}, +{ + "name":"java.lang.Void", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.invoke.VarHandle", + "methods":[{"name":"releaseFence","parameterTypes":[] }] +}, +{ + "name":"java.lang.management.BufferPoolMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ClassLoadingMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.CompilationMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.LockInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ManagementPermission", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.management.MemoryMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryManagerMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryPoolMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryUsage", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MonitorInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.PlatformLoggingMXBean", + "queryAllPublicMethods":true, + "methods":[{"name":"getLoggerLevel","parameterTypes":["java.lang.String"] }, {"name":"getLoggerNames","parameterTypes":[] }, {"name":"getParentLoggerName","parameterTypes":["java.lang.String"] }, {"name":"setLoggerLevel","parameterTypes":["java.lang.String","java.lang.String"] }] +}, +{ + "name":"java.lang.management.RuntimeMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ThreadInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.math.BigDecimal" +}, +{ + "name":"java.math.BigInteger" +}, +{ + "name":"java.rmi.Remote", + "queryAllPublicMethods":true +}, +{ + "name":"java.rmi.registry.Registry", + "queryAllPublicMethods":true +}, +{ + "name":"java.security.SecureRandomParameters" +}, +{ + "name":"java.util.AbstractCollection", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.AbstractList", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.AbstractMap", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Collection", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Date" +}, +{ + "name":"java.util.List", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Map", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.PropertyPermission", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] +}, +{ + "name":"java.util.concurrent.ForkJoinTask", + "fields":[{"name":"aux"}, {"name":"status"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicBoolean", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicReference", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.Striped64", + "fields":[{"name":"base"}, {"name":"cellsBusy"}] +}, +{ + "name":"java.util.concurrent.atomic.Striped64$Cell", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.logging.LogManager", + "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] +}, +{ + "name":"java.util.logging.LoggingMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.util.zip.CRC32C", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"java.util.zip.Checksum", + "methods":[{"name":"update","parameterTypes":["java.nio.ByteBuffer"] }] +}, +{ + "name":"javax.management.MBeanOperationInfo", + "queryAllPublicMethods":true, + "methods":[{"name":"getSignature","parameterTypes":[] }] +}, +{ + "name":"javax.management.MBeanServerBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"javax.management.ObjectName" +}, +{ + "name":"javax.management.StandardEmitterMBean", + "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }] +}, +{ + "name":"javax.management.openmbean.CompositeData" +}, +{ + "name":"javax.management.openmbean.OpenMBeanOperationInfoSupport" +}, +{ + "name":"javax.management.openmbean.TabularData" +}, +{ + "name":"javax.management.remote.rmi.RMIServer", + "queryAllPublicMethods":true, + "methods":[{"name":"getVersion","parameterTypes":[] }, {"name":"newClient","parameterTypes":["java.lang.Object"] }] +}, +{ + "name":"javax.management.remote.rmi.RMIServerImpl_Skel" +}, +{ + "name":"javax.management.remote.rmi.RMIServerImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"jdk.management.jfr.ConfigurationInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.EventTypeInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.FlightRecorderMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.FlightRecorderMXBeanImpl", + "queryAllPublicConstructors":true, + "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }, {"name":"getNotificationInfo","parameterTypes":[] }] +}, +{ + "name":"jdk.management.jfr.RecordingInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.SettingDescriptorInfo", + "queryAllPublicMethods":true +}, +{ + "name":"kafka.tools.ConsoleProducer$LineMessageReader", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.tools.DefaultMessageFormatter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.utils.Log4jController", + "queryAllPublicConstructors":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.utils.Log4jControllerMBean", + "queryAllPublicMethods":true +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeCompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4HCJavaSafeCompressor", + "fields":[{"name":"INSTANCE"}], + "methods":[{"name":"","parameterTypes":["int"] }] +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeFastDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeSafeDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash32JavaSafe", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash64JavaSafe", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash32JavaSafe$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash64JavaSafe$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4HCJNICompressor", + "fields":[{"name":"INSTANCE"}], + "methods":[{"name":"","parameterTypes":["int"] }] +}, +{ + "name":"net.jpountz.lz4.LZ4JNICompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JNIFastDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JNISafeDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash32JNI$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash64JNI$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash32JNI", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash64JNI", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"org.apache.kafka.clients.consumer.CooperativeStickyAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.clients.consumer.RangeAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.metrics.JmxReporter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.serialization.ByteArraySerializer", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfo", + "queryAllPublicConstructors":true +}, +{ + "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfoMBean", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.kafka.coordinator.group.assignor.RangeAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.log4j.AppenderSkeleton", + "queryAllPublicMethods":true, + "methods":[{"name":"setThreshold","parameterTypes":["org.apache.log4j.Priority"] }] +}, +{ + "name":"org.apache.log4j.AppenderSkeletonBeanInfo" +}, +{ + "name":"org.apache.log4j.AppenderSkeletonCustomizer" +}, +{ + "name":"org.apache.log4j.ConsoleAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setTarget","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.ConsoleAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.ConsoleAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setDatePattern","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.FileAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"setFile","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.FileAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.FileAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.Layout", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.log4j.LayoutBeanInfo" +}, +{ + "name":"org.apache.log4j.LayoutCustomizer" +}, +{ + "name":"org.apache.log4j.Log4jLoggerFactory" +}, +{ + "name":"org.apache.log4j.PatternLayout", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setConversionPattern","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.PatternLayoutBeanInfo" +}, +{ + "name":"org.apache.log4j.PatternLayoutCustomizer" +}, +{ + "name":"org.apache.log4j.WriterAppender", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.log4j.WriterAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.WriterAppenderCustomizer" +}, +{ + "name":"org.apache.zookeeper.ClientCnxnSocketNIO", + "methods":[{"name":"","parameterTypes":["org.apache.zookeeper.client.ZKClientConfig"] }] +}, +{ + "name":"org.osgi.framework.BundleEvent" +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$IterableWrapperTrait", + "queryAllDeclaredMethods":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MapWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MutableBufferWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MutableMapWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$SeqWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.reflect.ScalaSignature", + "queryAllPublicMethods":true +}, +{ + "name":"sun.management.ClassLoadingImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.CompilationImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.ManagementFactoryHelper$1", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.ManagementFactoryHelper$PlatformLoggingImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryManagerImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryPoolImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.RuntimeImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.misc.Signal", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"getName","parameterTypes":[] }, {"name":"handle","parameterTypes":["sun.misc.Signal","sun.misc.SignalHandler"] }] +}, +{ + "name":"sun.misc.SignalHandler", + "methods":[{"name":"handle","parameterTypes":["sun.misc.Signal"] }] +}, +{ + "name":"sun.misc.Unsafe", + "fields":[{"name":"theUnsafe"}], + "methods":[{"name":"invokeCleaner","parameterTypes":["java.nio.ByteBuffer"] }] +}, +{ + "name":"sun.rmi.registry.RegistryImpl_Skel", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.rmi.registry.RegistryImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"sun.rmi.transport.DGCImpl_Skel", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.rmi.transport.DGCImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"sun.security.provider.ConfigFile", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.MD5", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.NativePRNG", + "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] +}, +{ + "name":"sun.security.provider.SHA", + "methods":[{"name":"","parameterTypes":[] }] +} +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/resource-config.json b/docker/native-image/native-image-configs/resource-config.json new file mode 100644 index 0000000000000..8b489fd205938 --- /dev/null +++ b/docker/native-image/native-image-configs/resource-config.json @@ -0,0 +1,20 @@ +{ + "resources":{ + "includes":[{ + "pattern":"\\Qkafka/kafka-version.properties\\E" + }, { + "pattern":"\\Qlinux/amd64/libzstd-jni-1.5.5-6.so\\E" + }, { + "pattern":"\\Qnet/jpountz/util/linux/amd64/liblz4-java.so\\E" + }, { + "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/VERSION\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/native/Linux/x86_64/libsnappyjava.so\\E" + }]}, + "bundles":[{ + "name":"net.sourceforge.argparse4j.internal.ArgumentParserImpl", + "locales":["und"] + }] +} \ No newline at end of file diff --git a/docker/native-image/native-image-configs/serialization-config.json b/docker/native-image/native-image-configs/serialization-config.json new file mode 100644 index 0000000000000..0d85b152974ca --- /dev/null +++ b/docker/native-image/native-image-configs/serialization-config.json @@ -0,0 +1,17 @@ +{ + "types":[ + { + "name":"java.rmi.server.RemoteObject" + }, + { + "name":"java.rmi.server.RemoteStub" + }, + { + "name":"javax.management.remote.rmi.RMIServerImpl_Stub" + } + ], + "lambdaCapturingTypes":[ + ], + "proxies":[ + ] +} \ No newline at end of file diff --git a/docker/native-image/singleScript b/docker/native-image/singleScript new file mode 100755 index 0000000000000..bec0214784a6f --- /dev/null +++ b/docker/native-image/singleScript @@ -0,0 +1,9 @@ +#!/bin/sh + +ub render-properties /etc/kafka/resources/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties +ub render-template /etc/kafka/resources/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties +ub render-template /etc/kafka/resources/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties + + +result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) +exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties \ No newline at end of file From 06234352e0a318dc036e33f9b647e7702ee8622e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 15:51:14 +0530 Subject: [PATCH 04/46] Refactor scripts running in Dockerfile --- docker/docker_build_test.py | 10 +- docker/jvm/Dockerfile | 5 +- .../etc/kafka/docker/ensure => jvm/launch} | 25 +- .../{docker_scripts => }/bash-config | 0 .../kafka/docker => common_scripts}/configure | 2 - .../configureDefaults | 0 .../kafka-log4j.properties.template | 0 .../kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../etc/kafka/docker => common_scripts}/run | 6 - .../include/etc/kafka/docker/launch | 30 -- docker/test/report.html | 327 ++++++++++++++++++ 12 files changed, 356 insertions(+), 49 deletions(-) rename docker/{resources/docker_scripts/include/etc/kafka/docker/ensure => jvm/launch} (54%) rename docker/resources/{docker_scripts => }/bash-config (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/configure (99%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/configureDefaults (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-log4j.properties.template (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-propertiesSpec.json (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-tools-log4j.properties.template (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/run (92%) delete mode 100755 docker/resources/docker_scripts/include/etc/kafka/docker/launch create mode 100644 docker/test/report.html diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 1e9d5d9862b02..7ae22188845d0 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -19,11 +19,11 @@ def run_jvm_tests(image, tag): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("image") - parser.add_argument("tag") - parser.add_argument("image_type", default="all") - parser.add_argument("-ku", "--kafka-url", dest="kafka_url") - parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only builds the image, don't run tests") + parser.add_argument("image", help="Image name that you want to keep for the Docker image") + parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") + parser.add_argument("-type", "--image-type", default="all", dest="image_type", help="Image type you want to build. By default it's all") + parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 738dd6e62e766..80ca045a41b45 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -29,7 +29,7 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY resources/docker_scripts/bash-config /etc/kafka/docker/bash-config +COPY resources/bash-config /etc/kafka/docker/bash-config # exposed ports EXPOSE 9092 @@ -78,7 +78,8 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser resources/docker_scripts/include/etc/kafka/docker /etc/kafka/docker +COPY --chown=appuser:appuser resources/common_scripts /etc/kafka/docker +COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/ensure b/docker/jvm/launch similarity index 54% rename from docker/resources/docker_scripts/include/etc/kafka/docker/ensure rename to docker/jvm/launch index a50f7812ca863..778313c7ed349 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/ensure +++ b/docker/jvm/launch @@ -17,11 +17,25 @@ # limitations under the License. ############################################################################### -. /etc/kafka/docker/bash-config -export KAFKA_DATA_DIRS=${KAFKA_DATA_DIRS:-"/var/lib/kafka/data"} -echo "===> Check if $KAFKA_DATA_DIRS is writable ..." -ub path "$KAFKA_DATA_DIRS" writable +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi # KRaft required step: Format the storage directory with provided cluster ID unless it already exists. if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] @@ -33,3 +47,6 @@ then echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } fi + +# Start kafka broker +exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties diff --git a/docker/resources/docker_scripts/bash-config b/docker/resources/bash-config similarity index 100% rename from docker/resources/docker_scripts/bash-config rename to docker/resources/bash-config diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/configure b/docker/resources/common_scripts/configure similarity index 99% rename from docker/resources/docker_scripts/include/etc/kafka/docker/configure rename to docker/resources/common_scripts/configure index 059d03c0819ab..6793fbe3041ac 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/configure +++ b/docker/resources/common_scripts/configure @@ -17,8 +17,6 @@ # limitations under the License. ############################################################################### -. /etc/kafka/docker/bash-config - # --- for broker # If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults b/docker/resources/common_scripts/configureDefaults similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults rename to docker/resources/common_scripts/configureDefaults diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/resources/common_scripts/kafka-log4j.properties.template similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template rename to docker/resources/common_scripts/kafka-log4j.properties.template diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/resources/common_scripts/kafka-propertiesSpec.json similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json rename to docker/resources/common_scripts/kafka-propertiesSpec.json diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/resources/common_scripts/kafka-tools-log4j.properties.template similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template rename to docker/resources/common_scripts/kafka-tools-log4j.properties.template diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/run b/docker/resources/common_scripts/run similarity index 92% rename from docker/resources/docker_scripts/include/etc/kafka/docker/run rename to docker/resources/common_scripts/run index d32e8e59f4441..ed794d0247b1b 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/run +++ b/docker/resources/common_scripts/run @@ -19,9 +19,6 @@ . /etc/kafka/docker/bash-config -#TODO: REMOVE THIS -echo $(date +"%H:%M:%S.%3N") - # Set environment values if they exist as arguments if [ $# -ne 0 ]; then echo "===> Overriding env params with args ..." @@ -43,8 +40,5 @@ fi echo "===> Configuring ..." /etc/kafka/docker/configure -echo "===> Running preflight checks ... " -/etc/kafka/docker/ensure - echo "===> Launching ... " exec /etc/kafka/docker/launch diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/launch b/docker/resources/docker_scripts/include/etc/kafka/docker/launch deleted file mode 100755 index b29294011c780..0000000000000 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/launch +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - - -# Start kafka broker -# TODO: REMOVE THIS -echo "$(date +"%H:%M:%S.%3N") ===> Launching kafka ... " -/opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties & # your first application -P1=$! # capture PID of the process - -# Wait for process to exit -wait -n $P1 -# Exit with status of process that exited first -exit $? diff --git a/docker/test/report.html b/docker/test/report.html new file mode 100644 index 0000000000000..de108c5619f1f --- /dev/null +++ b/docker/test/report.html @@ -0,0 +1,327 @@ + + + + + Test Report + + + + + + + + + +
+

Test Report

+

Start Time: 2023-10-16 15:46:29

+

Duration: 0:04:16.391973

+

Status: Pass 2

+ +

This demonstrates the report output.

+
+ + + +

Show +Summary +Failed +All +

+ ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Test Group/Test caseCountPassFailErrorView
DockerSanityTestKraftMode1100Detail
test_bed
 + + + + pass + + + + +
DockerSanityTestZookeeper1100Detail
test_bed
 + + + + pass + + + + +
Total2200 
+ +
 
+ + + From 1ccffa58c64d4d8f28a5dcbb3cc0cc2bf7360dea Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:04:29 +0530 Subject: [PATCH 05/46] KAFKA-15444: Add Licence details --- .../main/scala/kafka/KafkaNativeWrapper.scala | 17 ++++++++++++++ docker/native-image/Dockerfile | 21 +++++++++++++---- docker/native-image/launch | 23 +++++++++++++++++++ docker/native-image/singleScript | 9 -------- 4 files changed, 57 insertions(+), 13 deletions(-) create mode 100755 docker/native-image/launch delete mode 100755 docker/native-image/singleScript diff --git a/core/src/main/scala/kafka/KafkaNativeWrapper.scala b/core/src/main/scala/kafka/KafkaNativeWrapper.scala index 898b4aac88e0f..a5975afc56854 100644 --- a/core/src/main/scala/kafka/KafkaNativeWrapper.scala +++ b/core/src/main/scala/kafka/KafkaNativeWrapper.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kafka import kafka.tools.StorageTool diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile index 155f8418e911c..15119f1871903 100644 --- a/docker/native-image/Dockerfile +++ b/docker/native-image/Dockerfile @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM golang:1.21-bullseye AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser @@ -40,9 +55,7 @@ WORKDIR /app COPY --from=build-ub /build/ub /usr/bin COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . -#COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/config/kraft/server.properties server.properties -#COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/config/log4j.properties log4j.properties COPY resources/docker_scripts/include/etc/kafka/docker /etc/kafka/resources -COPY singleScript /etc/kafka/resources/ +COPY launch /etc/kafka/resources/ -CMD ["/etc/kafka/resources/singleScript"] +CMD ["/etc/kafka/resources/launch"] diff --git a/docker/native-image/launch b/docker/native-image/launch new file mode 100755 index 0000000000000..53190f1755260 --- /dev/null +++ b/docker/native-image/launch @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ub render-properties /etc/kafka/resources/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties +ub render-template /etc/kafka/resources/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties +ub render-template /etc/kafka/resources/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties + + +result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) +exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties \ No newline at end of file diff --git a/docker/native-image/singleScript b/docker/native-image/singleScript deleted file mode 100755 index bec0214784a6f..0000000000000 --- a/docker/native-image/singleScript +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -ub render-properties /etc/kafka/resources/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties -ub render-template /etc/kafka/resources/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties -ub render-template /etc/kafka/resources/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties - - -result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) -exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties \ No newline at end of file From 0c7471ec80ca74b350a0692e7c2e91884b19c23a Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:13:33 +0530 Subject: [PATCH 06/46] KAFKA-15444: NIT --- core/src/main/scala/kafka/KafkaNativeWrapper.scala | 2 +- docker/native-image/Dockerfile | 2 +- docker/native-image/launch | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/KafkaNativeWrapper.scala b/core/src/main/scala/kafka/KafkaNativeWrapper.scala index a5975afc56854..d0a6bd07dc0b0 100644 --- a/core/src/main/scala/kafka/KafkaNativeWrapper.scala +++ b/core/src/main/scala/kafka/KafkaNativeWrapper.scala @@ -35,4 +35,4 @@ object KafkaNativeWrapper extends Logging { s"Please provide a valid operation: 'storage-tool' or 'kafka'.") } } -} \ No newline at end of file +} diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile index 15119f1871903..c023337037267 100644 --- a/docker/native-image/Dockerfile +++ b/docker/native-image/Dockerfile @@ -55,7 +55,7 @@ WORKDIR /app COPY --from=build-ub /build/ub /usr/bin COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . -COPY resources/docker_scripts/include/etc/kafka/docker /etc/kafka/resources +COPY resources/common_scripts /etc/kafka/resources COPY launch /etc/kafka/resources/ CMD ["/etc/kafka/resources/launch"] diff --git a/docker/native-image/launch b/docker/native-image/launch index 53190f1755260..75eed734b7cef 100755 --- a/docker/native-image/launch +++ b/docker/native-image/launch @@ -20,4 +20,4 @@ ub render-template /etc/kafka/resources/kafka-tools-log4j.properties.template > result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) -exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties \ No newline at end of file +exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties From eef401b26f992d59300265ec3938146317eeeca0 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 17:41:58 +0530 Subject: [PATCH 07/46] Refactor bash scripts --- docker/docker_build_test.py | 2 +- docker/jvm/Dockerfile | 16 +- .../{ => common-scripts}/bash-config | 0 .../configure | 0 .../configureDefaults | 0 .../kafka-log4j.properties.template | 0 .../kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../{common_scripts => common-scripts}/run | 2 +- .../test/__pycache__/constants.cpython-39.pyc | Bin 952 -> 0 bytes docker/test/fixtures/kraft/docker-compose.yml | 4 +- docker/test/report.html | 327 ------------------ 12 files changed, 7 insertions(+), 344 deletions(-) rename docker/resources/{ => common-scripts}/bash-config (100%) rename docker/resources/{common_scripts => common-scripts}/configure (100%) rename docker/resources/{common_scripts => common-scripts}/configureDefaults (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-log4j.properties.template (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-propertiesSpec.json (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-tools-log4j.properties.template (100%) rename docker/resources/{common_scripts => common-scripts}/run (97%) delete mode 100644 docker/test/__pycache__/constants.cpython-39.pyc diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 7ae22188845d0..85b5e5e0a536f 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,7 +15,7 @@ def build_jvm(image, tag, kafka_url): shutil.rmtree("jvm/resources") def run_jvm_tests(image, tag): - subprocess.Popen(["python", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 80ca045a41b45..ae1cff10a6dcb 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -29,16 +29,14 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY resources/bash-config /etc/kafka/docker/bash-config - # exposed ports EXPOSE 9092 USER root # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments -ARG kafka_url=unspecified -ARG build_date=unspecified +ARG kafka_url +ARG build_date LABEL org.label-schema.name="kafka" \ @@ -50,14 +48,6 @@ LABEL org.label-schema.name="kafka" \ ENV KAFKA_URL=$kafka_url -# allow arg override of required env params -ARG KAFKA_ZOOKEEPER_CONNECT -ENV KAFKA_ZOOKEEPER_CONNECT=${KAFKA_ZOOKEEPER_CONNECT} -ARG KAFKA_ADVERTISED_LISTENERS -ENV KAFKA_ADVERTISED_LISTENERS=${KAFKA_ADVERTISED_LISTENERS} -ARG CLUSTER_ID -ENV CLUSTER_ID=${CLUSTER_ID} - RUN set -eux ; \ apt-get update ; \ apt-get upgrade -y ; \ @@ -78,7 +68,7 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser resources/common_scripts /etc/kafka/docker +COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser diff --git a/docker/resources/bash-config b/docker/resources/common-scripts/bash-config similarity index 100% rename from docker/resources/bash-config rename to docker/resources/common-scripts/bash-config diff --git a/docker/resources/common_scripts/configure b/docker/resources/common-scripts/configure similarity index 100% rename from docker/resources/common_scripts/configure rename to docker/resources/common-scripts/configure diff --git a/docker/resources/common_scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults similarity index 100% rename from docker/resources/common_scripts/configureDefaults rename to docker/resources/common-scripts/configureDefaults diff --git a/docker/resources/common_scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template similarity index 100% rename from docker/resources/common_scripts/kafka-log4j.properties.template rename to docker/resources/common-scripts/kafka-log4j.properties.template diff --git a/docker/resources/common_scripts/kafka-propertiesSpec.json b/docker/resources/common-scripts/kafka-propertiesSpec.json similarity index 100% rename from docker/resources/common_scripts/kafka-propertiesSpec.json rename to docker/resources/common-scripts/kafka-propertiesSpec.json diff --git a/docker/resources/common_scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template similarity index 100% rename from docker/resources/common_scripts/kafka-tools-log4j.properties.template rename to docker/resources/common-scripts/kafka-tools-log4j.properties.template diff --git a/docker/resources/common_scripts/run b/docker/resources/common-scripts/run similarity index 97% rename from docker/resources/common_scripts/run rename to docker/resources/common-scripts/run index ed794d0247b1b..5104c1c11fa62 100755 --- a/docker/resources/common_scripts/run +++ b/docker/resources/common-scripts/run @@ -31,7 +31,7 @@ fi echo "===> User" id -if [[ -z "${KAFKA_ZOOKEEPER_CONNECT}" ]] +if [[ -z "${KAFKA_ZOOKEEPER_CONNECT-}" ]] then echo "===> Setting default values of environment variables if not already set." . /etc/kafka/docker/configureDefaults diff --git a/docker/test/__pycache__/constants.cpython-39.pyc b/docker/test/__pycache__/constants.cpython-39.pyc deleted file mode 100644 index 75258fcc2b41dfd28c547d2ed05b9959b5c5ec66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 952 zcma)5-EPw`6i&Nt{ab%lKw<+iZjh)DHHZtQ389tD9nrdGa#F#3V`a8WN`G9%E~35S zb$AsXA-7!d3S8lYw#!N{aFnC-eaGjV&pD1)DisX9zCEe)A8Et*8G+g)nd8!RF zu)$I|#nL#?Sn z2p3Je(}AWNgRJAUW2g2G+2;`Ht&ZD7aj;XbhW}LPqIS5d5P`DUwqVC4u5|(Jt{Wb# z_jGkd;@X#%8P=Aq6WrmFbL=iMAz45hMPAD~3-gi0zxkEOHJhYun@!iUJK_D9fXKC4 zmZsoMq#8xe;nfD_vgtSf!n+^q13;E)i^5AH%y#&9~UNT^xS;ud%?&9LP{|3dIJwILG@fgH0kJG>*5#vT{x|tZn@no;q!Zr4{{y|LCH?>a diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 704280c290f0d..5fe95ae3bc04e 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: broker: - image: {$IMAGE} + image: apache/kafka:3.6.0 hostname: broker container_name: broker ports: @@ -25,7 +25,7 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' broker-ssl: - image: {$IMAGE} + image: apache/kafka:3.6.0 hostname: broker-ssl container_name: broker-ssl ports: diff --git a/docker/test/report.html b/docker/test/report.html index de108c5619f1f..e69de29bb2d1d 100644 --- a/docker/test/report.html +++ b/docker/test/report.html @@ -1,327 +0,0 @@ - - - - - Test Report - - - - - - - - - -
-

Test Report

-

Start Time: 2023-10-16 15:46:29

-

Duration: 0:04:16.391973

-

Status: Pass 2

- -

This demonstrates the report output.

-
- - - -

Show -Summary -Failed -All -

- -------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test Group/Test caseCountPassFailErrorView
DockerSanityTestKraftMode1100Detail
test_bed
 - - - - pass - - - - -
DockerSanityTestZookeeper1100Detail
test_bed
 - - - - pass - - - - -
Total2200 
- -
 
- - - From 522b5e89ad44a348e13997cc81c3664b57defc4a Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:46:21 +0530 Subject: [PATCH 08/46] KAFKA-15444: Change folder name --- docker/native-image/Dockerfile | 17 ++++++++++------- docker/native-image/launch | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile index c023337037267..512d077c3b279 100644 --- a/docker/native-image/Dockerfile +++ b/docker/native-image/Dockerfile @@ -24,14 +24,17 @@ RUN go test ./... FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image +ARG kafka_url + WORKDIR /app -COPY kafka_2.13-3.7.0-SNAPSHOT.tgz kafka_2.13-3.7.0-SNAPSHOT.tgz +ENV KAFKA_URL=$kafka_url COPY native-image-configs native-image-configs -RUN tar -xzf kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ - rm kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ - cd kafka_2.13-3.7.0-SNAPSHOT ; \ +RUN wget -nv -O kafka.tgz "$KAFKA_URL"; \ + tar xfz kafka.tgz -C /kafka --strip-components 1; \ + rm kafka.tgz ; \ + cd kafka ; \ native-image --no-fallback \ --allow-incomplete-classpath \ --report-unsupported-elements-at-runtime \ @@ -55,7 +58,7 @@ WORKDIR /app COPY --from=build-ub /build/ub /usr/bin COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . -COPY resources/common_scripts /etc/kafka/resources -COPY launch /etc/kafka/resources/ +COPY resources/common-scripts /etc/kafka/docker +COPY launch /etc/kafka/docker/ -CMD ["/etc/kafka/resources/launch"] +CMD ["/etc/kafka/docker/launch"] diff --git a/docker/native-image/launch b/docker/native-image/launch index 75eed734b7cef..a09fc492f11fe 100755 --- a/docker/native-image/launch +++ b/docker/native-image/launch @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ub render-properties /etc/kafka/resources/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties -ub render-template /etc/kafka/resources/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties -ub render-template /etc/kafka/resources/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) From 45c2272c1a49536eebebbd050732b8efc162f47f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 18:00:28 +0530 Subject: [PATCH 09/46] Remove redundant files --- docker/jvm/docker-compose.yml | 25 ---------------- docker/test/fixtures/input.txt | 0 docker/test/fixtures/kraft/docker-compose.yml | 4 +-- docker/test/fixtures/output.txt | 0 .../secrets/broker_broker-ssl_cert-file | 17 ----------- .../secrets/broker_broker-ssl_cert-signed | 20 ------------- .../fixtures/secrets/broker_broker_cert-file | 17 ----------- .../secrets/broker_broker_cert-signed | 20 ------------- docker/test/fixtures/secrets/ca-cert.key | 30 ------------------- docker/test/fixtures/secrets/ca-cert.srl | 1 - .../fixtures/secrets/client_python_client.req | 17 ----------- docker/test/report.html | 0 12 files changed, 2 insertions(+), 149 deletions(-) delete mode 100644 docker/jvm/docker-compose.yml delete mode 100644 docker/test/fixtures/input.txt delete mode 100644 docker/test/fixtures/output.txt delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_cert-file delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_cert-signed delete mode 100644 docker/test/fixtures/secrets/broker_broker_cert-file delete mode 100644 docker/test/fixtures/secrets/broker_broker_cert-signed delete mode 100644 docker/test/fixtures/secrets/ca-cert.key delete mode 100644 docker/test/fixtures/secrets/ca-cert.srl delete mode 100644 docker/test/fixtures/secrets/client_python_client.req delete mode 100644 docker/test/report.html diff --git a/docker/jvm/docker-compose.yml b/docker/jvm/docker-compose.yml deleted file mode 100644 index 53386ea471184..0000000000000 --- a/docker/jvm/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -version: '2' -services: - - broker: - image: kafka/test:3.5.1 - hostname: broker - container_name: broker - ports: - - "9092:9092" - environment: - KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' diff --git a/docker/test/fixtures/input.txt b/docker/test/fixtures/input.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 5fe95ae3bc04e..704280c290f0d 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: broker: - image: apache/kafka:3.6.0 + image: {$IMAGE} hostname: broker container_name: broker ports: @@ -25,7 +25,7 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' broker-ssl: - image: apache/kafka:3.6.0 + image: {$IMAGE} hostname: broker-ssl container_name: broker-ssl ports: diff --git a/docker/test/fixtures/output.txt b/docker/test/fixtures/output.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_cert-file b/docker/test/fixtures/secrets/broker_broker-ssl_cert-file deleted file mode 100644 index 3a0c3c9ee9f72..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker-ssl_cert-file +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN NEW CERTIFICATE REQUEST----- -MIICyzCCAbMCAQAwVjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH -EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMRMwEQYDVQQDEwpicm9rZXIt -c3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI -+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UGDl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK -1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8z -tinteU+0wLu6lQXU2d0MHGY4CuBDp3VwtGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajr -PMZ8Mp4fpERdblrXFZNyUnycMOhchAoDMdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+ -55RCwD98uIs4C+ZNdUD91W2baXaYMXdUF7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2r -FnPYxwIDAQABoDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQUCe7i0TB0oEfd -DmuM4WWcWgCxV+8wDQYJKoZIhvcNAQELBQADggEBAHFcgQDrj7F0Oi3CannGvOB6 -XLTf6S5+f7fd9aIkq+cRIVV7aIacu8xXmTKyLgbuJMN/AhPqzZwt79jnIm54/mWh -mTBM3B9BRQT4GreJ2b1xgb543JB85LyCU2eMxx5UxOvUV/7VMxee2mRcWQUPw6Jo -0YCJqeNFZwsg80MzuQMOPA6wmGPNvgJ8LmcwMMfUnnaUlnvYL1cdw9n79Ddkuvm+ -8I63wrws9ejuO45i6o4uIL7sy9n2egwZ85oz/8hboUQgaOs+V8A2LE8xLnoLUHAV -p5pvjlB3alfhxRJEhKf4W16i0CXT3tMBl/v1o9o7NA/CllfZeb0ElboBfZA2GpI= ------END NEW CERTIFICATE REQUEST----- diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed deleted file mode 100644 index 0a5ccf415e8f6..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDQTCCAikCCQDO815g0gGg1DANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO -TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE -CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA -LzAgFw0yMzEwMTIxNzM2MzBaGA8yMDUxMDIyNzE3MzYzMFowVjELMAkGA1UEBhMC -Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV -BAsTAk5OMRMwEQYDVQQDEwpicm9rZXItc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UG -Dl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8 -a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8ztinteU+0wLu6lQXU2d0MHGY4CuBDp3Vw -tGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajrPMZ8Mp4fpERdblrXFZNyUnycMOhchAoD -MdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+55RCwD98uIs4C+ZNdUD91W2baXaYMXdU -F7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2rFnPYxwIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCYERHx3CzqOixzxtWZSugqRFahFdxWSFiuTTIv/3JhSpLjiMZGQt2YqX85 -YLnSvW0luChw8IW5S0Mtkn/Mgpnt9hzPsr1vY1aQ5By8PUXIqCNMmnIY8yC+HYNs -7DzTbU5Lin/YwRzMLnqq/9zvh+YBVdPhBrFpSXRjEkjqTXfs4fzNm8m1MsJifz3f -Q4t0iPOPjrbXrq1CQ+MstcpMwTi3eHHxcvyNHHlFLs10GH74NIymYKYwDG8fsatl -jScfxkn2rLuMFWuo42QqdHsPgS7QyTrZjvCM+5w1aUo6a35NPEcOqFD311/PJh/Y -vlSocIMIFklDRWFVh1D946t2Z42/ ------END CERTIFICATE----- diff --git a/docker/test/fixtures/secrets/broker_broker_cert-file b/docker/test/fixtures/secrets/broker_broker_cert-file deleted file mode 100644 index 9c7787fd1cea3..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker_cert-file +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN NEW CERTIFICATE REQUEST----- -MIICxzCCAa8CAQAwUjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH -EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMQ8wDQYDVQQDEwZicm9rZXIw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxSel2eRmGXIC3j/SdGlXZ -LFdbuENPekTBfIKDeKCGni6wpcZQM6OC2GUX1GALLGm1slkHFVUyuN/YgG7lQ4OA -jGRagOY92HUN9QGbelXWzCQCj4e1AgPxJjW3pfpmP4LWDrLN/IE6ECZGO4QOXyeO -NOJARCn10e+RWERSFrAAdWIpySRtIcmcsj69I66/EEuqatqER8CSEhyfaEXi1lZN -1WXraC+i49qDTBSbEBBAdZed7V69R1JYTam43qMgqUcymLZzW38Uq18zc5fv6XoO -5CiHipkmEG1H8tv1XALNpGDPN8wdp1ylTi842N+noXDMimGgGBMFAzXPlJ/QrQF/ -AgMBAAGgMDAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSwXHUp4Hj8v70a+xBm -dwWZLGlIPzANBgkqhkiG9w0BAQsFAAOCAQEAHOj/IV6oJc8BFHv1EUC9SEdNU4S7 -PnsA3bmpZ/wM5SNnmiCMppeHRX5fY3ehW/kiTCadreLXz5fjrLW6xMOXEXlojb8S -12IHK/3qcB2W/9BTmHFcnijr7oXaBwi9OBTRF2U5hUZ6vlF63jMrP6+Kfa6S6ICx -a+6o57b2RdvBsTNrwT05IHHvm3fdjCjm1MrP1kpRvXsO1WzzU7fwvhYB8Ax8hl8a -FAqqFet+5w3iEyx7a/DwUrChkgrv3zCSYZpU1O8PjwmOwHyCU8o/p/qtSE4KTj8J -PK7CJKYT/0MhaEH4EN/XpzAExybQCuGsGbYnAvLrEkkQrDoM1IwE8yDEsA== ------END NEW CERTIFICATE REQUEST----- diff --git a/docker/test/fixtures/secrets/broker_broker_cert-signed b/docker/test/fixtures/secrets/broker_broker_cert-signed deleted file mode 100644 index 414a580f1400c..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker_cert-signed +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDPTCCAiUCCQDO815g0gGg1zANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO -TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE -CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA -LzAgFw0yMzEwMTMwNTA2MjJaGA8yMDUxMDIyODA1MDYyMlowUjELMAkGA1UEBhMC -Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV -BAsTAk5OMQ8wDQYDVQQDEwZicm9rZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQCxSel2eRmGXIC3j/SdGlXZLFdbuENPekTBfIKDeKCGni6wpcZQM6OC -2GUX1GALLGm1slkHFVUyuN/YgG7lQ4OAjGRagOY92HUN9QGbelXWzCQCj4e1AgPx -JjW3pfpmP4LWDrLN/IE6ECZGO4QOXyeONOJARCn10e+RWERSFrAAdWIpySRtIcmc -sj69I66/EEuqatqER8CSEhyfaEXi1lZN1WXraC+i49qDTBSbEBBAdZed7V69R1JY -Tam43qMgqUcymLZzW38Uq18zc5fv6XoO5CiHipkmEG1H8tv1XALNpGDPN8wdp1yl -Ti842N+noXDMimGgGBMFAzXPlJ/QrQF/AgMBAAEwDQYJKoZIhvcNAQELBQADggEB -ALVAXtrZgxHpsxvvD2DJR5QzTnD+XeNmP5NtEGak3XgEkeoLm5s/QDe9Xf/8HVKB -6X0FqNoL2M/cODY1+3SOmOOw62pt+8PL4enIwjT7Yz7qnCPYBK9kaGuEDeTcyGDl -LI3ZwXIdiXXGUTdb1I30lQ/FwQQE+7ICQcZ6qTzBFhkJMr8R8fEnOULITnZqhH2N -cgJuIythTVRK1Mzy84f26AYt9WYpH6EjCDk90hS8INGfHeEb6lJ863wq1ORcSDVW -jCBSE8giKYhDTES/ZglX9l1M4yeY7b9rsySHDByJcFmiwhwFLNp+2U48YPW7qwMv -8kQTCammGGThiweU3ZyUAmc= ------END CERTIFICATE----- diff --git a/docker/test/fixtures/secrets/ca-cert.key b/docker/test/fixtures/secrets/ca-cert.key deleted file mode 100644 index 1459db022ff46..0000000000000 --- a/docker/test/fixtures/secrets/ca-cert.key +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN ENCRYPTED PRIVATE KEY----- -MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIjq9pTch7ScICAggA -MB0GCWCGSAFlAwQBKgQQtPgFCeVYgKYeCBkdA6fcDQSCBNDYadryv5Lfz7KD4t0L -hN2b3/+o9zHWh9JvfPHp7No9QAnW+eP30CC3stnxQpMqrNU/fwaYPw4S7T5Iwlsj -zf1hmycxEdCnAHJeaw1EPmXBDGqRj9uu7CNo1T/Jgdv80uH3gcHKX4OSUWA5IlYh -5CFRLDW9NdjkCb/WuAKB4iF4mTVXU1tctedjbWmSZWKzX4IShe9zu0G6UZ9bbdSc -P2CRISo8MYEr8gTEOwOLrcL3nrgzpy1BcPjmANdb+6HfYjzYCARydgWvuAbTWcpR -ertpaYDEZHozr5CDOOVftlfsNm1nA6myU50t0KZLwOl9V6iES5kY4S7yAic4PSzx -TWM8hftocRVKUBnupugb6F+snlndgTEygakv/xndqtlTvQlrCz4jGGJ4ubVkg1rL -+rXN4ok7CRqxgYx6MgqKeYuy4+UK9EN+zvtOpznWac2B1K5vunLjZ5jg2OTnHe/d -FOBfNv9NONzAIu1FbRi8csO6fcGzTZ+JyP02EetPgv3WtGRzKUYmnuGDiNPZmZRZ -fSv8uGvl4Na7kqkeQUn8O4bh3uiThvfJXgmGHPzQTcTyXhhIeqzBIc4kqDjOY40G -do8fgmY8nBBGKc3DLBP85tu37dBMkXyxxacYnl+ustFbWvmpqnPBvxvKQ/whHj2z -s1T5SGBGqW5ubYgrbg3K5dsD7GBUiT+9uFEyH5W/mv09HLu3KXKoLL2+OBnsOQjE -DQvWjO4lF/lV11p7eoHJxpAd0KABbnQxWPRY9EdNmaQuNZfeaVx5Z/yDUa1rRkyJ -gJk6R/exbOn+SQV4CfThzgrFRfB9FUYAqD4QPPpbm04sGVDeUsX0vZB6rxHGKGYg -KoqZo6kXs6AGt2R+GovLDvTcaxWL6ksKa4SRtuthPd47goHECv7n/YG++XqPfYNv -aOIKs217fQ818NNYbSc0+Uji+EjYG3PxDR+gxwMpX8qnIW3HSX7J+0I0vkPqPNCq -2VksaR3NvmjmwaNt6jq6hlbXbmtWMgwiDuO/nCqG/5n7usPQdKCwlvZF8OZH86J6 -uZPquWVNU56RRJCjjrRyuYbRjd3TSTpSLxE4LL4pSAhPSfSb4/Z1cJ52DhCg3mk1 -xNyAJ5mvDhNfDsvV82UuYB4bRUJtGxKe47YceJl32v0hHGRqzR1xLGMH30aIy+SA -gpOFAzOOCCPkiCkRgutuxwOUJpQdhZnH7ufOsGysAIwB/BoUc/Z0gDI49g7+NZGi -AqJP+EnKHpnygHt6R1PfCd9xUKiv0/GQZchFCnxoqrI2gDH0tD7+aSkaARpbmABP -Pp8o4hCnp8wT2tMh+aW78esW9JaqKxJRYaqITVkm0UaRvP/+RI1EmZoeepszBGcA -KXRy2HgbtJLxZ4XhFyMw/+B8P6tgsAOIw8BRtw2zdKHMQoMEC8BzPGFQ0ixXA5W7 -5+KSoNveD5f8/eQlFKBo+7cyFzu7Ru+BgcymoX/TPU4tBCCPGzYz71cwPE2K+ElG -fvziAi6hvkj2lyEIsvyVxWIs5RaKpNzx3L+xZl/tNJtyON5CrNUMn1Pb1+7HkDOo -2Ak48dWk/8wuOV5l1mWgQNxFyq2WAWZIyB/9jNUkQAGUJ/ZCaV4KGPhMtZ5tH5zz -p4UcKVYiNyjEjDT5SJCv0gEk7w== ------END ENCRYPTED PRIVATE KEY----- diff --git a/docker/test/fixtures/secrets/ca-cert.srl b/docker/test/fixtures/secrets/ca-cert.srl deleted file mode 100644 index bdea1459911d0..0000000000000 --- a/docker/test/fixtures/secrets/ca-cert.srl +++ /dev/null @@ -1 +0,0 @@ -CEF35E60D201A0D7 diff --git a/docker/test/fixtures/secrets/client_python_client.req b/docker/test/fixtures/secrets/client_python_client.req deleted file mode 100644 index 09e134033148c..0000000000000 --- a/docker/test/fixtures/secrets/client_python_client.req +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICsDCCAZgCAQAwUjELMAkGA1UEBhMCTk4xCzAJBgNVBAgMAk5OMQswCQYDVQQH -DAJOTjELMAkGA1UECgwCTk4xCzAJBgNVBAsMAk5OMQ8wDQYDVQQDDAZweXRob24w -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCuluzHsu5QJA1a8qgilJok -NXcCfOhZWOXZcNDBleI/yiNcqFM6wF7ezXalOJMYVISIzuuh8KIqcCvlbmFD9n6g -1Qygub4Yaq1eczAk9VR/kPpgHf70UKhV0KRRJ0LqqSQOBOmvRYfTHAD0uW36q9vL -PEKsutrKK/8i4pSy54g6/ScSZ/k4XySDYbBmJxGzK45qAAyol84lThjlI19SA77S -g7PBcnirUZHqzJRa1iclTBZ6ypUC89OxbZnGSoxXrjl1drgwEPnIxn49QGKrbHDG -183sR+ObFjTQymLkvyvaZXRx609mSSuvsC8I7+jvnykqrO71qMoQFS7e9kBBijqz -AgMBAAGgGTAXBgkqhkiG9w0BCQcxCgwIYWJjZGVmZ2gwDQYJKoZIhvcNAQELBQAD -ggEBAFqbC3KtdSQmIUiAqhDHRdtNNkmJH9jWb6Czz4jgFST0vF9vjJTy/k4tWLg+ -irua3EE33qhQcae+/oYxf9r/F4Aqu11fobpJpX/KOSnLFzPBooB8Dv2FBzH3Hxka -35RSvFYaIj3SI+94rGY9TMCLeSiDSX5iq60m2J+WRWjI7FdW3ddvXAGaJiSG/5aB -Adv4jhN5/ZpmVbYKWo0sYf+w0yMmQSnII6TZt8uSw2t3o27JUMeAgIQFZ5UAWoIw -jF17pzxkTopPN0ecYCRFnQbVwgruW6+umqp8cU2zYpNLv3lIy+oQAsuKnFyuLz4S -XknoTUqNHPL3/2HCHAJyvkZvhTQ= ------END CERTIFICATE REQUEST----- diff --git a/docker/test/report.html b/docker/test/report.html deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 12534a32c7160a463537edc95b5ca2fcbf7b75f4 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 18:21:46 +0530 Subject: [PATCH 10/46] Add licence to the files --- docker/docker_build_test.py | 15 +++++++++++ docker/resources/common-scripts/bash-config | 25 ++++++++----------- docker/resources/common-scripts/configure | 25 ++++++++----------- .../common-scripts/configureDefaults | 25 ++++++++----------- docker/resources/common-scripts/run | 25 ++++++++----------- docker/resources/ub/go.mod | 16 +++++++++++- docker/resources/ub/ub.go | 15 +++++++++++ docker/resources/ub/ub_test.go | 15 +++++++++++ docker/test/constants.py | 15 +++++++++++ docker/test/docker_sanity_test.py | 16 +++++++++++- docker/test/fixtures/kraft/docker-compose.yml | 15 +++++++++++ .../fixtures/zookeeper/docker-compose.yml | 15 +++++++++++ 12 files changed, 164 insertions(+), 58 deletions(-) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 85b5e5e0a536f..0c1850744fbb4 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import subprocess from datetime import date import argparse diff --git a/docker/resources/common-scripts/bash-config b/docker/resources/common-scripts/bash-config index 5cff1d24f6912..b6971610ef695 100644 --- a/docker/resources/common-scripts/bash-config +++ b/docker/resources/common-scripts/bash-config @@ -1,20 +1,17 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### set -o nounset \ -o errexit diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 6793fbe3041ac..b05d226a75913 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### # --- for broker diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index 6868f14d220a1..a732ad60c37fb 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### declare -A env_defaults env_defaults=( diff --git a/docker/resources/common-scripts/run b/docker/resources/common-scripts/run index 5104c1c11fa62..41273ea5b7a59 100755 --- a/docker/resources/common-scripts/run +++ b/docker/resources/common-scripts/run @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### . /etc/kafka/docker/bash-config diff --git a/docker/resources/ub/go.mod b/docker/resources/ub/go.mod index 1723614f1826f..3ec7b827348d0 100644 --- a/docker/resources/ub/go.mod +++ b/docker/resources/ub/go.mod @@ -1,4 +1,18 @@ -//module base-lite +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + module ub go 1.19 diff --git a/docker/resources/ub/ub.go b/docker/resources/ub/ub.go index 47dec76966475..194b8480d53eb 100644 --- a/docker/resources/ub/ub.go +++ b/docker/resources/ub/ub.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index 8ec46258597cb..70aedba34c504 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/docker/test/constants.py b/docker/test/constants.py index 46010920ebc72..7b1787f93dab4 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + SCHEMA_REGISTRY_URL="http://localhost:8081" CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 7b9378299f9c5..0134b55352593 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -1,6 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import subprocess -import sys from confluent_kafka import Producer, Consumer from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer import confluent_kafka.admin diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 704280c290f0d..0d14c21de57ff 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- version: '2' services: diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml index 1057375bf7b2c..291ec7675b1af 100644 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ b/docker/test/fixtures/zookeeper/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- version: '2' services: From 3eaef7a20ce8b1963cec5414e51ae44789e601ba Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 17 Oct 2023 09:41:37 +0530 Subject: [PATCH 11/46] Updated to 21-jre and fix typos --- docker/jvm/Dockerfile | 5 +++-- docker/test/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index ae1cff10a6dcb..e9eea40877690 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -27,7 +27,7 @@ USER appuser RUN go test ./... -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre # exposed ports EXPOSE 9092 @@ -65,7 +65,8 @@ RUN set -eux ; \ chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - rm kafka.tgz; + rm kafka.tgz; \ + rm kafka.tgz.asc; COPY --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 011440694c092..9bf7356f9bd38 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,4 +1,4 @@ -confluent_kafka +confluent-kafka urllib3 requests fastavro From 3a7aa14599de77829800155fb76c0691431533c1 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:51:41 +0530 Subject: [PATCH 12/46] KAFKA-15444: Use main() method directly --- core/src/main/scala/kafka/Kafka.scala | 4 ---- core/src/main/scala/kafka/KafkaNativeWrapper.scala | 7 ++++--- core/src/main/scala/kafka/tools/StorageTool.scala | 4 ---- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/kafka/Kafka.scala b/core/src/main/scala/kafka/Kafka.scala index 69c4906c8d5ab..a1791ccbe0bf1 100755 --- a/core/src/main/scala/kafka/Kafka.scala +++ b/core/src/main/scala/kafka/Kafka.scala @@ -86,10 +86,6 @@ object Kafka extends Logging { } def main(args: Array[String]): Unit = { - process(args) - } - - def process(args: Array[String]): Unit = { try { val serverProps = getPropsFromArgs(args) val server = buildServer(serverProps) diff --git a/core/src/main/scala/kafka/KafkaNativeWrapper.scala b/core/src/main/scala/kafka/KafkaNativeWrapper.scala index d0a6bd07dc0b0..83fb5960b593a 100644 --- a/core/src/main/scala/kafka/KafkaNativeWrapper.scala +++ b/core/src/main/scala/kafka/KafkaNativeWrapper.scala @@ -23,13 +23,14 @@ import kafka.utils.Logging object KafkaNativeWrapper extends Logging { def main(args: Array[String]): Unit = { if (args.length == 0) { - + throw new RuntimeException(s"Error: No operation input provided. " + + s"Please provide a valid operation: 'storage-tool' or 'kafka'.") } val operation = args.head val arguments = args.tail operation match { - case "storage-tool" => StorageTool.process(arguments) - case "kafka" => Kafka.process(arguments) + case "storage-tool" => StorageTool.main(arguments) + case "kafka" => Kafka.main(arguments) case _ => throw new RuntimeException(s"Unknown operation $operation. " + s"Please provide a valid operation: 'storage-tool' or 'kafka'.") diff --git a/core/src/main/scala/kafka/tools/StorageTool.scala b/core/src/main/scala/kafka/tools/StorageTool.scala index fb6fc40e0796b..2aa1e02853e1f 100644 --- a/core/src/main/scala/kafka/tools/StorageTool.scala +++ b/core/src/main/scala/kafka/tools/StorageTool.scala @@ -42,10 +42,6 @@ import scala.collection.mutable.ArrayBuffer object StorageTool extends Logging { def main(args: Array[String]): Unit = { - process(args) - } - - def process(args: Array[String]): Unit = { try { val namespace = parseArguments(args) val command = namespace.getString("command") From 0c3b65aaa67cf3ee8340f52f289b0b0648a9b822 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 18 Oct 2023 16:00:30 +0530 Subject: [PATCH 13/46] Add a release script for pushing docker images to dockerhub --- docker/docker_build_test.py | 2 ++ docker/docker_release.py | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 docker/docker_release.py diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 0c1850744fbb4..89701d8c93675 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/docker/docker_release.py b/docker/docker_release.py new file mode 100644 index 0000000000000..4135da8f38463 --- /dev/null +++ b/docker/docker_release.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Python script to build and push docker image +Usage: docker_release.py + +Interactive utility to push the docker image to dockerhub +""" + +import subprocess +from distutils.dir_util import copy_tree +from datetime import date +import shutil + +def push_jvm(docker_account, image_name, image_tag, kafka_url): + copy_tree("resources", "jvm/resources") + subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", + "--push", + "--platform", "linux/amd64,linux/arm64", + "--tag", f"{docker_account}/{image_name}:{image_tag}", "jvm"]) + shutil.rmtree("jvm/resources") + +def login(): + subprocess.run(["docker", "login"]) + +def create(): + subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) + +def remove(): + subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) + +if __name__ == "__main__": + print("\ + This script will build and push docker images of apache kafka.\n\ + Please ensure that image has been sanity tested before pushing the image") + login() + create() + docker_account = input("Enter the dockerhub account you want to push the image to: ") + image_name = input("Enter the image name: ") + image_tag = input("Enter the image tag for the image: ") + kafka_url = input("Enter the url for kafka binary tarball: ") + push_jvm(docker_account, image_name, image_tag, kafka_url) + remove() \ No newline at end of file From f75cc5fa1f0f0f0c9684057d3b51c66d57e367a9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 18 Oct 2023 19:07:57 +0530 Subject: [PATCH 14/46] Update release script to support registry and add error handling --- docker/docker_release.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/docker/docker_release.py b/docker/docker_release.py index 4135da8f38463..344476a577bbc 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -29,21 +29,24 @@ from datetime import date import shutil -def push_jvm(docker_account, image_name, image_tag, kafka_url): +def push_jvm(image, kafka_url): copy_tree("resources", "jvm/resources") subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", "--push", "--platform", "linux/amd64,linux/arm64", - "--tag", f"{docker_account}/{image_name}:{image_tag}", "jvm"]) + "--tag", image, "jvm"]) shutil.rmtree("jvm/resources") def login(): - subprocess.run(["docker", "login"]) + status = subprocess.run(["docker", "login"]) + if status.returncode != 0: + print("Docker login failed, aborting the docker release") + raise PermissionError -def create(): +def create_builder(): subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) -def remove(): +def remove_builder(): subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) if __name__ == "__main__": @@ -51,10 +54,27 @@ def remove(): This script will build and push docker images of apache kafka.\n\ Please ensure that image has been sanity tested before pushing the image") login() - create() - docker_account = input("Enter the dockerhub account you want to push the image to: ") + docker_registry = input("Enter the docker registry you want to push the image to [docker.io]: ") + if docker_registry == "": + docker_registry = "docker.io" + docker_namespace = input("Enter the docker namespace you want to push the image to: ") image_name = input("Enter the image name: ") + if image_name == "": + raise ValueError("image name cannot be empty") image_tag = input("Enter the image tag for the image: ") + if image_tag == "": + raise ValueError("image tag cannot be empty") kafka_url = input("Enter the url for kafka binary tarball: ") - push_jvm(docker_account, image_name, image_tag, kafka_url) - remove() \ No newline at end of file + if kafka_url == "": + raise ValueError("kafka url cannot be empty") + image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" + print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") + proceed = input("Should we proceed? [y/N]: ") + if proceed == "y": + print("Building and pushing the image") + create_builder() + push_jvm(image, kafka_url) + remove_builder() + print(f"Image has been pushed to {image}") + else: + print("Image push aborted") From 6f64ce54254b985a50b3c5d8d63711cc6b71ab18 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:33:59 +0530 Subject: [PATCH 15/46] Remove confluent artifacts from sanity test Remove confluent kafka python client as dependency Remove schema registry and connect from compose files Download kafka binary and use it's scripts to run tests --- docker/docker_build_test.py | 10 +- docker/test/constants.py | 13 +- docker/test/docker_sanity_test.py | 200 ++++-------------- docker/test/fixtures/kraft/docker-compose.yml | 48 +---- .../broker_broker-ssl_server.keystore.jks | Bin 4750 -> 0 bytes .../broker_broker-ssl_server.truststore.jks | Bin 1238 -> 0 bytes .../secrets/broker_broker_server.keystore.jks | Bin 4750 -> 0 bytes .../broker_broker_server.truststore.jks | Bin 1238 -> 0 bytes docker/test/fixtures/secrets/ca-cert | 20 -- .../fixtures/secrets/client-ssl.properties | 9 + .../test/fixtures/secrets/client.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/secrets/client_python_client.key | 30 --- .../fixtures/secrets/client_python_client.pem | 20 -- .../fixtures/secrets/kafka.truststore.jks | Bin 0 -> 1126 bytes .../fixtures/secrets/kafka01.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/secrets/kafka02.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/zookeeper/docker-compose.yml | 56 +---- docker/test/requirements.txt | 3 - 18 files changed, 69 insertions(+), 340 deletions(-) delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker_server.keystore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker_server.truststore.jks delete mode 100644 docker/test/fixtures/secrets/ca-cert create mode 100644 docker/test/fixtures/secrets/client-ssl.properties create mode 100644 docker/test/fixtures/secrets/client.keystore.jks delete mode 100644 docker/test/fixtures/secrets/client_python_client.key delete mode 100644 docker/test/fixtures/secrets/client_python_client.pem create mode 100644 docker/test/fixtures/secrets/kafka.truststore.jks create mode 100644 docker/test/fixtures/secrets/kafka01.keystore.jks create mode 100644 docker/test/fixtures/secrets/kafka02.keystore.jks diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 89701d8c93675..a841e6f0ea548 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -31,8 +31,14 @@ def build_jvm(image, tag, kafka_url): return shutil.rmtree("jvm/resources") -def run_jvm_tests(image, tag): +def run_jvm_tests(image, tag, kafka_url): + subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) + subprocess.run(["ls"]) + subprocess.run(["mkdir", "./test/fixtures/kafka"]) + subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + subprocess.run(["rm", "kafka.tgz"]) + shutil.rmtree("./test/fixtures/kafka") if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -51,4 +57,4 @@ def run_jvm_tests(image, tag): raise ValueError("--kafka-url is a required argument for jvm image") if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): - run_jvm_tests(args.image, args.tag) \ No newline at end of file + run_jvm_tests(args.image, args.tag, args.kafka_url) \ No newline at end of file diff --git a/docker/test/constants.py b/docker/test/constants.py index 7b1787f93dab4..621aa8f41e84b 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,6 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +KAFKA_TOPICS="./fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" + +KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" +ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" + SCHEMA_REGISTRY_URL="http://localhost:8081" CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 @@ -20,11 +27,7 @@ CONNECT_TEST_TOPIC="test_topic_connect" CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" -SSL_TOPIC="test_topic_ssl" -SSL_CA_LOCATION="./fixtures/secrets/ca-cert" -SSL_CERTIFICATE_LOCATION="./fixtures/secrets/client_python_client.pem" -SSL_KEY_LOCATION="./fixtures/secrets/client_python_client.key" -SSL_KEY_PASSWORD="abcdefgh" +SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 0134b55352593..1022365cac8ac 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -15,13 +15,7 @@ import unittest import subprocess -from confluent_kafka import Producer, Consumer -from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer -import confluent_kafka.admin import time -import socket -from confluent_kafka.serialization import SerializationContext, MessageField -from confluent_kafka.schema_registry import SchemaRegistryClient from HTMLTestRunner import HTMLTestRunner import constants import argparse @@ -69,154 +63,50 @@ def destroyCompose(self, filename) -> None: s = s.replace(old_string, new_string) f.write(s) - def create_topic(self, topic): - kafka_admin = confluent_kafka.admin.AdminClient({"bootstrap.servers": "localhost:9092"}) - new_topic = confluent_kafka.admin.NewTopic(topic, 1, 1) - kafka_admin.create_topics([new_topic,]) - timeout = constants.CLIENT_TIMEOUT - while timeout > 0: - timeout -= 1 - if topic not in kafka_admin.list_topics().topics: - time.sleep(1) - continue - return topic - return None - + def create_topic(self, topic, topic_config): + command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command.extend(topic_config) + subprocess.run(command) + check_command = [constants.KAFKA_TOPICS, "--list"] + check_command.extend(topic_config) + output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + if topic in output.decode("utf-8"): + return True + return False + def produce_message(self, topic, producer_config, key, value): - producer = Producer(producer_config) - producer.produce(topic, key=key, value=value) - producer.flush() - del producer + command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command.extend(producer_config) + print(" ".join(command)) + subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - consumer = Consumer(consumer_config) - consumer.subscribe([topic]) - timeout = constants.CLIENT_TIMEOUT - while timeout > 0: - message = consumer.poll(1) - if message is None: - time.sleep(1) - timeout -= 1 - continue - del consumer - return message - raise None - - def schema_registry_flow(self): - print("Running Schema Registry tests") - errors = [] - schema_registry_conf = {'url': constants.SCHEMA_REGISTRY_URL} - schema_registry_client = SchemaRegistryClient(schema_registry_conf) - avro_schema = "" - with open("fixtures/schema.avro") as f: - avro_schema = f.read() - avro_serializer = AvroSerializer(schema_registry_client=schema_registry_client, - schema_str=avro_schema) - producer_config = { - "bootstrap.servers": "localhost:9092", - } - - avro_deserializer = AvroDeserializer(schema_registry_client, avro_schema) + command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command.extend(consumer_config) + print(" ".join(command)) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + return message.decode("utf-8").strip() - key = {"key": "key", "value": ""} - value = {"value": "message", "key": ""} - self.produce_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, producer_config, key=avro_serializer(key, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.KEY)), value=avro_serializer(value, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.VALUE))) - time.sleep(3) - - consumer_config = { - "bootstrap.servers": "localhost:9092", - "group.id": "test-group", - 'auto.offset.reset': "earliest" - } - - message = self.consume_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, consumer_config) - - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - return - - deserialized_value = avro_deserializer(message.value(), SerializationContext(message.topic(), MessageField.VALUE)) - deserialized_key = avro_deserializer(message.key(), SerializationContext(message.topic(), MessageField.KEY)) - try: - self.assertEqual(deserialized_key, key) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - try: - self.assertEqual(deserialized_value, value) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - print("Errors in Schema Registry Test Flow:-", errors) - return errors - - def connect_flow(self): - print("Running Connect tests") - errors = [] - try: - self.assertEqual(self.create_topic(constants.CONNECT_TEST_TOPIC), constants.CONNECT_TEST_TOPIC) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - return errors - subprocess.run(["curl", "-X", "POST", "-H", "Content-Type:application/json", "--data", constants.CONNECT_SOURCE_CONNECTOR_CONFIG, constants.CONNECT_URL]) - consumer_config = { - "bootstrap.servers": "localhost:9092", - "group.id": "test-group", - 'auto.offset.reset': "earliest" - } - message = self.consume_message(constants.CONNECT_TEST_TOPIC, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - return errors - try: - self.assertIn('User', message.key().decode('ascii')) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - try: - self.assertIsNotNone(message.value()) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - print("Errors in Connect Test Flow:-", errors) - return errors - def ssl_flow(self): print("Running SSL flow tests") errors = [] - producer_config = {"bootstrap.servers": "localhost:9093", - "security.protocol": "SSL", - "ssl.ca.location": constants.SSL_CA_LOCATION, - "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, - "ssl.key.location": constants.SSL_KEY_LOCATION, - "ssl.endpoint.identification.algorithm": "none", - "ssl.key.password": constants.SSL_KEY_PASSWORD, - 'client.id': socket.gethostname() + '2'} + producer_config = ["--bootstrap-server", "localhost:9093", + "--producer.config", constants.SSL_CLIENT_CONFIG] self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") - consumer_config = { - "bootstrap.servers": "localhost:9093", - "group.id": "test-group-5", - 'auto.offset.reset': "earliest", - "security.protocol": "SSL", - "ssl.ca.location": constants.SSL_CA_LOCATION, - "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, - "ssl.key.location": constants.SSL_KEY_LOCATION, - "ssl.endpoint.identification.algorithm": "none", - "ssl.key.password": constants.SSL_KEY_PASSWORD - } + consumer_config = [ + "--bootstrap-server", "localhost:9093", + "--property", "auto.offset.reset=earliest", + "--consumer.config", constants.SSL_CLIENT_CONFIG, + ] message = self.consume_message(constants.SSL_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) try: - self.assertEqual(message.key(), b'key') - except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) - try: - self.assertEqual(message.value(), b'message') + self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) print("Errors in SSL Flow:-", errors) @@ -226,12 +116,12 @@ def broker_restart_flow(self): print("Running broker restart tests") errors = [] try: - self.assertEqual(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC), constants.BROKER_RESTART_TEST_TOPIC) + self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors - producer_config = {"bootstrap.servers": "localhost:9092", 'client.id': socket.gethostname()} + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Image") @@ -241,7 +131,7 @@ def broker_restart_flow(self): print("Resuming Image") self.resumeImage() time.sleep(15) - consumer_config = {"bootstrap.servers": "localhost:9092", 'group.id': 'test-group-1', 'auto.offset.reset': 'smallest'} + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) @@ -249,11 +139,7 @@ def broker_restart_flow(self): errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: - self.assertEqual(message.key(), b'key') - except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - try: - self.assertEqual(message.value(), b'message') + self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) print("Errors in Broker Restart Flow:-", errors) @@ -261,42 +147,32 @@ def broker_restart_flow(self): def execute(self): total_errors = [] - try: - total_errors.extend(self.schema_registry_flow()) - except Exception as e: - print("Schema registry error") - total_errors.append(str(e)) - try: - total_errors.extend(self.connect_flow()) - except Exception as e: - print("Connect flow error") - total_errors.append(str(e)) try: total_errors.extend(self.ssl_flow()) except Exception as e: - print("SSL flow error") + print("SSL flow error", str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print("Broker restart flow error") + print("Broker restart flow error", str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestKraftMode(DockerSanityTestCommon): def setUp(self) -> None: - self.startCompose("fixtures/kraft/docker-compose.yml") + self.startCompose(constants.KRAFT_COMPOSE) def tearDown(self) -> None: - self.destroyCompose("fixtures/kraft/docker-compose.yml") + self.destroyCompose(constants.KRAFT_COMPOSE) def test_bed(self): self.execute() class DockerSanityTestZookeeper(DockerSanityTestCommon): def setUp(self) -> None: - self.startCompose("fixtures/zookeeper/docker-compose.yml") + self.startCompose(constants.ZOOKEEPER_COMPOSE) def tearDown(self) -> None: - self.destroyCompose("fixtures/zookeeper/docker-compose.yml") + self.destroyCompose(constants.ZOOKEEPER_COMPOSE) def test_bed(self): self.execute() diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 0d14c21de57ff..ed06c819917d0 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -16,7 +16,6 @@ --- version: '2' services: - broker: image: {$IMAGE} hostname: broker @@ -62,54 +61,11 @@ services: KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks deleted file mode 100644 index a823cf3b7ca0fd48292238f98cf99d893e6aa067..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4750 zcma)AWmFUlv)*00yBnly7o@uz=_Le~mX4*HB^8j65Rhg;KoD3`LIeaU>5vXdK|s2% z_uPBl@7`bE_hZg8^E@;2^Eqb*j39jl0-}Ktq)#z%c;eLKE(w5Gz#;@GJ34}t$W+{8w(|JrC~R7cqg@3nOY(ktEN*K!K_n}PyfsR9A` z`;tZ#bH?Bq+(hL+Bay47(T0bCeZrZJwti&jUk;;w>`&UO4i2A^ANHX(2LEgckmZid zzU&v0aZz47Z2IhPveMZ_|4=*mCifI4DDg_tRO07t3VyOe5`TH;&QZ0gMUEkR>7cG~ zm4KeP%#s{kWU2J4MQof9d=hf<=Qjnm9NNj2Aoc}|lm2)e0CyfV+GHr8zBji;m#XKrTC7*4EkH0>#s~ zXilCT@C04lALcg`q4qAzFWxpZ-j*xIoxwEkDPTh9nF)e=a;wqJrZWri^*2j;$b$<5 zmHW^x;}r2wYeK4V(cNABGQ-SK*LL6OZCw>p$)Z#+T4|X4i zP210c(536xa;P+$NYRA6U~_T-BlQ*!m2{XBCvm5~8i|{h9x?4(Y@+Rq*@2i=r0EV8 z&RQM`S@umi9f~4O%lolqn=HNqQo9@b6GM~%4esT=rMQO|TQc@+NrDHPPyg5w-$w&{ zx3Jr3z6Lt(rFS%SsJzVN$Zhv#sy6f(_ z0ml+K(U&jP48n%I|eF;yOYCVS#$o}&yWjerzEx?&6` zag~U@6-S#CdDm7FY#t09nbhinARAXmvbwj%8?cvAJaJbs_GJ2>bS3i__J^XLof|U2 zo5b~X3Z%~jE<(5$fw=2Gc1JXVP-}WbLwSwACYe!@|JD>PIZ%t{aP2q>EsyL9Zbp9 z3}?m&=G5lawL3^rlfso-1tNT&4{_jMK@aNEtB8^C150h*swD#=ogazuK^9xa3K9us$K?D|fk$QIii~Jh+`5$kd0m-c1paNhplCX2y&S*`bUAj~?}s zu(+D4SQ$uF{e+-%t{g5(JUHI2+LfBP3r@&JH5Hv^EkWVXZhJ4@o zRuKMWtBtUM+tH{Sdl@F0YO1*!B{Vi>`Qu}YX4L|%Ae+=E`}FVmK=(&??qkI5CLRL<6kgoL&+cV5uu)sLrB2u_1WRBIbAJKMaW#aQ~8q_S)*#E zp>JddGevqstn6-pPFIS>J6~6Q+zqG)gQp_Jjl!F+tRGuDW|;O`y_$UsGX3>oaf8;4 zopRD&y2gs_L-#Ot;g5XK)ICLFeXLv&u#SM&bo4uuS?O4;HPp`jnXm@wN88zsp zo+1A3;!7fD;<;#|KT$3Acfh_4TChyn-dSGRrdg5yw`W1Gr)th!_bslWGu0%NfzN=I!5&ip(iAv4c?H{b0v-v`vJCoRHCi-yNToT+HTa`Hup!ADXzBt-OS zmc5IYxG99VxYXt<-2~ea$;=>NnJKIX&9SIsfP6b{L@NZ%ObcELm_qgwZVD^(goh}d zq@lIkyfT~x6p4me&stEUF9#;Jy7*7IV*Mx$~di= z6!p6aZ#EWCblm_*y5)`hr^QFw%1{Th0~JdBh-dSwS*(;Yw)P>qba3H!%o(+U=ZQ2i zl5jWm*X?mt`#W)+qj~~G1)qJC&YGWxQx@1bkV|v(l&~QWtqeAzQsiUvd3@hSYWOlI zSsVxw>3y!zWwNx%tn58aswz&Aor$~l4YS){LGZcwF3%jd%#VbL$gz;9d3S%KrDwH( zy*l}U7u?HkBURdE_T*#=lI6x?~_A^q1UPN{e6y_4H1 zTXLf96`MU4G!BIN18kq!6um`6Y#SwnEDr8Ud70HQpq7k}NFDFgto*6HuAuz+rXT0& z?D)}E_S$#c^vftIdn?hz7vLlQ%k=PKE7ees4+-Jv0Y5vl$MaWWwiGdNCnWLWE_o=4+i#Li9pK|}n)Unxq|lDe zrmb7$t}6*Hkww@=jgnCwdp@9GkGA44XjAvShQW#_2`$iMeuc-rmpM}!&?1B3W~xn=N~A?3!czwxm>T_cl~(e zB4~Ge{3l2Cou$4&uSFBTBeY`jII6O&iWS((2ycJn3n1!YxEU}A*xTRy?Qr018CrmE zIHjW(O7xqquuah0=|HZU)4voOY;yX0vT2bssUIO1Zf+tb2xCTxE#CK-`<&;IeVgtIoRSs*3kg`^J}E{+Yu(NWSDexvQMJr; ztFSp-9>d5cMs8FkwKGqp|LWCTtgJzlJD{r>^KfM5Pez5)(7P774h_hoJ=eE}>J`%n z(T5+51)tP@9+#L{@EQNYwIz92@)4-v<86U0d{OZGel01gEfBM5EcoVx16HvLp4X}tiScZKOF6_@%o?ADd@BN%PCo4HLvuN%4vL;McneKqY2olC)U7BM6ldn%(f-3)qP$10#&AH_IADQ8vTmSYK&`mu=zdGPPtEq-|^HEa}L zc*d)oCJyb)b+eP4R^1KW%H)RGKUtS5$UQ27FD98pxf6oOPi^D7KlD=SE%ez85+JC~ z;+OSeI;3pLEdHka zj?8Vt1ax-r0wj0A=|d4`ct{)2QJk^kv3K-Hv6ytYIN+tV-gnc_-q3CILOEZbLrh3B z;=^&!vXeFYBUh)Rk*tdxt%Gap#Va3umEbXEfZkP0p7KJWhXH_K%l>Wf7jr}EDFPC_ zF^mrjb4fK#xMw|hi227X&F$iC8A{O<_xKsn>i{^|Qn=LJsA|GG?@a17%Co-uH-RdY zVkBSNR?SzXT|`LerP2Z;qmaHRgYhj5sVH{3cWlJ#SJY#Zr4pA=p?gKXE%X@A| zXm+C+>#wAd_b1fR&N)aF$~X7<1C49)mhlOW1q}3!Qzz!G3Ov%Q;yYH&pSK#D+z)I# zm*`Ur+v;cAGP3%DgIN7TU(2H<8xX{R&sd|Jgz%me36s0GDHDrg6~-2ANTHr*>imT= zlKe`p4|8ZNyG-F9AXzL zq72pYB#=&68CBpdCveDWQSCUd73NsVTBEW6>wqC(f`5KAARq<+o!Rx-vx&AZ7mdTK zECEq&ubrRCwqIMx?+}kGE3ToLB^#rYd(20o+(x`q9_{K5ti_RK*Jja4|9w*Y7a0`6 A#Q*>R diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks deleted file mode 100644 index b526dd149726d5b5f14d458b9855af077c197cd2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1238 zcmV;{1S$J4f&|h60Ru3C1bhYwDuzgg_YDCD0ic2eZ3Kb@X)uBWWiWySVFn2*hDe6@ z4FLxRpn?QaFoFb50s#Opf&@nf2`Yw2hW8Bt2LUi<1_>&LNQU+thDZTr0|Wso1Q6)P*k=lo4{ribAXc|spM-#d1JG88u`XIkMH^{>lLExi+PFW^ zB?pbblmwhZl95UB&R59%mp?wOpl^w#7q&J%nb0|CPS8hqe$+89sZf3p`|@?MQggrY z6-TNTHFq}tQ4)^G7zuG;HGzvqC@!I48tfp4qd)UU(Kj6NScU}N?a>JNhW1c+A13Kl zQ;Gkn-)StuoBuk`kEidVAYIfab!|_{$!9`==yeKm56pr&aUOF59w1g@6CtZ+DLoPF zue;~{vjWt-o$4Z?bJHvPRbqi_>=-n=5#sPVYd%jPe$41HQsXhU+Sz21vR+BYQO*og z{6yh#t00ubb!pV@S`;iS(~urt!&#{-R) zY62Zj_OkN8`>Va1GLM*izh!bI!4gaCL_1FLPyQ*?6v9jUuFUufcLBhlS}(A@IlKKu zxMC+TOY`5ZsY9tu+anTqy{8^YXu^z+V$7P}i;;7K^m9cb@c>Ck;2sj0qlWW5CJTMM z)X3$Z@7WeGAxL$Q2f6>!EJlv>3mN6Af6F_-2QmB=+FecQu=LTp8ZHfZKUQWYn&MqLMb6rJ=V3r9OK=UbbQmJ{-3l~6Cid$41Y%|!$t z>wLkiAQWfy;H)DG$1dv?2aHy|pZDjDvTa-LS`@X?N0$H7Ic%{;1PWaE^gcvROeZtW zg#_N~lac)VV99}H)q52!ikbFeIFi^-g`H$?$5hFDX6FYs(*p+)ZPdjrK^#l>yt8%| zP@5^P#;gUlV-@??hv}0MAKcMiwYbt`1+OL^`n`@aJZHb1n> zCeuxysJaF7)iW};l{)45V;DO9Wx(`wo;=YX!8E~;v4pWK4GjEI-@(LBYo_2SD9Nb7 z4BU-#b_sGTJarFflL<1_@w>NC9O71OfpC00bb=VYHg|wC=Y_ z>K$^fKz=nohNEb?2Wq)Xg7;r0_z`ae6bHMGdOoJ>5YY_$M+66>H8dkc51?leY6p)gJl@gF{L6Me@ z>pl0L_q+Gk_x+gj%skJ`{Cv)tfuhJGfmqm36!{|@e7;cS&~sueT&x@vITr{;&i0pP zf}+3;|4#xJf>7YxzjWr`NW~%i-zg#jEMN`_Ec=&~g4+Bu0VakTL&g795<&Svly0Zr zvpr2A6+;g#xs5AXJKP1FP(UEh10WVFln4j!e>MW~!2l=|4*u&8WP>$fB?tu?m+m8Nn=89iSLpKUz&~ij?5t76%_(NzbzwB%YQVh zwdy;plk(XZ3hlNz4nt;Ffg&GXnr+t^nE0v7Y#f=U+1}1)1q1}CzM>PRuk#fiBV{e# zhUag3xxA%cEFH`0@Hraj==T0WH_J)mNF*;%5$j4z*(`c}UrdgWaJ?&0DcC7#0P!HH zWRJ)3PB9dGi-Z{-HuMCfXDZ>;p2P{G?-jj*y^ZcszuU^ge=(1Ifsh(6y-XFPqnO9) z(@8fwqwgu-S6n|EZYaYl#{8oG`07aXeD<@~S=^cR4kdxWO4LK?$HgR`W~7Ez5tt<9 zM12j%2o3Mo7RL%t^!8f|3x_fUqpN}|9X_S696t4`o|;rXXv+++RH`y4q0j>}$XPPq zFv-rLcGg+fTN_UeONYgD%g<8Rt~_MY1CBZ^hIYO7-iZrQNEgdGr^@tDslHm~>h}y+ zS#o3vY(cML=JaS5)Uv4BoL05WMKe1>;R{@biNRR75h*7C>FX>$c!^6ma2&~hWm_$( z$q+Th#(1f`+2bpWEO=b7ZZ%9O=ppW8KCwn1!Pm#$5^d-SN{}tHu8r%%V;r!^U#*~p zG=0nP6*P4$)vJgHj*jgJldsVEx;sAne0_B)gMQK&ZUx{QTvhV=VC`grNT4kkdPesy z_GJannJ59tL2$m=#5@F7!dxQTrln60DJZz(j7J5ZY6ko5U?KVHRim*^+KUpj_5 zDf@B#W@M1~8Ct$jjp!zqXz;*Z?$34_$BLcoH+9`L+et>9G?mMl_=Mhc_s&-GeBeio zO)jO!y#ZaZi7lENy=mQt>7TEy37$my%$)nSaY0aQSj6b|6`^iuW7xb|N`nai#*urn z*PX#^zq4meWwd1>WD|^gom7RuB$EXG&h2G%waS30{O%5_U3+x>ZdCJdFuOllU%gd| zsG39SY%rDy5w(YW?wnwSRGR$B<8cJH17Q}X-&fOq&MS9R$9cisk0+3k)*T&JUs0FY zhlgA^<+-;OYh2+E=^$l(swe&Y&uuSxs`nkH>!eijTJ?|l_n|Bvc5gBKHDIjqWT&_= zTRtI22#!&df#WOPCx@90{aFl4Xqq?c2X(ixQNXRa5t5tb_Y~JC1^3w5^Zp{!#b}Cc z`2-s-FjEcDj6gFbgl;vvtb5d4z*(1}IIHY#fPek_+;w4!Ws<(EPg^Du=dYUaGos=`nap8z#sUQs z-vdMzFGC2P<~ZYsarx&SI1oIilKgTLA8FEi32xXt<~H3I!~3Pw$cH1Iry&yw z4t{bqEye?yeKpfLy<#k-`i*v*$rh@TD6c&9rm>#(j%WJ*q4_txsjP1M6=`(dB?b0} zS{vANDBC}FiBAcmWF`k70L}nQfcamv`UJUij?gZWaLOV1S z;r9JtZn4=%_vw9R#!W5sfhfZ9G77)pGTWI?eO#qj0#WJ%f|8Hv9X+_{Dc`D$zlb6+ zu@a6oiI6C$Nd)p=LK1tKaK)dxw(340C!{AFRG@<(u~$z3&P2WxMg?k;l&~?=d?U$V zUwFyplyZ9RqwlLL8#M!G{ey_K3!lhycg4n$4GpYBeiECdRggWkKQ8xjRf;PQG`*z_ zwkR$2aSn$P2YB*a;Lt9-rXx5RPDUuhiJtnpX=|-0A`h7`HSEU|Txm<9J4^QBf^&xY za-w?&j%KLrPkSkIQg{eO-CB^R+q5#b=hF!p>GpWyb3!}*Q=Vh+pB8wpxf7{Wu}W_C z@m45Gybfy932i_7$)nQEd|rSa>DY8qrfn<8N5qoqCwOM%YdrvqCE1>CU}22E#nq2| zoW=8qhu56F-efSeNE2*v^B2FdSicTah&U?uTAZkp`N<7@iV+wH@=uAJAEE1w)!bU0i zelT!Xz2LsP5LXL^o2(@`H)*A*@ny=5thuko7d5ZxWLhLTl!V_Dzf9H~P7n}9ZyaG{ zQzyjqlqmCs{xOkSE6mX6fJs+6OL>2RqE9ea@ny=LtDcxnkqj6RHg3m7coY${z9ypvVN^eu_(b=_R6-cq$BA7;PbmY7$EISZ{25J2$^_3J?F!4neEZNLsgR^YjFV4@hSzP8Ka z8#rKBHkPWsd8xJ_4a0x8O~>(P%)=;GWh`${dylKAxV0I=-4GpW$*glhKYgYs<^AkJ z&nD{&%C3`OAX5I<6K6onx#;E_iHB_iJRjHdn?vQRR?2!FFG>q67S(jUlZ=pQ&2{h} zp00_GHl@C;Y+YCZ&_1%BGBi4Nk<3;$7#c^Ue&KXY*shLoxCT z!u1-o65PoQIA>smfmX@Y)I%?m%_+I?5L9V00O9h;-quPg)3h+p78 zV)fje#nw_ts3a|1Dj38B7+LYb7_r}kc4F^H7(J~V3CT#RIGuhvCZsX8nY^L2YJNZ# zN%VXf%FRWrYn|E;_5O3OuvsIdP|P^ehfI2r-pS7LRr`QMvqJ!sY_4hBh&_=O)Ke6^ zR<#?l=pmsn-<8KkU}#;({l0-65fyh3Z`+#UpqQB*CE=r$(v5d1hP)ty4T_qfl>uLv zs$(jJEY{MZsvL$9NaDsuPKU#nr=wB`a}57I`a^)5`bO|TgmT0UN@eSaTg)`nvG>Va zgd-9u8Q^>QX0?QOoQJYVt_t?d!8a@yu{_)JgZC#xAC0{%+qmw=sjS=>KRY`sOJOnM`Z=zwM2vrm8u(#)X zFQT71^M|CCP*x)WyLqPs>;AhG|ETsWD{x~iEW3>`YmEB z2Nm3oG8XZN+m1JGf^35JhDxdL>}hH)a+KcbGS_GLGT@`X`LKj7^`JYgv_k~flfK}e zI8`b>DEri({^2m2RzR{{x2-)Vpm8mz1wAg0g#)iCT*f6TI=EGKT1ak7SN2@}u3p(0 zgVx|>T{9t7zk47I+cuOIw06Y@Xob*A+J3_4C@JoryeUAObGx@Sa0`jrkPg4B2FJ3l zAjZ@`RXIs=cp6(sKJoVN8)%rR2qQ>6EpC##m|$K8ZvDBy@eQ1(cOna!wjcf93|6@b@TisdL^kpAU&r?6!s zay_TZX;JH?z?!jaZ06hDn&%Ue@U0XELisIxhH^5SYqJl8w$~<*X%&nMZ8PXf&BeWoB_|&m&B$+&k#Jc2akQp^Q2My+HP-v?ro~s2 zEl*}inG}~qzW$n>VPi-axYbrFoWj7RPxgLLrk<>(Sa(i4Ui5IC1SJIK{p1m13BR2# z$2(wd&*C6)-sPYOC5>qs@f~dPiR9>yfgUpW&D^CgC!&cvQ6@zxk`EX3$z#ta_Qy@OjEUf!g^w_!gS!n79dosCH%gGeE%PA#`PzJRXWv}|SuPcW)Asq2wIumCa z_;z^M^;_cZqOc9#*Jf?GaFgVV!%eD+lb9_w2)?J9Xqp5fmNjdPxM@P)iG^lVZ?kjO zqo*gA4dFHR0%5(z1&!3}Y?evg$CTzcO z%Gz95Ig@UG1o@*@#NmXf!9HQH0KgMjH z;FC~FRbGg8pBY%u=3`k%&*xL;AvLL5qFLK3+)~l3Jq$Gb3VWOsb5*) znG`sC+SiP;$18c~D2^*;&1a0V$n?B@4&slq*#2VUB7}rfPuSoV`)Ba#z;0pzwgpZ^ zFtmS3DJfC#@uD~;-z3~L1`m63+9>()@bhitI%Tvzf#Y>maXjkeL6tD}q(HHN%*UXp z*{#ek4KA)_E?4r!s{v<*&rj%|Fk(MsAT}(;GN9MleA6Xv!1d6mrXz3NxuP)L6b=Du z?x5M@ao zn4;J`%P-xiY-`WXde5@&zGRcsoo0oz6@?)Yq3+?{ot_pq@^aM??=tp47moNoNQzC- zBcXeH$OI`k$lC93O<8#8dDQ735PDy3PDUN-DyN(gb?3+J#f4R3YQ}$&P}nA#T$?FJ zR=OuBbqJrx;hZkic$fwt(@#{Hs><#-2YhS@r89hKDf3+h)$-P=yEMZ>K#G8kzvt&w z{O&4Kys1{J`Y^6jsFW{u{cjG{Cdl>0%E&950Qopr-{ec6z7=Rx!($!#731jI37;f& z90+=yvy~oh^D%kineRO&LNQUCc5!rUdCMV@CO0{ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5jYAgfh1*n&#?ebGsJHYtFD1JGFzj_}5R)2_-ag-=_~ny9w}M8;}F zDKA=%k;NKXcVz33DvTJqV2-?frBKif~zW%qBuAh6lp?@njnlUH;-<%m@7iIV63WOQ8hiY zZ0#38umsN<)9g#{C(4<@`8M^hLu6q)QIoR{FrtvATf+#49?+MB|Lwmu!$nvjMyY-O zk@sgxe1d&w`Xi-O=77DD)!LT9{svF#!e`tue(yujIOBOHJYrxbEJsQ?D}_9FlI0iIeIJOSKK;Twz2W8 zg5S85RYijnsXD?@af8pubU+qrw+anAL~axgG*T$8RaCYFe^} z%o)5U$KOAGzYU@`QbY)*TCCIvw6-qb9bA`Ky))GpTgLi!G$bcLe?B+*#YEID~)LQ4I{G;o7 zs}A$G>35i-vq1j_why`uBe$NsrxA3JoZZnJ(CmE4qHc~6cnGUeJ zJfYcIc!#@{pT;)dzIp(F?mXWEC8sjoGb>h^s63!~lNkDWtWFg%C(w}Rk|BW@-pbTN zI9E~88JAM|@2N$p3GY;2whx-8Y!!E0q_b${Y-B;}=yAKF^&$#L;oY7dg9Js-jrx#) zDxUBOGN_{BkV1JYZIHpk@yS&ooA^_*k&MKF=r!=Tv-wCn0!$FupntmE;oN4cMe2L- z?hnOTGrapoa+p(Yhb`<-kR=^*4}^HUA;z}hT;BDVcnqlK*)%{B%E+9r`#esnNRFflL<1_@w>NC9O71OfpC00baQabg^ag;b(MyQlTa@T5qW7p#5~96&lno(> z&gwO*-`sQWdB1yqecz8cGxN+m^YfWGgA9h!0|^L`!IV89Qm$~7@ZS&u5&}#xWg}5A zW!)dR3K>k?@c&B0MMS~G`G4S?Kb8R^`(GA0m;i_gCYJdFqL8-#Fo+>YBc#~BQgS3W z5uA`<=iSaI+|Qpn91^w9;H9$eWC{dwPyz{9kmR5{|1%LtN(?|Uf=JE7RS0Ydi3qre zD8Jlla?S>EyGhg2GIXmgdJ_c`eWS7U_!%FstKy<*e=&#d63~S6q0`!_6)3{SU+sM8 z{LX<7N<+n|;v?;>Owp@L)p{@nB_=7BEG~_=FCpmFzg?!@n&H!>ENl>#R||}H1KYZ1 zjKZY|&g1oHB}whXyH{mEbx%TQJFje_`OL2zYGE7~oYn5GhTy(+?28)jum4K?0SY%+4X zYKU+1SFK|vnbV)Qdt2J5DUsSX%}hgHi~Cw9ptt;e)3fy~ zST`~Uz6aPg{F)^aVERfG7Du$5FtKT;sW!v(d}Z*Mu=}-UCf%vHG44Tieda4vOSUX+ zD-rudX9-hsslf(j@<4cZ`4mfHa~cxR<-}EEEoaS0#Zap(pHZQ@5&YzOC|9<8%`;Zv z$uKjs>=t7(hM4uM?aNp|o!`7Esi6be2E+5@)(|Z@e9q@g>F|vesGuXYKsV@rp8f{*V#pUhXwG1y#eb2=$iE9{bgi>qs z8fCC{Mbxx|UetA_(IR^o0vcrxX zdGq0|tNq%Sd&XsX{$|yew!0Wt`W}mHMuT~Z* zHqFOXk0WNT7q7KLqBo@hz9ukp$l0YhGrQIcSdM*#@%^S*RNhT6 zFx{Im9_BRnvJ{tS=JcWq-$N0laG2kolZ}?NLd;u7qFo@Iu3;w8h5Q|fd}C8!Ku1oB zX#;JLmUW5Yxf>U!J1v58zvZOFoxzNo7$|@%(s4SU-~>PynA1>CMM`enETSf1m{*0pO3D{#S+I---1xND#7-j zh;?&#&2bIG15Ot0KGB@OudRW>U~|~^Ng^}>6xYFV0I;aN*b$}&BPyIELfh96f>Er- z{uFmRH|m2VE-B8wzTub*B@-eA8LFS|ziN}AGJ3drcFGue())b9MruK)zfjiMZmpiS z)L-E*Ogvy-ePnC~$)wmRB-Uc_Sb=xOr{;OSjUL!Z#{KV>SQDgQLyom?#c2q2h0Lfh zag5#wpH*_4zaVesS0Gxyxez!ni46|$${&*Helu55q8r6SWp+oew}g&^D6EzahrNor zZ{fNJ)G5dgcd44B9QcB>(N8mZLRGGl8i8){kqc)PuD>?iy_`({TL(0`>hXBzZZ|Yn zUoV^0AayC?D>^smsK%blqg$aHiEaGVoWZ%HbZk-915 z>R0!i6XrTIF>W33mW&H~hgBWPO^P0J@HnI>U_pyui^eMFe?gmNV;WS z*3sxsGlke`bM-V@dvSlU0r#*ezU-3%Y5;09IK=4?(j2tA(`M`OU4F=U2=f_(z_Hp#V`{GSd?A1CH*&aMkP`9yy zyuQa;d|9$0TAyFS^IVulerppGCqKI?VZT zn~<&J&Ws+T|C3=DbD8BsS;^7UvR#c&S>uv{S7YmlqxOrQ#NG~-Hz+8$hfs7JS}()2 zo#-qKWgmoX0H_i7GE&pBTo2Nyb0%H+QZ&9rhYzs-O_Qyk{y-hQ^+9xjj3RWX7ayb* zv8uS@Q=+j;Jpw5731#{=9Q9O7xf{s{A8zm zW$Jl#B|Y{XXrnaHN3Va(c0oxN@!pS$!T;%xTmd6&@0|iF%BqvQ&{j6|V_6!lL9Zu~ zMbxa=1ehWXnl_8NvuLVO|1VQr&lAFV--nl9oTCPfzu;8Dj88GkKNip{5Cv4WG%e>LT7z9b#1Da z7tUkd)1M1oVq*qd;!lVxtixNKn?~JadrG4Iap*i}5hgsbuH6b3^2<#a%M_0rbY=h-5X4Ctn};1 zrL1HJb}i=egf)#r{vFkN=a2;Xrx){j#lfHl?wcx^9fDH{!VW*?yl8(c%ByX^J~rWK zHZ6Ycxf+w_YTElltz#?hH}9On2zfv}l^0vn)@)x_buNJu6XoorQPi%v3CkJDq=ndq zhGq7U*WSpEW*#i39QW7Wk&MScv) zw*WC~cDWGIra>KOyR0{@?lhAM382z zu&xKq<)`mb-F+vxrKi2><&Ll-jRDnxMXDt~FfWfiHVb&U+h@Ja>@->+9$pZ&n&12buM_;}(#3mEAL!=nH zAur~j8b|>o&LNQU+thDZTr0|Wso1Q6P#-ID$KntFh&I*VUrbi;sx17MWv7jr3h*nh0lMGhuY4o^29 z@Y$tU;vmV6U@Aoo`nii%ITe%x2B9$+n+0pVr!;SWnQL3wi}J2`Ves>mGtl50reI>8 zQoIa@tkx`gd7F0B9LLuFm|_+PZ6xQtp!dBXiV1e*I!P?UAnr-XfTps;$^ZA@e95~M z#6~`;_V1rSvqvk0cY8@yKg89TlqbuDdOnvILbJpX+>kV5)dD$%eOxZ%`xV_)Gh%hS zI1B(;MyianAp3Z{rEf`mylz_V4SzC8r(krK1g~Y0PouSZ?ou@{Mc&u_cGF7K>U2;N z46+Fi><%2!voa)&)Xl8`JC>LcU^!4E%=&@c)z@{&W-WjNvVQ~G983uO`O^l>C>`|v z^d%QqhP!RE4?b{6cL)_`Jc{tb75z3ns6ll-h^jaho_JkC_*}1wI%nR`@-I{|kv|e? z?SVDE@tU8W#S#pr?da3vlxRUHJD@PAy{T`an$_czlDnDD!D(eNDF7Cp(LEV`4-(Pk zP}&Se(j+3Q2`qhU-XJf!vjN6{FKzSQAdh#afc3zQYW+0tx>Tc}jG{MZR<6ood6lm7 z`%9|u)wX1&*kVzfrvyTdw1(2W>rW>Vm_7a`^vSD06tVE;6_KX7K$8{IrYhqvq4jMeO@I`eY)xC9J6o357PJ*W-T)F8yb2!5j!#)5P|FB@1yqtZO752@XgDIR48D(rT24<0%EOCIWDJm9k&??MfWDNC9O71OfpC00bb^Rm3+UnTTn#OHPE%<$O)N sHV-%uzF?3Jgj~ii%ky~z6v}D!W84x~O;Cx#kpePducmy3&H@4_5QdEUu>b%7 literal 0 HcmV?d00001 diff --git a/docker/test/fixtures/secrets/kafka01.keystore.jks b/docker/test/fixtures/secrets/kafka01.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..953c3355b6e6d0524ed8029abaf8572b93de7416 GIT binary patch literal 4382 zcma)AWmFUlvu1$>mfB@OM35GdUZlGlTm_`Nq)}=WB!#70LVD?1Vd)fUq&p;}LvjgG z@_Ns`=l$;e^?g6)%*-?M%#Zo;oH+vvrl7?I;K70+-2@;`q$=`)3_u9T4TjX=2SYG_ zVKgilSo{A1E77=MiDt|#Ss?BX;x=&bo*8l~$KE_~dq?McPUiDlwYv|LpB z`8+PlUoGc7xK;e~N)-WIx0Ea}dQU)k4nke?>63l5_|9+2h#ALMFRed=j5A~wiBxUF zVc>UHHI>lUlB2q9S85HXpCSv?ZIp&maEClSkk{+L8;7Z;*!cy4ae|T(Y{NkPv-d)I zfz#^Wq|m1}V7zjNaxi0ah1qVq+G6c#H9raoe26g-#LeHS6DibTXr`72H52ypDkpqr zJm4&}ae1_)DkjiYfn}DSQ%Qf*Z#Dnu(7Fq8n`?6JwBWMwlkOc6u+2cBfSJf=+R~Mh z=^e10+lsP|g5N)~c@Zi?WdwSd6|-^uGjmlc>$6;_W=2o{rFTGYm7*PlYqcf(G||-}SoWqaH}e z7d}`|6O~IB(&Ksh#Mo=Y-rGXi_D~w-BXiLw`{pn%UZO{gb9ToG>ZNlfYsPE4C`Ki9 z(P_{FCi51kDf#K;l_ZdBmV2f`3G0>=CniXIhI#!yl&lz zm_5|F1R^LXmvJT-V?eAbc|hXN?(A779^741*;tbbzK>om$CyyN@|&`9Xg~|9aK-%Q zJU`odP4Ht1$ix8p&%|)dw?f;izbcgPWu7hbwW&vBaU)^AT~Hg?9-pczHo*o1Cc09m5IhXL~K;cwzHf3f05~9X6LYv8*_c#i`kV26D&Z z&a1|aS%Lf2&#N=)uFNRy+CK zA0j=S=;qz(T*zLyhHfpVx!>TpPM#63aO{vf-xdLebDZx2}^ zqS3AIPx+JVb)i;e*(8Hka<1KF0}*1ba6N}XJCdW8MCBg1?xy+r91(6bCW(79 z%HANM7y0^5KxZ>Zzk?k`SmZWm@`=JEg3#`sDeoBM zWxv(v7TignSCM+=MSRF6)sbvkko&yx^bbdOQMJ^wYs4&fwGqAv>q-{d^fuk_(+uNS z3BNLo`5(CiQSnnTKyjRL+;FUMEdP4@zmW%y=l^ZF$4`Y1)pc;OW#ShW6BQB_6c!c{ zc={9;OpN_!4k1BqF!9=7Xc-rP^ViM(s{;6UV!gnmYpJ$zVl$XER8y8+d0~U2O8B3M z^%;nPtt{+7(6o`u$djdqGBB9fVvMLnt0Gm#vHqEL#m(u*wLt1pG{Q_q%I@`wM$gM)m0%p(S(?3_7@o_^s4~fy-YZ*W@Ztj(?q&{mY#DR3Ycn?Uvi{lLOy`5;us`CY{>V1~ z=88kxQaEL|0@-1x%(jPjT3_g)oyKcEzO(K8U%esan^*lCUsrFoL}?2y35^xBoAzRT zBYeRuC%L^*G>=6NZid_+qDE-v><wBg{f*GRn zvm4P8B&ev!z`eIaU3OEEfT)JjX{K==VU`qcdFV%k?vg##nUW#W!&($Xj1_1+N5RGI zU#Nsbkjx6Sa_SA-1e&H0ee6qPfdA2^sKskouV*hGMT=u}t3~8sU(;%6#+*sy9i zgoj8BNDV$XI1nRkD}WVl8c>zLe8;sd9yfl?Z{%!*KjCwC+Vv*0_S?q2$R~jDxjRbp&j63ERR)X#10-2QJTXV5Lw?m!E*{z)5dBJjjqx8L3GrqNb z?1kHRGoB%;C( z^??*G`g97NFs%|BJ>+jQ6L-#h*Nr5w_a}Wf{JuEXb-oVqRE-lkkn zk62V%gsYWaTe{bNZz$43>rSGK8-jtE`^inIy$O{|kmkP9i7U}SXbw+UA>BniDqzkm zwi~j-KJsjt?(~l3YfLoW&59(c*o#@0bD^|;&@3DzZ_uyi(dyI4)JIJs9O3YFSiEvN zQ^!tm*H7ZWV+6nc4Zrp^Z_pFHBgEPQUw&nR8D(sQmcB;~l|=nCfTnrv%P2V|Gcbd} zmfNU@O8M7;h%r~)X(G6`7tk<4tsHjM64ddGb&_9Tw85!er&BCy3;f52dEeLzh4=lC zX=f+q6cKvWuv2qif=+2=h3scy$3rrQpT+Hd<3{;3WsPe&P*1@b4-$?z?qQp#*TlO- z!PIt+yIU#~;f=3CNB`VD^EmfNrUYr1?Sdvh$J~;xwMavueu>OyY%Ky=7O&P_#zisfLMUi)*6=^29tVAAd4(0#ffHw7dLhxfokdz&LK7sAWz=epb*F zyexevC?Wd#DAd&S4B;CXnfKJtE-CD*>~O~$pV1he{q`hzAqGWD1k_AMV*YRjZy8E* zD4R_`%Ydaf+~VuWGZ~Z@nyvcyAr&v}#w*_?Q?Br^_V+_ykmIe)dc?NnbMv#%J}z#F zT77a_JH~C1*ye|WvTd}A-cyS%zs0L-iu9k+&!1o{tQ&eAn|}b`AmSRavxM{T7r(S8 z&U0_`xI53Bp^c0MqQotBf{j{_i>EXgFJondc^}~ta+V9-d3NPn>q!tl$75cgOJP8# z=c1Ch%x&ggE7;KN_NL@tgc~?Xlz_7kO{^Iry^Az&HYOtSTk^IYD=?0^EY{3kFwcrl z%Ml&ZQa$;vTr;{!GQkwO&M+&o-E2WzzHyw$R`^--&wB9rdqn0%y z5XJoJI_7zm&YEI#-Onk|L?(e{)+1xtH7hD4rE9&#>5`Du$-tvIEX zd>mJzV6!S(T{N>mhllnUZ?#h-vLFR}r9a?lZbOILUoe(*tf06rmV37a!;b>L0>8DZ zKGY)VR7gB^Y7}111uyT&?;w=uUwNOGWhf}C5Fiw(M~tO3rEod})j|Y4Wn7KGJh_`s?@&oX_P*z;uKht%8xok` zrmSBA3s3et(hR&Q-OFXA-xNiK9sUUy8O``nSj}V~H^(y;Mf6is7+LUUuSG%RSog`1 zf8FcWF3L3PioK#u?3HVlh%oASrb6B9XwH+8qjtUhZS!Yml_jOTh2{kXM)d5>y=E!f z?bh^QaaS6GbX=DrmAQ0{ctH$UKx>87aVu@5J_|J(+GYCKI*6WJLsAbZ)$3y&AUh@} z59u};1Nze{MVnT4W+e=VlNT5fyJwwW`VR@l-TB%P#Qt$u=Bz3(r0yh6_+a$m*4@DH z9mf$ZuNDXF?DkZ>etM0Y@#yHIB*jOCzy$zt@R?%1-bI|h4LJ{AaDaOE w1TbiNf|J`G2Pq#JlVoS}%Yg8y{4r~NEpfk9XGRIB445e6qbQc#vT59PAk#3L{DX9fST$XMD5v36c=@eKRq@+Ux z=?;nOJ@=mXyZ6`k{h0I2JkQMhe9oMKg;Cwd1LDKND7y&3{81`VXJo)Tz;|JkHJ~uc zs=ss@ER5jG|C0y`L16@Wf9dSMkxoeRzf+{dK)iQh1hRigB+Txg2?8>hF%15%k`x97 z(b)Sk^GDEiFQ7D1!=p9;_pD`tf8ycsP~rhuVWfma|FaPfOaOo}5rQqFRDiblARs@8 za&x*0Qi=6+>XbC+RGX3*0fm7^bb1HH4YP^!+rn+vEnbgA;92fAYEc~VFZljZ*sD-i z`nvXqNC1wjy)oH-ubvUHZhymTPe`m6d!DR1cI^5Lk+ zavjve>z7GjBe1EYUgMpd@TmxIYnW@bVmD{;5tEw0vK*7n<~9{|1M!+nHs$NexbqKe zG5SP7)mzX$Klt@ykA<-0g>==mh{DHq#_x;V?vEjUou$PhwNO`-AxuRaerDe@Wu>^d z2YW~*@Ha|yxC*#=lF5mLLc+rCzt)Zpl6SDY5&zZ508E#Sh%Wx)xz(l*~ch} zr$qy9kPy)e!w*28E_}3oT z{JSKvsc|f(cTrNKG7pdEYr!b|jTukF$6xg@b?_A?;29XkeKz7@?L1otCzRW68|fRY z^k^;aEX{Yx=#Cx)j6vHig%X&lfl`CU6B@!f)>`ek1 zV`ob`8hv(9ISo_ui`Ing92~<#1!Q+xjV=#|Vu?6~BwLagDL?tg7)-L=19_#1`+4v~ z*E$|H0IgGAaAIh0xBwNPJGB1zF*=XzDDI~pw-Z|JtUXP3^Z$f(shhHK@%GDO!DrqU zw5o7S*L7Y#>^Qnh@+m-j42^!9$KSw!CU3PJ?$Z$RmSnU_PpKFR5>S|eLU!#y)r&FByA4);ez z>BkLj1`Nau79il*Mjt04)quC3jpxtK+&*UV=X_+6@3xZ=LHNVEG1m6l6M}7uqucT} zi>WxUiE90wf$7zVRxbE0BuWX8g8^>AmfqTHOE+p3txv+SYW+Ct{LmgH^N0hzS0)@r zDiS&>5>rhySJT$GaI|r@UI8HVRTD{eAWf8Ii~0+(k>d=xmu&jB{ER)?rGW;4l#VRN zLm8aPe5o%P5n#0zMy?2W|Zx41BRkxoG5b@jBp3sTRLMif}sn zw916;*vW8ro5cOgC5@g@Ty={@OK<44;p+1XyzUkcD3eDUes}GI7m_jrPj2<0oXj_+ zAfKOS0j+*g+BcY#~tBaRe%F+cVi7YaYPv9KPJ{)}Mn zusnfT1@MJw#H>*=W*W3jxTRQtyNg_G=AQ<%_O_-ymwn;yyT30ls*z)^p0?|u#JyXe zCVNwVcmCvG6NrV%*n zErBK)+rA^>{V4Nk+_HyLGgJoSqcX-xEsR?&w^lSs=59G9Hp+rKCI&Iui$3oQa(7wvcx> z$5jc8eq?KUC%FIox3j0{F0#&?B?!AW$D-oLz?VkW|3g8VioM3+BIT7?-yAi4jujR1rMtppB&C$Iiq5#Iq6dh?0 z*BOpL!Cv9khrL0Qf!Y4NHBl?j*$#a=pOC9cCN&jcW_}#O^hjE;ea`dzpdea%TcN`X z;?w-YMQj(*iN|>71WT_<$d%FQM~x=DSoiK&u0R$(i(~lV-VwPe-Bm0=g-&HZ z>l;1>HF5why$kf<)XCCPjVM2K&6Xe-%sk)VGAYj!2DNzez9my2()fPVHt*V?%QNQO z&9RlX;tzh04O-l{->fR&uL&A<%g&3dhw3t5q65u?6HeUtX4IY_oT@nmUOwHMs0F$U z*d5X@+{2nU_)ZCXvRKx9@?P$7wx2h`(O8PvcRqh3;sf@z4<<@*E2<1+H7+4J=UcEt zyk}!YI8JFkGBzLg8=XI`5Q;g4QEJb0y>qu4xW9h13tC{(ay}D1VxY81?CLmr=vpwu z-;hQ&3v?5TR>e%dvkPM%&k|?5fd)N2@0o2cJ;5ZM47FCv|K>0wZ`(PlWrA{LB6XrG zmK3wJuIw1wcn3zs9A>b*M^jB~*$Y<$(&&Ei&-6-^zn*h%&82N^rQQDgz^>`DCu8Yi zc;d8EWZEt$0N|*-J2;5f$dTDYu=k1SP6>IzbYaX9A+%5oTTvwJ3}MeOoFp+I43EYe zK2MB6w{=yzKZ==doOg7|Vy;744VqvvsRAyj(_4aXUrJ0dK0G-V@82?b3uU-`UbR;; z?O45edEQ*=$1@iiExm5(y+1pO#Gn0+L>l()X}K1jVmuyhPdKOGdrDK zTio6O`i4>4(G8!vcwdZ?ZojdnjNV)P*S^B@C1sl4F}_5s`nDbFWVbjIGhDs{xmbZF zS2p_QTLgucP&~fMWJNx@yMs8XA~Df_I{VohU8W<^cL!;xNVSt}$WT$)TIZny(9a;`@Cr$fGjUO4&!rDQb16>O1MEp;W5Pc-PT{ahkEa&fuH)g$xT zdPP|-U^q}vbcO0P58sE!>0`a;d&OH=oimNqx0&rZ$`*Zk1!^|4BW1tSdV1?L;l~EP z_HVp@@9*JXPtNgBP%ulV@G{7sT~zjv8}dWmR~n1;8`!9pq~_gh^0U{zco-Sm)Y*HJ z*h2d4_1ihK9Ya3NBIjT?nx3^}5pgAg(NJsBV9l@=h0}EQ&L(smVcMj>@>efhhFwql%`cVKg ztNoZ;Wgd8`cM-5%7_;MhbOxmPraf>?;c0FlesXHy^SJD%cKb{cQoP)dWn!4(bSD3) z^p#{Ka-fVxe6&xb$iiGdg}ShEcbaEGO43~BLWB4cOxI6?v|eD2TZfYCUBotR{szTX z_Y?Z@)v7(_+uq7el?#w$FI=lOEwA*UQ{G5C-B;JobTO!(e?Oh$dT1W2&ZZF1FRO|# z%H5hG*3}W(f^P0;!7&8iWAbC*Vbz;{&y{Ve;5CsdgN0oE!kM?NC;|@Tx|?nt=}r&B z)mN)KEj>GBFCqtNf{2P|2<>>RIItdRS}eZCUN6~4?g|Q{93p9EY_zo(WWT)!f-9i50Yq1%8ej$y;ghR0VrgRrbp<1b-R7puO2$^(poe)m*PA zl}=euS1y~U=7hX9$ZeCm=4k$D`(|A{yJn>mCvbqML-?tc+xjSfp6rqS5$CZ)Ol5_| zIca!anBd;r(kWWdt-FduCmsk*9vP^3Dlg)(V_13x1SC$FpNLs7?>yC1(p(@-Dr-$@eUS)JHDf%Yd?IasMnK5u?S`JZ(V{Z)z(#o#mXS=k6s_FmR0 z<|Oxu+5Jnn?~+t?piwTF8~c+^ElnNoT;p3RTl3_Zjra=BNU>&&(?j0cAGm;iSm;F0 z!QyPD#;$Ifpu_Mb>Z_-$(wRXnS2_gO;?Z>-q{mqd@!XBvGNH)G#jTK_BFKEi7VY=H zTi;Agc^N3_EQNDN5(C;#QBTd>u84d2f0wWIZuHUER_C~7)@r#lU0$AbI&Qx@ZL$*D z$rgG~eX5WG5JUl5V{%WDsABOUhw<;ejK8>pH5ipCFo`CTa&Pi1)$H_f)Cg|g& z%k$qlyd&D2{>UC zq~yORMNOfoTcmh9?rhCDg5Cl-pvTOh*(#JSdvQ;FkVkgP#8y_T)sJ}Yua$^4h|Us? zkdqt0ah#+Lb);ftzEnQeF;$aCH~oVb7pzG-$r&lB4TwG^z+N6NApO@{EHl@$M!yKOTF zxC$s8OKR6=Z8h#?=zPxM-5Pd9ZpTn6C$7_-_-xxagsyQb!H1*N#&1fRF zmDli*lvRF_Uh@DES{4Y@ik25%CZtZ3f9c^B6Ax4ZLk1?)&FLY(Nc|ia2YM=BbQ0b! zKD2J&az6g*rQRN2l&A%maJDSc6sn{g(9Oog5TTyY2e&K0&M)cn&?tcLSNEjdh literal 0 HcmV?d00001 diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml index 291ec7675b1af..9c8f0d9c61147 100644 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ b/docker/test/fixtures/zookeeper/docker-compose.yml @@ -16,7 +16,6 @@ --- version: '2' services: - zookeeper-1: image: confluentinc/cp-zookeeper:latest ports: @@ -55,10 +54,10 @@ services: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" @@ -82,10 +81,10 @@ services: KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" @@ -98,50 +97,3 @@ services: depends_on: - zookeeper-1 - zookeeper-2 - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 9bf7356f9bd38..401ad63aa6179 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,6 +1,3 @@ -confluent-kafka urllib3 requests -fastavro -jsonschema HTMLTestRunner-Python3 \ No newline at end of file From 72a9765adb58f1a08db7ac5e1dcef9daff505705 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:40:30 +0530 Subject: [PATCH 16/46] Remove redundant files --- docker/test/fixtures/schema.avro | 8 -------- docker/test/fixtures/source_connector.json | 14 -------------- 2 files changed, 22 deletions(-) delete mode 100644 docker/test/fixtures/schema.avro delete mode 100644 docker/test/fixtures/source_connector.json diff --git a/docker/test/fixtures/schema.avro b/docker/test/fixtures/schema.avro deleted file mode 100644 index d85b0e1204773..0000000000000 --- a/docker/test/fixtures/schema.avro +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "record", - "name": "Message", - "fields": [ - {"name": "key", "type": "string"}, - {"name": "value", "type": "string"} - ] -} \ No newline at end of file diff --git a/docker/test/fixtures/source_connector.json b/docker/test/fixtures/source_connector.json deleted file mode 100644 index 495bed2db8f1b..0000000000000 --- a/docker/test/fixtures/source_connector.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "datagen-users", - "config": { - "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", - "kafka.topic": "test_topic_connect", - "quickstart": "users", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "false", - "max.interval": 1000, - "iterations": 10, - "tasks.max": "1" - } -} \ No newline at end of file From 82775b36625dedba52073bbe647bc0d31f873514 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:55:23 +0530 Subject: [PATCH 17/46] Fix tests Add SSL_TOPIC again in constants.py --- docker/test/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/constants.py b/docker/test/constants.py index 621aa8f41e84b..0a55731af90e8 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -28,6 +28,7 @@ CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" +SSL_TOPIC="test_topic_ssl" BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" From 1c0d3c512c5462e6c4039dad690c9fc6d7bb70d1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 14:59:56 +0530 Subject: [PATCH 18/46] Use CDS to start kafka --- docker/jvm/Dockerfile | 40 +++++++++++++++++++++++++++++++--------- docker/jvm/jsa_launch | 15 +++++++++++++++ docker/jvm/launch | 12 +++++++++--- 3 files changed, 55 insertions(+), 12 deletions(-) create mode 100755 docker/jvm/jsa_launch diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e9eea40877690..44010e026f587 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -16,9 +16,7 @@ # limitations under the License. ############################################################################### -ARG GOLANG_VERSION=1.21.1 - -FROM golang:${GOLANG_VERSION} AS build-ub +FROM golang:latest AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser COPY --chown=appuser:appuser resources/ub/ ./ @@ -27,7 +25,29 @@ USER appuser RUN go test ./... -FROM eclipse-temurin:21-jre +FROM eclipse-temurin:21-jre-alpine AS build-jsa + +USER root + +# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments +ARG kafka_url + +ENV KAFKA_URL=$kafka_url + +COPY jsa_launch /etc/kafka/docker/jsa_launch + +RUN set -eux ; \ + apk update ; \ + apk upgrade ; \ + apk add --no-cache wget gcompat; \ + mkdir opt/kafka; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; + +RUN /etc/kafka/docker/jsa_launch + + +FROM eclipse-temurin:21-jre-alpine # exposed ports EXPOSE 9092 @@ -49,9 +69,9 @@ LABEL org.label-schema.name="kafka" \ ENV KAFKA_URL=$kafka_url RUN set -eux ; \ - apt-get update ; \ - apt-get upgrade -y ; \ - apt-get install -y --no-install-recommends curl wget gpg dirmngr gpg-agent; \ + apk update ; \ + apk upgrade ; \ + apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ @@ -61,14 +81,16 @@ RUN set -eux ; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ mkdir -p /etc/kafka/docker /usr/logs; \ - useradd --no-log-init --create-home --shell /bin/bash appuser; \ + adduser -h /home/appuser -D --shell /bin/bash appuser; \ chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ rm kafka.tgz; \ + apk cache clean; \ rm kafka.tgz.asc; -COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa +COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch new file mode 100755 index 0000000000000..eced8a568792b --- /dev/null +++ b/docker/jvm/jsa_launch @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" +opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties +KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & +PIDS=$! + +sleep 10 +echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 +sleep 5 +echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) +sleep 5 + +kill -s TERM $PIDS +sleep 10 \ No newline at end of file diff --git a/docker/jvm/launch b/docker/jvm/launch index 778313c7ed349..33b8333c756ed 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -20,7 +20,7 @@ # Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. if [ -z "$KAFKA_JMX_OPTS" ]; then - export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " fi # The JMX client needs to be able to connect to java.rmi.server.hostname. @@ -33,8 +33,8 @@ export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} if [ "$KAFKA_JMX_PORT" ]; then # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. - export JMX_PORT=$KAFKA_JMX_PORT - export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi # KRaft required step: Format the storage directory with provided cluster ID unless it already exists. @@ -48,5 +48,11 @@ then { echo $result && (exit 1) } fi +if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then + export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" +else + export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" +fi + # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties From a9faadb3c2aadff2a4b1436d8a81d495fddf19cc Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 20:47:05 +0530 Subject: [PATCH 19/46] Resolve PR comments - Remove zookeeper support & tests - Optimise image by removing redundant packages - Minor code fixes --- docker/jvm/Dockerfile | 12 +-- docker/jvm/jsa_launch | 4 +- docker/resources/common-scripts/configure | 5 +- .../common-scripts/kafka-propertiesSpec.json | 7 +- docker/resources/common-scripts/run | 3 - docker/test/docker_sanity_test.py | 12 +-- .../fixtures/zookeeper/docker-compose.yml | 99 ------------------- 7 files changed, 13 insertions(+), 129 deletions(-) delete mode 100644 docker/test/fixtures/zookeeper/docker-compose.yml diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 44010e026f587..b2044d1e7d5bf 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,7 +39,7 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat; \ + apk add --no-cache wget gcompat procps; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; @@ -82,12 +82,12 @@ RUN set -eux ; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ mkdir -p /etc/kafka/docker /usr/logs; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ - chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ - chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chown appuser:appuser -R /usr/logs /opt/kafka; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - rm kafka.tgz; \ - apk cache clean; \ - rm kafka.tgz.asc; + rm kafka.tgz kafka.tgz.asc KEYS; \ + apk del curl wget gpg dirmngr gpg-agent; \ + apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index eced8a568792b..e9b24a7cbdf21 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -11,5 +11,5 @@ sleep 5 echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) sleep 5 -kill -s TERM $PIDS -sleep 10 \ No newline at end of file +opt/kafka/bin/kafka-server-stop.sh +sleep 5 \ No newline at end of file diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index b05d226a75913..7819c2dce127a 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -36,9 +36,8 @@ then ub ensure KAFKA_ADVERTISED_LISTENERS fi else - echo "Running in Zookeeper mode..." - ub ensure KAFKA_ZOOKEEPER_CONNECT - ub ensure KAFKA_ADVERTISED_LISTENERS + echo "Only KRaft mode is supported KAFKA_PROCESS_ROLES is a mandatory config" + exit 1 fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing diff --git a/docker/resources/common-scripts/kafka-propertiesSpec.json b/docker/resources/common-scripts/kafka-propertiesSpec.json index 0e10d92d6473e..b67200c5fb305 100644 --- a/docker/resources/common-scripts/kafka-propertiesSpec.json +++ b/docker/resources/common-scripts/kafka-propertiesSpec.json @@ -1,10 +1,8 @@ { "prefixes": { - "KAFKA": false, - "CONFLUENT": true + "KAFKA": false }, "renamed": { - "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET": "zookeeper.clientCnxnSocket" }, "excludes": [ "KAFKA_VERSION", @@ -16,8 +14,7 @@ "KAFKA_GC_LOG_OPTS", "KAFKA_LOG4J_ROOT_LOGLEVEL", "KAFKA_LOG4J_LOGGERS", - "KAFKA_TOOLS_LOG4J_LOGLEVEL", - "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET" + "KAFKA_TOOLS_LOG4J_LOGLEVEL" ], "defaults": { }, diff --git a/docker/resources/common-scripts/run b/docker/resources/common-scripts/run index 41273ea5b7a59..4ff47c85558b9 100755 --- a/docker/resources/common-scripts/run +++ b/docker/resources/common-scripts/run @@ -28,11 +28,8 @@ fi echo "===> User" id -if [[ -z "${KAFKA_ZOOKEEPER_CONNECT-}" ]] -then echo "===> Setting default values of environment variables if not already set." . /etc/kafka/docker/configureDefaults -fi echo "===> Configuring ..." /etc/kafka/docker/configure diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 1022365cac8ac..12652112e139b 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -77,13 +77,11 @@ def create_topic(self, topic, topic_config): def produce_message(self, topic, producer_config, key, value): command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - print(" ".join(command)) subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - print(" ".join(command)) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() @@ -168,14 +166,6 @@ def tearDown(self) -> None: def test_bed(self): self.execute() -class DockerSanityTestZookeeper(DockerSanityTestCommon): - def setUp(self) -> None: - self.startCompose(constants.ZOOKEEPER_COMPOSE) - def tearDown(self) -> None: - self.destroyCompose(constants.ZOOKEEPER_COMPOSE) - def test_bed(self): - self.execute() - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("image") @@ -186,7 +176,7 @@ def test_bed(self): test_classes_to_run = [] if args.mode in ("all", "jvm"): - test_classes_to_run.extend([DockerSanityTestKraftMode, DockerSanityTestZookeeper]) + test_classes_to_run.extend([DockerSanityTestKraftMode]) loader = unittest.TestLoader() suites_list = [] diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml deleted file mode 100644 index 9c8f0d9c61147..0000000000000 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -version: '2' -services: - zookeeper-1: - image: confluentinc/cp-zookeeper:latest - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 - - zookeeper-2: - image: confluentinc/cp-zookeeper:latest - ports: - - "2182:2182" - environment: - ZOOKEEPER_CLIENT_PORT: 2182 - ZOOKEEPER_SERVER_ID: 2 - ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 - - broker: - image: {$IMAGE} - hostname: broker - container_name: broker - ports: - - "9092:9092" - - "29092:29092" - - "39092:39092" - volumes: - - ../secrets:/etc/kafka/secrets - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:19092,EXTERNAL://localhost:9092,SSL://localhost:39092,DOCKER://host.docker.internal:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT,SSL:SSL - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" - KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" - KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" - KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker-ssl: - image: {$IMAGE} - hostname: broker-ssl - container_name: broker-ssl - ports: - - "29093:29093" - - "9093:9093" - - "39093:39093" - volumes: - - ../secrets:/etc/kafka/secrets - environment: - KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" - KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" - KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" - KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - depends_on: - - zookeeper-1 - - zookeeper-2 From 6289c19984b787cc034ddb21aadc8d366e0e7064 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 09:44:43 +0530 Subject: [PATCH 20/46] Add github actions workflow for build and test of jvm docker image --- .github/workflows/docker_build_and_test.yml | 47 +++++++++++++++++++++ docker/docker_build_test.py | 7 ++- docker/test/docker_sanity_test.py | 13 +++--- 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docker_build_and_test.yml diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml new file mode 100644 index 0000000000000..0cbc2881d7e2a --- /dev/null +++ b/.github/workflows/docker_build_and_test.yml @@ -0,0 +1,47 @@ +name: Docker build test + +on: + workflow_dispatch: + inputs: + image_type: + type: choice + description: Docker image type to build and test + options: + - "jvm" + kafka_url: + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/test/requirements.txt + - name: Run tests + working-directory: ./docker + run: | + python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} + - name: Run Vulnerability scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'kafka/test:test' + format: 'table' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + output: scan_${{ github.event.inputs.image_type }}.txt + - uses: actions/upload-artifact@v3 + with: + name: report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html + - uses: actions/upload-artifact@v3 + with: + name: scan_${{ github.event.inputs.image_type }}.txt + path: scan_${{ github.event.inputs.image_type }}.txt \ No newline at end of file diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index a841e6f0ea548..5cefa723a2f4f 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -33,7 +33,6 @@ def build_jvm(image, tag, kafka_url): def run_jvm_tests(image, tag, kafka_url): subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - subprocess.run(["ls"]) subprocess.run(["mkdir", "./test/fixtures/kafka"]) subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") @@ -44,17 +43,17 @@ def run_jvm_tests(image, tag, kafka_url): parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", default="all", dest="image_type", help="Image type you want to build. By default it's all") + parser.add_argument("-type", "--image-type", choices=["jvm"], dest="image_type", help="Image type you want to build") parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() - if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)): + if args.image_type == "jvm" and (args.build_only or not (args.build_only or args.test_only)): if args.kafka_url: build_jvm(args.image, args.tag, args.kafka_url) else: raise ValueError("--kafka-url is a required argument for jvm image") - if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): + if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): run_jvm_tests(args.image, args.tag, args.kafka_url) \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 12652112e139b..d891af6391a8e 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -19,8 +19,9 @@ from HTMLTestRunner import HTMLTestRunner import constants import argparse +import socket -class DockerSanityTestCommon(unittest.TestCase): +class DockerSanityTest(unittest.TestCase): CONTAINER_NAME="broker" IMAGE="apache/kafka" @@ -158,7 +159,7 @@ def execute(self): self.assertEqual(total_errors, []) -class DockerSanityTestKraftMode(DockerSanityTestCommon): +class DockerSanityTestKraftMode(DockerSanityTest): def setUp(self) -> None: self.startCompose(constants.KRAFT_COMPOSE) def tearDown(self) -> None: @@ -172,11 +173,11 @@ def test_bed(self): parser.add_argument("mode", default="all") args = parser.parse_args() - DockerSanityTestCommon.IMAGE = args.image + DockerSanityTest.IMAGE = args.image test_classes_to_run = [] - if args.mode in ("all", "jvm"): - test_classes_to_run.extend([DockerSanityTestKraftMode]) + if args.mode == "jvm": + test_classes_to_run = [DockerSanityTestKraftMode] loader = unittest.TestLoader() suites_list = [] @@ -184,7 +185,7 @@ def test_bed(self): suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report.html", "w") + outfile = open(f"report_{args.mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', From c018f54089fbffa60ab56646ac68c94f83c34006 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 09:54:09 +0530 Subject: [PATCH 21/46] Add description to kafka url link --- .github/workflows/docker_build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 0cbc2881d7e2a..82a9a81a19ed2 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -9,6 +9,7 @@ on: options: - "jvm" kafka_url: + description: Kafka url to be used to build the docker image required: true jobs: From 8df9b59f8f535d1091c1bbd51934dab5fc9989d8 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 12:38:44 +0530 Subject: [PATCH 22/46] Refactors jsa launch script with error handling and timeouts --- docker/jvm/Dockerfile | 2 +- docker/jvm/jsa_launch | 33 +++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index b2044d1e7d5bf..7cbfe38455cc0 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,7 +39,7 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat procps; \ + apk add --no-cache wget gcompat procps netcat-openbsd; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index e9b24a7cbdf21..4962f97937832 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -3,13 +3,34 @@ KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & -PIDS=$! -sleep 10 +check_timeout() { + if [ $TIMEOUT -eq 0 ]; then + echo "Server startup timed out" + exit 1 + fi + echo "Check will timeout in $(( TIMEOUT-- )) seconds" + sleep 1 +} + +TIMEOUT=20 +while ! nc -z localhost 9092; do + check_timeout +done + +opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 +[ $? -eq 0 ] || exit 1 + echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 -sleep 5 -echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) -sleep 5 +[ $? -eq 0 ] || exit 1 + +opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 +[ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh -sleep 5 \ No newline at end of file + +TIMEOUT=20 +until [ -f /kafka.jsa ] +do + check_timeout +done \ No newline at end of file From ee53281078c3448caa99b1f91a1dc420fff08f03 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:19:18 +0530 Subject: [PATCH 23/46] Remove static sleep from sanity tests --- docker/test/constants.py | 14 ++---- docker/test/docker_sanity_test.py | 76 +++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/docker/test/constants.py b/docker/test/constants.py index 0a55731af90e8..ace6ff9d66efa 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -20,19 +20,15 @@ KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" -SCHEMA_REGISTRY_URL="http://localhost:8081" -CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 -SCHEMA_REGISTRY_TEST_TOPIC="test_topic_schema" -CONNECT_TEST_TOPIC="test_topic_connect" -CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" +SSL_FLOW_TESTS="SSL Flow Tests" SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" -SSL_TOPIC="test_topic_ssl" +SSL_TOPIC="test-topic-ssl" -BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" +BROKER_RESTART_TESTS="Broker Restart Tests" +BROKER_CONTAINER="broker" +BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" -SCHEMA_REGISTRY_ERROR_PREFIX="SCHEMA_REGISTRY_ERR" -CONNECT_ERROR_PREFIX="CONNECT_ERR" SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index d891af6391a8e..49655998c6c09 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -22,45 +22,32 @@ import socket class DockerSanityTest(unittest.TestCase): - CONTAINER_NAME="broker" IMAGE="apache/kafka" def resumeImage(self): - subprocess.run(["docker", "start", self.CONTAINER_NAME]) + subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) def stopImage(self) -> None: - subprocess.run(["docker", "stop", self.CONTAINER_NAME]) + subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) def startCompose(self, filename) -> None: old_string="image: {$IMAGE}" new_string=f"image: {self.IMAGE}" - with open(filename) as f: s = f.read() - if old_string not in s: - print('"{old_string}" not found in {filename}.'.format(**locals())) - with open(filename, 'w') as f: - print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) s = s.replace(old_string, new_string) f.write(s) subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - time.sleep(25) def destroyCompose(self, filename) -> None: old_string=f"image: {self.IMAGE}" new_string="image: {$IMAGE}" - subprocess.run(["docker-compose", "-f", filename, "down"]) - time.sleep(10) with open(filename) as f: s = f.read() - if old_string not in s: - print('"{old_string}" not found in {filename}.'.format(**locals())) - with open(filename, 'w') as f: - print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) s = s.replace(old_string, new_string) f.write(s) @@ -85,13 +72,35 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() + + def wait_for_port(self, host, port, open, timeout): + start_time = time.perf_counter() + while time.perf_counter() - start_time < timeout: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + status = sock.connect_ex((host, port)) + sock.close() + if (open and status == 0) or (not open and status != 0): + return + else: + time.sleep(1) + raise TimeoutError("Timed out while waiting for the port", host, port) def ssl_flow(self): - print("Running SSL flow tests") + print(f"Running {constants.SSL_FLOW_TESTS}") errors = [] + try: + self.wait_for_port('localhost', 9093, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors + try: + self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + return errors + producer_config = ["--bootstrap-server", "localhost:9093", "--producer.config", constants.SSL_CLIENT_CONFIG] - self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") consumer_config = [ @@ -108,12 +117,21 @@ def ssl_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) - print("Errors in SSL Flow:-", errors) + if errors: + print(f"Errors in {constants.SSL_FLOW_TESTS}:- {errors}") + else: + print(f"No errors in {constants.SSL_FLOW_TESTS}") return errors def broker_restart_flow(self): - print("Running broker restart tests") + print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] + try: + self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors + try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: @@ -123,13 +141,20 @@ def broker_restart_flow(self): producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") - print("Stopping Image") + print("Stopping Container") self.stopImage() - time.sleep(15) - + try: + self.wait_for_port('localhost', 9092, False, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors print("Resuming Image") self.resumeImage() - time.sleep(15) + try: + self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: @@ -141,7 +166,10 @@ def broker_restart_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - print("Errors in Broker Restart Flow:-", errors) + if errors: + print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") + else: + print(f"No errors in {constants.BROKER_RESTART_TESTS}") return errors def execute(self): From 58aa37ea1219850c4164a8c75503d6ed4b52f7b9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:30:05 +0530 Subject: [PATCH 24/46] Rely on scripts to detect when server is up --- docker/test/docker_sanity_test.py | 35 +------------------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 49655998c6c09..5571bc125bb06 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -15,11 +15,9 @@ import unittest import subprocess -import time from HTMLTestRunner import HTMLTestRunner import constants import argparse -import socket class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" @@ -72,27 +70,10 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() - - def wait_for_port(self, host, port, open, timeout): - start_time = time.perf_counter() - while time.perf_counter() - start_time < timeout: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - status = sock.connect_ex((host, port)) - sock.close() - if (open and status == 0) or (not open and status != 0): - return - else: - time.sleep(1) - raise TimeoutError("Timed out while waiting for the port", host, port) def ssl_flow(self): print(f"Running {constants.SSL_FLOW_TESTS}") errors = [] - try: - self.wait_for_port('localhost', 9093, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors try: self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: @@ -126,11 +107,6 @@ def ssl_flow(self): def broker_restart_flow(self): print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] - try: - self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) @@ -143,18 +119,9 @@ def broker_restart_flow(self): print("Stopping Container") self.stopImage() - try: - self.wait_for_port('localhost', 9092, False, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors print("Resuming Image") self.resumeImage() - try: - self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: From 24863bba82606df733f038b67a0f5c4c546d2741 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:42:31 +0530 Subject: [PATCH 25/46] Removed redundant wait in jsa generation --- docker/jvm/jsa_launch | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 4962f97937832..7f40f86f97058 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -13,22 +13,18 @@ check_timeout() { sleep 1 } -TIMEOUT=20 -while ! nc -z localhost 9092; do - check_timeout -done - opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 +opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 [ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh +# Wait until jsa file is generated TIMEOUT=20 until [ -f /kafka.jsa ] do From 07004979d5866588712f478985009a02f4d914a2 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:40:19 +0530 Subject: [PATCH 26/46] KAFKA-15444: Accept kafka-url in the build --- docker/docker_build_test.py | 15 ++++++++------- docker/native-image/Dockerfile | 8 +++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 789c6d6bf0ccd..28328bc4e6879 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -31,12 +31,11 @@ def build_jvm(image, tag, kafka_url): return shutil.rmtree("jvm/resources") - -def build_native(image, tag): +def build_native(image, tag, kafka_url): image = f'{image}:{tag}' copy_tree("resources", "native-image/resources") result = subprocess.run( - ["docker", "build", "-f", "native-image/Dockerfile", "-t", image, + ["docker", "build", "-f", "native-image/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f'build_date={date.today()}', "native-image"]) if result.stderr: print(result.stdout) @@ -51,12 +50,11 @@ def run_jvm_tests(image, tag, kafka_url): subprocess.run(["rm", "kafka.tgz"]) shutil.rmtree("./test/fixtures/kafka") - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", choices=["jvm"], dest="image_type", help="Image type you want to build") + parser.add_argument("-type", "--image-type", choices=["jvm", "native"], dest="image_type", help="Image type you want to build") parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") @@ -68,8 +66,11 @@ def run_jvm_tests(image, tag, kafka_url): else: raise ValueError("--kafka-url is a required argument for jvm image") - if args.image_type in ("all", "native-image") and (args.build_only or not (args.build_only or args.test_only)): - build_native(args.image, args.tag) + if args.image_type == "native" and (args.build_only or not (args.build_only or args.test_only)): + if args.kafka_url: + build_native(args.image, args.tag, args.kafka_url) + else: + raise ValueError("--kafka-url is a required argument for building docker image") if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): run_jvm_tests(args.image, args.tag, args.kafka_url) diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile index 512d077c3b279..8439fa16a71c9 100644 --- a/docker/native-image/Dockerfile +++ b/docker/native-image/Dockerfile @@ -31,8 +31,10 @@ WORKDIR /app ENV KAFKA_URL=$kafka_url COPY native-image-configs native-image-configs -RUN wget -nv -O kafka.tgz "$KAFKA_URL"; \ - tar xfz kafka.tgz -C /kafka --strip-components 1; \ +RUN mkdir kafka; \ + microdnf install wget; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + tar xfz kafka.tgz -C kafka --strip-components 1; \ rm kafka.tgz ; \ cd kafka ; \ native-image --no-fallback \ @@ -57,7 +59,7 @@ RUN apk update && \ WORKDIR /app COPY --from=build-ub /build/ub /usr/bin -COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . +COPY --from=build-native-image /app/kafka/kafka.kafkanativewrapper . COPY resources/common-scripts /etc/kafka/docker COPY launch /etc/kafka/docker/ From ff310624628138ec2b93753feecd80ffcaae98c4 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 14:16:02 +0530 Subject: [PATCH 27/46] Add promotion script --- docker/docker_promote.py | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docker/docker_promote.py diff --git a/docker/docker_promote.py b/docker/docker_promote.py new file mode 100644 index 0000000000000..ef8c8421ebf60 --- /dev/null +++ b/docker/docker_promote.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Python script to promote an rc image. + +Follow the interactive guide to pull an RC image and promote it desired dockerhub repository. + +Usage: docker_promote.py + +Interactive utility to promote a docker image +""" + +import subprocess +import requests +from getpass import getpass + +def execute(command): + if subprocess.run(command).returncode != 0: + raise SystemError("Failure in executing following command:- ", " ".join(command)) + +def login(): + execute(["docker", "login"]) + +def pull(rc_image, promotion_image): + execute(["docker", "pull", "--platform=linux/amd64", rc_image]) + execute(["docker", "tag", rc_image, f"{promotion_image}-amd64"]) + execute(["docker", "pull", "--platform=linux/arm64", rc_image]) + execute(["docker", "tag", rc_image, f"{promotion_image}-arm64"]) + +def push(promotion_image): + execute(["docker", "push", f"{promotion_image}-amd64"]) + execute(["docker", "push", f"{promotion_image}-arm64"]) + +def push_manifest(promotion_image): + execute(["docker", "manifest", "create", promotion_image, + "--amend", f"{promotion_image}-amd64", + "--amend", f"{promotion_image}-arm64"]) + + execute(["docker", "manifest", "push", promotion_image]) + +def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token): + if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-amd64", headers={"Authorization": f"JWT {token}"}).status_code != 204: + raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64 is removed from dockerhub") + if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-arm64", headers={"Authorization": f"JWT {token}"}).status_code != 204: + raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64 is removed from dockerhub") + subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) + subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) + +if __name__ == "__main__": + login() + username = input("Enter dockerhub username: ") + password = getpass("Enter dockerhub password: ") + + token = (requests.post("https://hub.docker.com/v2/users/login/", json={"username": username, "password": password})).json()['token'] + if len(token) == 0: + raise PermissionError("Dockerhub login failed") + + rc_image = input("Enter the RC docker image that you want to pull (in the format ::): ") + promotion_image_namespace = input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") + promotion_image_name = input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") + promotion_image_tag = input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: 4.0.0]: ") + promotion_image = f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}" + + pull(rc_image, promotion_image) + push(promotion_image) + push_manifest(promotion_image) + remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) + print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") \ No newline at end of file From 9a24709b3ebd43ece84c9d0897314ebe0af04a2f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 14:20:12 +0530 Subject: [PATCH 28/46] Add requirements.txt for promotion script --- docker/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 docker/requirements.txt diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000000000..663bd1f6a2ae0 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file From e6582989c58b7f412e1e16b13fd91c637a450c6f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 21:33:04 +0530 Subject: [PATCH 29/46] Fix property file location --- docker/jvm/launch | 4 ++-- docker/resources/common-scripts/configure | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/jvm/launch b/docker/jvm/launch index 33b8333c756ed..e01c5810a9248 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -43,7 +43,7 @@ then echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this - result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /etc/kafka/kafka.properties 2>&1) || \ + result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } fi @@ -55,4 +55,4 @@ else fi # Start kafka broker -exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties +exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kafka.properties diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 7819c2dce127a..33bb195a137ff 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -36,7 +36,7 @@ then ub ensure KAFKA_ADVERTISED_LISTENERS fi else - echo "Only KRaft mode is supported KAFKA_PROCESS_ROLES is a mandatory config" + echo "Only KRaft mode is supported. KAFKA_PROCESS_ROLES is a mandatory config" exit 1 fi @@ -49,7 +49,7 @@ then KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') fi -ub path /etc/kafka/ writable +ub path /opt/kafka/config/ writable if [[ -z "${KAFKA_LOG_DIRS-}" ]] then @@ -141,6 +141,6 @@ fi # --- for broker -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/tools-log4j.properties +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /opt/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /opt/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /opt/kafka/config/tools-log4j.properties From 9cbec87b114c3c2da8a889de23b886a92451b676 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 14:41:19 +0530 Subject: [PATCH 30/46] Add support for supplying properties through file mounting Changes to add support for file mounting of properties. Env variables, if defined, will override the file input properties. --- docker/jvm/Dockerfile | 22 ++++++++---------- docker/jvm/launch | 15 ++++-------- docker/resources/common-scripts/configure | 23 ++++++++----------- .../common-scripts/configureDefaults | 13 ----------- .../kafka-log4j.properties.template | 12 ++++------ .../kafka-tools-log4j.properties.template | 10 ++++---- .../resources/common-scripts/kafka.properties | 13 +++++++++++ .../resources/common-scripts/log4j.properties | 15 ++++++++++++ .../common-scripts/tools-log4j.properties | 6 +++++ .../ub/testResources/sampleLog4j.template | 7 +----- docker/test/requirements.txt | 2 -- 11 files changed, 67 insertions(+), 71 deletions(-) create mode 100644 docker/resources/common-scripts/kafka.properties create mode 100644 docker/resources/common-scripts/log4j.properties create mode 100644 docker/resources/common-scripts/tools-log4j.properties diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 7cbfe38455cc0..0e0fd6ff623db 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -32,8 +32,6 @@ USER root # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url -ENV KAFKA_URL=$kafka_url - COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ @@ -41,7 +39,7 @@ RUN set -eux ; \ apk upgrade ; \ apk add --no-cache wget gcompat procps netcat-openbsd; \ mkdir opt/kafka; \ - wget -nv -O kafka.tgz "$KAFKA_URL"; \ + wget -nv -O kafka.tgz "$kafka_url"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; RUN /etc/kafka/docker/jsa_launch @@ -66,25 +64,23 @@ LABEL org.label-schema.name="kafka" \ org.label-schema.schema-version="1.0" \ maintainer="apache" -ENV KAFKA_URL=$kafka_url - RUN set -eux ; \ apk update ; \ apk upgrade ; \ apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ mkdir opt/kafka; \ - wget -nv -O kafka.tgz "$KAFKA_URL"; \ - wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ + wget -nv -O kafka.tgz "$kafka_url"; \ + wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ gpg --import KEYS; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ - mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ - mkdir -p /etc/kafka/docker /usr/logs; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka; \ + mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ - chown appuser:appuser -R /usr/logs /opt/kafka; \ - chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka; \ rm kafka.tgz kafka.tgz.asc KEYS; \ apk del curl wget gpg dirmngr gpg-agent; \ apk cache clean; @@ -96,6 +92,6 @@ COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser -VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data"] +VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"] CMD ["/etc/kafka/docker/run"] diff --git a/docker/jvm/launch b/docker/jvm/launch index e01c5810a9248..a9844b60953e1 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -37,16 +37,11 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi -# KRaft required step: Format the storage directory with provided cluster ID unless it already exists. -if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] -then - echo "===> Using provided cluster id $CLUSTER_ID ..." - - # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this - result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ - echo $result | grep -i "already formatted" || \ - { echo $result && (exit 1) } -fi +echo "===> Using provided cluster id $CLUSTER_ID ..." +# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this +result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ + echo $result | grep -i "already formatted" || \ + { echo $result && (exit 1) } if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 33bb195a137ff..2d29322298d89 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -35,15 +35,12 @@ then else ub ensure KAFKA_ADVERTISED_LISTENERS fi -else - echo "Only KRaft mode is supported. KAFKA_PROCESS_ROLES is a mandatory config" - exit 1 fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing # hosts with 0.0.0.0. This is good default as it ensures that the broker # process listens on all ports. -if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) +if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) && [[ -n "${KAFKA_ADVERTISED_LISTENERS}" ]] then export KAFKA_LISTENERS KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') @@ -51,12 +48,6 @@ fi ub path /opt/kafka/config/ writable -if [[ -z "${KAFKA_LOG_DIRS-}" ]] -then - export KAFKA_LOG_DIRS - KAFKA_LOG_DIRS="/var/lib/kafka/data" -fi - # advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] then @@ -139,8 +130,12 @@ then fi fi +mv /etc/kafka/docker/kafka.properties /opt/kafka/config/kafka.properties +mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties +mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties -# --- for broker -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /opt/kafka/config/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /opt/kafka/config/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /opt/kafka/config/tools-log4j.properties +cp -R /mnt/shared/config/. /opt/kafka/config/ + +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json >> /opt/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template >> /opt/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index a732ad60c37fb..6f5bfa47bf4ea 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -19,19 +19,6 @@ env_defaults=( # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" # See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" - ["KAFKA_NODE_ID"]=1 - ["KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"]="CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" - ["KAFKA_LISTENERS"]="PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092" - ["KAFKA_ADVERTISED_LISTENERS"]="PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092" - ["KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"]=1 - ["KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS"]=0 - ["KAFKA_TRANSACTION_STATE_LOG_MIN_ISR"]=1 - ["KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR"]=1 - ["KAFKA_PROCESS_ROLES"]="broker,controller" - ["KAFKA_CONTROLLER_QUORUM_VOTERS"]="1@localhost:29093" - ["KAFKA_INTER_BROKER_LISTENER_NAME"]="PLAINTEXT" - ["KAFKA_CONTROLLER_LISTENER_NAMES"]="CONTROLLER" - ["KAFKA_LOG_DIRS"]="/tmp/kraft-combined-logs" ) for key in "${!env_defaults[@]}"; do diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index 3a7b4744e34c4..5241723d8cc7b 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,11 +1,9 @@ -log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout +{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }} +{{ if ne $value "INFO" }} +log4j.rootLogger=$value, stdout +{{ end }}{{ end }} -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} {{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} -{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +{{ range $k, $v := splitToMapDefaults "," "" $loggers}} log4j.logger.{{ $k }}={{ $v -}} {{ end }} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index c2df5bcf064a4..69fd05d437351 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,6 +1,4 @@ -log4j.rootLogger={{ getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN" }}, stderr - -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n -log4j.appender.stderr.Target=System.err +{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} +{{if ne $value "WARN"}} +log4j.rootLogger=$value, stderr +{{ end }}{{ end }} diff --git a/docker/resources/common-scripts/kafka.properties b/docker/resources/common-scripts/kafka.properties new file mode 100644 index 0000000000000..e9f40eddc1727 --- /dev/null +++ b/docker/resources/common-scripts/kafka.properties @@ -0,0 +1,13 @@ +advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 +controller.listener.names=CONTROLLER +controller.quorum.voters=1@localhost:29093 +group.initial.rebalance.delay.ms=0 +inter.broker.listener.name=PLAINTEXT +listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT +listeners=PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092 +log.dirs=/tmp/kraft-combined-logs +node.id=1 +offsets.topic.replication.factor=1 +process.roles=broker,controller +transaction.state.log.min.isr=1 +transaction.state.log.replication.factor=1 diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties new file mode 100644 index 0000000000000..148bd53b664b6 --- /dev/null +++ b/docker/resources/common-scripts/log4j.properties @@ -0,0 +1,15 @@ +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + +log4j.logger.kafka=INFO +log4j.logger.kafka.authorizer.logger=WARN +log4j.logger.kafka.controller=TRACE +log4j.logger.kafka.log.LogCleaner=INFO +log4j.logger.kafka.network.RequestChannel$=WARN +log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG +log4j.logger.kafka.request.logger=WARN +log4j.logger.state.change.logger=TRACE diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties new file mode 100644 index 0000000000000..27d9fbee48bf5 --- /dev/null +++ b/docker/resources/common-scripts/tools-log4j.properties @@ -0,0 +1,6 @@ +log4j.rootLogger=WARN, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template index 3aace55b81aba..6a6a8630b2c10 100644 --- a/docker/resources/ub/testResources/sampleLog4j.template +++ b/docker/resources/ub/testResources/sampleLog4j.template @@ -1,11 +1,6 @@ log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} {{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} -{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +{{ range $k, $v := splitToMapDefaults "," "" $loggers}} log4j.logger.{{ $k }}={{ $v -}} {{ end }} \ No newline at end of file diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 401ad63aa6179..cb2ef3545ecf1 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,3 +1 @@ -urllib3 -requests HTMLTestRunner-Python3 \ No newline at end of file From 108ab8595f979cbc8855e181f7aa87926e10109b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 14:56:42 +0530 Subject: [PATCH 31/46] Ensure that environment variable configs are always appended in newline --- docker/resources/common-scripts/configure | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 2d29322298d89..f3db5a2188b10 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -136,6 +136,6 @@ mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.proper cp -R /mnt/shared/config/. /opt/kafka/config/ -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json >> /opt/kafka/config/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template >> /opt/kafka/config/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template >> /opt/kafka/config/tools-log4j.properties +echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/kafka.properties +echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties +echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties From ed2f94fc1ba5f8757b25f6d1d7a25a43d964614f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 15:04:21 +0530 Subject: [PATCH 32/46] Add missing brackets in template --- .../resources/common-scripts/kafka-log4j.properties.template | 5 ++--- .../common-scripts/kafka-tools-log4j.properties.template | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index 5241723d8cc7b..f9da46cc7443d 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,6 +1,5 @@ -{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }} -{{ if ne $value "INFO" }} -log4j.rootLogger=$value, stdout +{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}{{ if ne $value "INFO" }} +log4j.rootLogger={{ $value }}, stdout {{ end }}{{ end }} {{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index 69fd05d437351..4c55b9bb9a2ae 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,4 +1,3 @@ -{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} -{{if ne $value "WARN"}} -log4j.rootLogger=$value, stderr +{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} {{if ne $value "WARN"}} +log4j.rootLogger={{ $value }}, stderr {{ end }}{{ end }} From 7a7c33c8a828695878b1ed964553ccf6eab5fc09 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 9 Nov 2023 10:38:14 +0530 Subject: [PATCH 33/46] Add test for file input --- docker/jvm/launch | 4 +- docker/resources/common-scripts/configure | 4 +- .../{kafka.properties => server.properties} | 0 docker/test/constants.py | 8 ++- docker/test/docker_sanity_test.py | 59 ++++++++++--------- .../fixtures/file-input/server.properties | 26 ++++++++ .../{kraft => jvm}/docker-compose.yml | 11 ++++ 7 files changed, 79 insertions(+), 33 deletions(-) rename docker/resources/common-scripts/{kafka.properties => server.properties} (100%) create mode 100644 docker/test/fixtures/file-input/server.properties rename docker/test/fixtures/{kraft => jvm}/docker-compose.yml (91%) diff --git a/docker/jvm/launch b/docker/jvm/launch index a9844b60953e1..5d821d1e43773 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -39,7 +39,7 @@ fi echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this -result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ +result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } @@ -50,4 +50,4 @@ else fi # Start kafka broker -exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kafka.properties +exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index f3db5a2188b10..4e812ad7cd731 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -130,12 +130,12 @@ then fi fi -mv /etc/kafka/docker/kafka.properties /opt/kafka/config/kafka.properties +mv /etc/kafka/docker/server.properties /opt/kafka/config/server.properties mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties cp -R /mnt/shared/config/. /opt/kafka/config/ -echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/kafka.properties +echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/common-scripts/kafka.properties b/docker/resources/common-scripts/server.properties similarity index 100% rename from docker/resources/common-scripts/kafka.properties rename to docker/resources/common-scripts/server.properties diff --git a/docker/test/constants.py b/docker/test/constants.py index ace6ff9d66efa..adc88a44bce8a 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -17,7 +17,7 @@ KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" -KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" +JVM_COMPOSE="fixtures/jvm/docker-compose.yml" ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" CLIENT_TIMEOUT=40 @@ -26,9 +26,13 @@ SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" +FILE_INPUT_FLOW_TESTS="File Input Flow Tests" +FILE_INPUT_TOPIC="test-topic-file-input" + BROKER_RESTART_TESTS="Broker Restart Tests" BROKER_CONTAINER="broker" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" SSL_ERROR_PREFIX="SSL_ERR" -BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" \ No newline at end of file +BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" +FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 5571bc125bb06..1afb599e3759e 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -22,13 +22,13 @@ class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" - def resumeImage(self): + def resume_container(self): subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) - def stopImage(self) -> None: + def stop_container(self) -> None: subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) - def startCompose(self, filename) -> None: + def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" new_string=f"image: {self.IMAGE}" with open(filename) as f: @@ -39,7 +39,7 @@ def startCompose(self, filename) -> None: subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - def destroyCompose(self, filename) -> None: + def destroy_compose(self, filename) -> None: old_string=f"image: {self.IMAGE}" new_string="image: {$IMAGE}" subprocess.run(["docker-compose", "-f", filename, "down"]) @@ -71,37 +71,37 @@ def consume_message(self, topic, consumer_config): message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() - def ssl_flow(self): - print(f"Running {constants.SSL_FLOW_TESTS}") + def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): + print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) return errors - producer_config = ["--bootstrap-server", "localhost:9093", + producer_config = ["--bootstrap-server", ssl_broker_port, "--producer.config", constants.SSL_CLIENT_CONFIG] - self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") + self.produce_message(topic, producer_config, "key", "message") consumer_config = [ - "--bootstrap-server", "localhost:9093", + "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", "--consumer.config", constants.SSL_CLIENT_CONFIG, ] - message = self.consume_message(constants.SSL_TOPIC, consumer_config) + message = self.consume_message(topic, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) if errors: - print(f"Errors in {constants.SSL_FLOW_TESTS}:- {errors}") + print(f"Errors in {test_name}:- {errors}") else: - print(f"No errors in {constants.SSL_FLOW_TESTS}") + print(f"No errors in {test_name}") return errors def broker_restart_flow(self): @@ -118,9 +118,9 @@ def broker_restart_flow(self): self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") - self.stopImage() - print("Resuming Image") - self.resumeImage() + self.stop_container() + print("Resuming Container") + self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) @@ -142,37 +142,42 @@ def broker_restart_flow(self): def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow()) + total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: - print("SSL flow error", str(e)) + print(constants.SSL_ERROR_PREFIX, str(e)) + total_errors.append(str(e)) + try: + total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) + except Exception as e: + print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print("Broker restart flow error", str(e)) + print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) -class DockerSanityTestKraftMode(DockerSanityTest): +class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.startCompose(constants.KRAFT_COMPOSE) + self.start_compose(constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroyCompose(constants.KRAFT_COMPOSE) + self.destroy_compose(constants.JVM_COMPOSE) def test_bed(self): self.execute() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("image") - parser.add_argument("mode", default="all") + parser.add_argument("mode") args = parser.parse_args() DockerSanityTest.IMAGE = args.image test_classes_to_run = [] if args.mode == "jvm": - test_classes_to_run = [DockerSanityTestKraftMode] + test_classes_to_run = [DockerSanityTestJVM] loader = unittest.TestLoader() suites_list = [] diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties new file mode 100644 index 0000000000000..0ae8fa924f44a --- /dev/null +++ b/docker/test/fixtures/file-input/server.properties @@ -0,0 +1,26 @@ +advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 +controller.listener.names=CONTROLLER +controller.quorum.voters=3@broker-ssl-file-input:29093 +group.initial.rebalance.delay.ms=0 +inter.broker.listener.name=BROKER +listener.name.internal.ssl.endpoint.identification.algorithm= +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT +listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 +log.dirs=/tmp/kraft-combined-logs +node.id=3 +offsets.topic.replication.factor=1 +process.roles=broker,controller +ssl.client.auth=required +ssl.endpoint.identification.algorithm= +ssl.key.credentials=kafka_ssl_key_creds +ssl.key.password=abcdefgh +ssl.keystore.credentials=kafka_keystore_creds +ssl.keystore.filename=kafka01.keystore.jks +ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks +ssl.keystore.password=abcdefgh +ssl.truststore.credentials=kafka_truststore_creds +ssl.truststore.filename=kafka.truststore.jks +ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks +ssl.truststore.password=abcdefgh +transaction.state.log.min.isr=1 +transaction.state.log.replication.factor=1 diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml similarity index 91% rename from docker/test/fixtures/kraft/docker-compose.yml rename to docker/test/fixtures/jvm/docker-compose.yml index ed06c819917d0..de89fbaf3206f 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -69,3 +69,14 @@ services: KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + broker-ssl-file-input: + image: {$IMAGE} + hostname: broker-ssl-file-input + container_name: broker-ssl-file-input + ports: + - "9094:9093" + volumes: + - ../secrets:/etc/kafka/secrets + - ../file-input:/mnt/shared/config + environment: + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' From 1c8ada44d5a838c7a63851997efd00d766c2091e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 9 Nov 2023 16:35:52 +0530 Subject: [PATCH 34/46] Bubble up test errors to root build test script --- .github/workflows/docker_build_and_test.yml | 24 +++--- docker/docker_build_test.py | 5 +- docker/requirements.txt | 3 +- .../common-scripts/configureDefaults | 1 - docker/test/__init__.py | 0 docker/test/constants.py | 11 ++- docker/test/docker_sanity_test.py | 75 +++++++++---------- .../fixtures/file-input/server.properties | 5 -- .../fixtures/secrets/client-ssl.properties | 4 +- docker/test/requirements.txt | 1 - 10 files changed, 63 insertions(+), 66 deletions(-) create mode 100644 docker/test/__init__.py delete mode 100644 docker/test/requirements.txt diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 82a9a81a19ed2..af2aa121a3679 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -24,12 +24,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r docker/test/requirements.txt - - name: Run tests + pip install -r docker/requirements.txt + - name: Build image and run tests working-directory: ./docker run: | python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} - - name: Run Vulnerability scan + - name: Run CVE scan + if: always() uses: aquasecurity/trivy-action@master with: image-ref: 'kafka/test:test' @@ -37,12 +38,17 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' - output: scan_${{ github.event.inputs.image_type }}.txt - - uses: actions/upload-artifact@v3 + output: scan_report_${{ github.event.inputs.image_type }}.txt + exit-code: '1' + - name: Upload test report + if: always() + uses: actions/upload-artifact@v3 with: name: report_${{ github.event.inputs.image_type }}.html - path: docker/test/report_${{ github.event.inputs.image_type }}.html - - uses: actions/upload-artifact@v3 + path: docker/report_${{ github.event.inputs.image_type }}.html + - name: Upload CVE scan report + if: always() + uses: actions/upload-artifact@v3 with: - name: scan_${{ github.event.inputs.image_type }}.txt - path: scan_${{ github.event.inputs.image_type }}.txt \ No newline at end of file + name: scan_report_${{ github.event.inputs.image_type }}.txt + path: scan_report_${{ github.event.inputs.image_type }}.txt diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 5cefa723a2f4f..25704711d7617 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -20,6 +20,7 @@ import argparse from distutils.dir_util import copy_tree import shutil +from test.docker_sanity_test import run_tests def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' @@ -35,9 +36,11 @@ def run_jvm_tests(image, tag, kafka_url): subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) subprocess.run(["mkdir", "./test/fixtures/kafka"]) subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) - subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + failure_count = run_tests(f"{image}:{tag}", "jvm") subprocess.run(["rm", "kafka.tgz"]) shutil.rmtree("./test/fixtures/kafka") + if failure_count != 0: + raise SystemError("Test Failure. Error count is non 0") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/docker/requirements.txt b/docker/requirements.txt index 663bd1f6a2ae0..bc4fdd44eaf21 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1 +1,2 @@ -requests \ No newline at end of file +requests +HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index 6f5bfa47bf4ea..14d28548a83eb 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -17,7 +17,6 @@ declare -A env_defaults env_defaults=( # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" -# See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" ) diff --git a/docker/test/__init__.py b/docker/test/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/test/constants.py b/docker/test/constants.py index adc88a44bce8a..89c27e0c3baf6 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,17 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_TOPICS="./fixtures/kafka/bin/kafka-topics.sh" -KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" -KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" -JVM_COMPOSE="fixtures/jvm/docker-compose.yml" -ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" +JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" CLIENT_TIMEOUT=40 SSL_FLOW_TESTS="SSL Flow Tests" -SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" +SSL_CLIENT_CONFIG="./test/fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" FILE_INPUT_FLOW_TESTS="File Input Flow Tests" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 1afb599e3759e..242ab37de28f1 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -16,17 +16,16 @@ import unittest import subprocess from HTMLTestRunner import HTMLTestRunner -import constants -import argparse +import test.constants class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" def resume_container(self): - subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) + subprocess.run(["docker", "start", test.constants.BROKER_CONTAINER]) def stop_container(self) -> None: - subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) + subprocess.run(["docker", "stop", test.constants.BROKER_CONTAINER]) def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" @@ -50,44 +49,44 @@ def destroy_compose(self, filename) -> None: f.write(s) def create_topic(self, topic, topic_config): - command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [test.constants.KAFKA_TOPICS, "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [constants.KAFKA_TOPICS, "--list"] + check_command = [test.constants.KAFKA_TOPICS, "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command, timeout=test.constants.CLIENT_TIMEOUT) if topic in output.decode("utf-8"): return True return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", test.constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + subprocess.run(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [test.constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", test.constants.SSL_CLIENT_CONFIG])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", constants.SSL_CLIENT_CONFIG] + "--producer.config", test.constants.SSL_CLIENT_CONFIG] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", constants.SSL_CLIENT_CONFIG, + "--consumer.config", test.constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) try: @@ -105,17 +104,17 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): return errors def broker_restart_flow(self): - print(f"Running {constants.BROKER_RESTART_TESTS}") + print(f"Running {test.constants.BROKER_RESTART_TESTS}") errors = [] try: - self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + self.assertTrue(self.create_topic(test.constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] - self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + self.produce_message(test.constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") self.stop_container() @@ -123,60 +122,55 @@ def broker_restart_flow(self): self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] - message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + message = self.consume_message(test.constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) if errors: - print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") + print(f"Errors in {test.constants.BROKER_RESTART_TESTS}:- {errors}") else: - print(f"No errors in {constants.BROKER_RESTART_TESTS}") + print(f"No errors in {test.constants.BROKER_RESTART_TESTS}") return errors def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9093', test.constants.SSL_FLOW_TESTS, test.constants.SSL_ERROR_PREFIX, test.constants.SSL_TOPIC)) except Exception as e: - print(constants.SSL_ERROR_PREFIX, str(e)) + print(test.constants.SSL_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: - total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9094', test.constants.FILE_INPUT_FLOW_TESTS, test.constants.FILE_INPUT_ERROR_PREFIX, test.constants.FILE_INPUT_TOPIC)) except Exception as e: - print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) + print(test.constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) + print(test.constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.start_compose(constants.JVM_COMPOSE) + self.start_compose(test.constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroy_compose(constants.JVM_COMPOSE) + self.destroy_compose(test.constants.JVM_COMPOSE) def test_bed(self): self.execute() -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("image") - parser.add_argument("mode") - args = parser.parse_args() - - DockerSanityTest.IMAGE = args.image +def run_tests(image, mode): + DockerSanityTest.IMAGE = image test_classes_to_run = [] - if args.mode == "jvm": + if mode == "jvm": test_classes_to_run = [DockerSanityTestJVM] loader = unittest.TestLoader() @@ -185,10 +179,11 @@ def test_bed(self): suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report_{args.mode}.html", "w") + outfile = open(f"report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', description='This demonstrates the report output.' ) - runner.run(big_suite) \ No newline at end of file + result = runner.run(big_suite) + return result.failure_count diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 0ae8fa924f44a..9647fcb7f2455 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -12,14 +12,9 @@ offsets.topic.replication.factor=1 process.roles=broker,controller ssl.client.auth=required ssl.endpoint.identification.algorithm= -ssl.key.credentials=kafka_ssl_key_creds ssl.key.password=abcdefgh -ssl.keystore.credentials=kafka_keystore_creds -ssl.keystore.filename=kafka01.keystore.jks ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks ssl.keystore.password=abcdefgh -ssl.truststore.credentials=kafka_truststore_creds -ssl.truststore.filename=kafka.truststore.jks ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh transaction.state.log.min.isr=1 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 2ee9218a3fe16..5ecd5582f4e69 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -1,7 +1,7 @@ security.protocol=SSL -ssl.truststore.location=./fixtures/secrets/kafka.truststore.jks +ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh -ssl.keystore.location=./fixtures/secrets/client.keystore.jks +ssl.keystore.location=./test/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1 diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt deleted file mode 100644 index cb2ef3545ecf1..0000000000000 --- a/docker/test/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -HTMLTestRunner-Python3 \ No newline at end of file From 10b85b9578458c674c20abd895a79f0f8fa1ea2b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 10 Nov 2023 14:00:22 +0530 Subject: [PATCH 35/46] Add license comment and refactor ub scripts to remove redundant code --- docker/docker_promote.py | 2 - docker/docker_release.py | 2 - .../resources/common-scripts/log4j.properties | 15 ++ .../common-scripts/server.properties | 14 ++ .../common-scripts/tools-log4j.properties | 14 ++ docker/resources/ub/ub.go | 190 +----------------- docker/resources/ub/ub_test.go | 104 ---------- docker/test/docker_sanity_test.py | 60 +++--- .../fixtures/file-input/server.properties | 15 ++ .../fixtures/secrets/client-ssl.properties | 15 ++ 10 files changed, 105 insertions(+), 326 deletions(-) diff --git a/docker/docker_promote.py b/docker/docker_promote.py index ef8c8421ebf60..45e14932f7d53 100644 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,7 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# """ Python script to promote an rc image. diff --git a/docker/docker_release.py b/docker/docker_release.py index 344476a577bbc..b92aa64bd85e4 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,7 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# """ Python script to build and push docker image diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties index 148bd53b664b6..7621ac44f42b8 100644 --- a/docker/resources/common-scripts/log4j.properties +++ b/docker/resources/common-scripts/log4j.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/docker/resources/common-scripts/server.properties b/docker/resources/common-scripts/server.properties index e9f40eddc1727..caa597ad96df5 100644 --- a/docker/resources/common-scripts/server.properties +++ b/docker/resources/common-scripts/server.properties @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 controller.listener.names=CONTROLLER controller.quorum.voters=1@localhost:29093 diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties index 27d9fbee48bf5..84f0e09405ddd 100644 --- a/docker/resources/common-scripts/tools-log4j.properties +++ b/docker/resources/common-scripts/tools-log4j.properties @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. log4j.rootLogger=WARN, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender diff --git a/docker/resources/ub/ub.go b/docker/resources/ub/ub.go index 194b8480d53eb..6c84fd2c35c90 100644 --- a/docker/resources/ub/ub.go +++ b/docker/resources/ub/ub.go @@ -18,23 +18,15 @@ package main import ( "context" "encoding/json" - "errors" "fmt" "io" - "net" - "net/http" - "net/url" "os" - "os/exec" "os/signal" + pt "path" "regexp" "sort" - "strconv" "strings" "text/template" - "time" - - pt "path" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -50,11 +42,6 @@ type ConfigSpec struct { } var ( - bootstrapServers string - configFile string - zookeeperConnect string - security string - re = regexp.MustCompile("[^_]_[^_]") ensureCmd = &cobra.Command{ @@ -84,27 +71,6 @@ var ( Args: cobra.ExactArgs(1), RunE: runRenderPropertiesCmd, } - - waitCmd = &cobra.Command{ - Use: "wait ", - Short: "waits for a service to start listening on a port", - Args: cobra.ExactArgs(3), - RunE: runWaitCmd, - } - - httpReadyCmd = &cobra.Command{ - Use: "http-ready ", - Short: "waits for an HTTP/HTTPS URL to be retrievable", - Args: cobra.ExactArgs(2), - RunE: runHttpReadyCmd, - } - - kafkaReadyCmd = &cobra.Command{ - Use: "kafka-ready ", - Short: "checks if kafka brokers are up and running", - Args: cobra.ExactArgs(2), - RunE: runKafkaReadyCmd, - } ) func ensure(envVar string) bool { @@ -283,27 +249,6 @@ func loadConfigSpec(path string) (ConfigSpec, error) { return spec, nil } -func invokeJavaCommand(className string, jvmOpts string, args []string) bool { - classPath := getEnvOrDefault("UB_CLASSPATH", "/usr/share/java/cp-base-lite/*") - - opts := []string{} - if jvmOpts != "" { - opts = append(opts, jvmOpts) - } - opts = append(opts, "-cp", classPath, className) - cmd := exec.Command("java", append(opts[:], args...)...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - var exitError *exec.ExitError - if errors.As(err, &exitError) { - return exitError.ExitCode() == 0 - } - return false - } - return true -} - func getEnvOrDefault(envVar string, defaultValue string) string { val := os.Getenv(envVar) if len(val) == 0 { @@ -312,89 +257,6 @@ func getEnvOrDefault(envVar string, defaultValue string) string { return val } -func checkKafkaReady(minNumBroker string, timeout string, bootstrapServers string, zookeeperConnect string, configFile string, security string) bool { - - opts := []string{minNumBroker, timeout + "000"} - if bootstrapServers != "" { - opts = append(opts, "-b", bootstrapServers) - } - if zookeeperConnect != "" { - opts = append(opts, "-z", zookeeperConnect) - } - if configFile != "" { - opts = append(opts, "-c", configFile) - } - if security != "" { - opts = append(opts, "-s", security) - } - jvmOpts := os.Getenv("KAFKA_OPTS") - return invokeJavaCommand("io.confluent.admin.utils.cli.KafkaReadyCommand", jvmOpts, opts) -} - -func waitForServer(host string, port int, timeout time.Duration) bool { - address := fmt.Sprintf("%s:%d", host, port) - startTime := time.Now() - connectTimeout := 5 * time.Second - - for { - conn, err := net.DialTimeout("tcp", address, connectTimeout) - if err == nil { - _ = conn.Close() - return true - } - if time.Since(startTime) >= timeout { - return false - } - time.Sleep(1 * time.Second) - } -} - -func waitForHttp(URL string, timeout time.Duration) error { - parsedURL, err := url.Parse(URL) - if err != nil { - return fmt.Errorf("error in parsing url %q: %w", URL, err) - } - - host := parsedURL.Hostname() - portStr := parsedURL.Port() - - if len(host) == 0 { - host = "localhost" - } - - if len(portStr) == 0 { - switch parsedURL.Scheme { - case "http": - portStr = "80" - case "https": - portStr = "443" - default: - return fmt.Errorf("no port specified and cannot infer port based on protocol (only http(s) supported)") - } - } - port, err := strconv.Atoi(portStr) - if err != nil { - return fmt.Errorf("error in parsing port %q: %w", portStr, err) - } - - if !waitForServer(host, port, timeout) { - return fmt.Errorf("service is unreachable on host = %q, port = %q", host, portStr) - } - - httpClient := &http.Client{ - Timeout: timeout * time.Second, - } - resp, err := httpClient.Get(URL) - if err != nil { - return fmt.Errorf("error retrieving url") - } - statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 - if !statusOK { - return fmt.Errorf("unexpected response for %q with code %d", URL, resp.StatusCode) - } - return nil -} - func runEnsureCmd(_ *cobra.Command, args []string) error { success := ensure(args[0]) if !success { @@ -440,48 +302,6 @@ func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { return nil } -func runWaitCmd(_ *cobra.Command, args []string) error { - port, err := strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("error in parsing port %q: %w", args[1], err) - } - - secs, err := strconv.Atoi(args[2]) - if err != nil { - return fmt.Errorf("error in parsing timeout seconds %q: %w", args[2], err) - } - timeout := time.Duration(secs) * time.Second - - success := waitForServer(args[0], port, timeout) - if !success { - return fmt.Errorf("service is unreachable for host %q and port %q", args[0], args[1]) - } - return nil -} - -func runHttpReadyCmd(_ *cobra.Command, args []string) error { - secs, err := strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("error in parsing timeout seconds %q: %w", args[1], err) - } - timeout := time.Duration(secs) * time.Second - - success := waitForHttp(args[0], timeout) - if success != nil { - return fmt.Errorf("error in http-ready check for url %q: %w", args[0], success) - } - return nil -} - -func runKafkaReadyCmd(_ *cobra.Command, args []string) error { - success := checkKafkaReady(args[0], args[1], bootstrapServers, zookeeperConnect, configFile, security) - if !success { - err := fmt.Errorf("kafka-ready check failed") - return err - } - return nil -} - func main() { rootCmd := &cobra.Command{ Use: "ub", @@ -489,18 +309,10 @@ func main() { Run: func(cmd *cobra.Command, args []string) {}, } - kafkaReadyCmd.PersistentFlags().StringVarP(&bootstrapServers, "bootstrap-servers", "b", "", "comma-separated list of kafka brokers") - kafkaReadyCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "path to the config file") - kafkaReadyCmd.PersistentFlags().StringVarP(&zookeeperConnect, "zookeeper-connect", "z", "", "zookeeper connect string") - kafkaReadyCmd.PersistentFlags().StringVarP(&security, "security", "s", "", "security protocol to use when multiple listeners are enabled.") - rootCmd.AddCommand(pathCmd) rootCmd.AddCommand(ensureCmd) rootCmd.AddCommand(renderTemplateCmd) rootCmd.AddCommand(renderPropertiesCmd) - rootCmd.AddCommand(waitCmd) - rootCmd.AddCommand(httpReadyCmd) - rootCmd.AddCommand(kafkaReadyCmd) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index 70aedba34c504..b5031a2b86692 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -16,13 +16,9 @@ package main import ( - "net" - "net/http" - "net/http/httptest" "os" "reflect" "testing" - "time" ) func assertEqual(a string, b string, t *testing.T) { @@ -359,103 +355,3 @@ func Test_splitToMapDefaults(t *testing.T) { }) } } - -func Test_waitForServer(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer mockServer.Close() - port := mockServer.Listener.Addr().(*net.TCPAddr).Port - - type args struct { - host string - port int - timeout time.Duration - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "invalid server address", - args: args{ - host: "localhost", - port: port + 1, - timeout: time.Duration(5) * time.Second, - }, - want: false, - }, - { - name: "valid server address", - args: args{ - host: "localhost", - port: port, - timeout: time.Duration(5) * time.Second, - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := waitForServer(tt.args.host, tt.args.port, tt.args.timeout); !reflect.DeepEqual(got, tt.want) { - t.Errorf("waitForServer() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_waitForHttp(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/names" { - w.WriteHeader(http.StatusOK) - } else { - http.NotFound(w, r) - } - })) - defer mockServer.Close() - - serverURL := mockServer.URL - - type args struct { - URL string - timeout time.Duration - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "valid server address, valid url", - args: args{ - URL: serverURL + "/names", - timeout: time.Duration(5) * time.Second, - }, - wantErr: false, - }, - { - name: "valid server address, invalid url", - args: args{ - URL: serverURL, - timeout: time.Duration(5) * time.Second, - }, - wantErr: true, - }, - { - name: "invalid server address", - args: args{ - URL: "http://invalidAddress:50111/names", - timeout: time.Duration(5) * time.Second, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := waitForHttp(tt.args.URL, tt.args.timeout); (err != nil) != tt.wantErr { - t.Errorf("waitForHttp() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 242ab37de28f1..217649d506c18 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -16,16 +18,16 @@ import unittest import subprocess from HTMLTestRunner import HTMLTestRunner -import test.constants +import test.constants as constants class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" def resume_container(self): - subprocess.run(["docker", "start", test.constants.BROKER_CONTAINER]) + subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) def stop_container(self) -> None: - subprocess.run(["docker", "stop", test.constants.BROKER_CONTAINER]) + subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" @@ -49,44 +51,44 @@ def destroy_compose(self, filename) -> None: f.write(s) def create_topic(self, topic, topic_config): - command = [test.constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [test.constants.KAFKA_TOPICS, "--list"] + check_command = [constants.KAFKA_TOPICS, "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=test.constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) if topic in output.decode("utf-8"): return True return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", test.constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) + subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - command = [test.constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", test.constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", test.constants.SSL_CLIENT_CONFIG] + "--producer.config", constants.SSL_CLIENT_CONFIG] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", test.constants.SSL_CLIENT_CONFIG, + "--consumer.config", constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) try: @@ -104,17 +106,17 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): return errors def broker_restart_flow(self): - print(f"Running {test.constants.BROKER_RESTART_TESTS}") + print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] try: - self.assertTrue(self.create_topic(test.constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] - self.produce_message(test.constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") self.stop_container() @@ -122,47 +124,47 @@ def broker_restart_flow(self): self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] - message = self.consume_message(test.constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) if errors: - print(f"Errors in {test.constants.BROKER_RESTART_TESTS}:- {errors}") + print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") else: - print(f"No errors in {test.constants.BROKER_RESTART_TESTS}") + print(f"No errors in {constants.BROKER_RESTART_TESTS}") return errors def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow('localhost:9093', test.constants.SSL_FLOW_TESTS, test.constants.SSL_ERROR_PREFIX, test.constants.SSL_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: - print(test.constants.SSL_ERROR_PREFIX, str(e)) + print(constants.SSL_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: - total_errors.extend(self.ssl_flow('localhost:9094', test.constants.FILE_INPUT_FLOW_TESTS, test.constants.FILE_INPUT_ERROR_PREFIX, test.constants.FILE_INPUT_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) except Exception as e: - print(test.constants.FILE_INPUT_ERROR_PREFIX, str(e)) + print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print(test.constants.BROKER_RESTART_ERROR_PREFIX, str(e)) + print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.start_compose(test.constants.JVM_COMPOSE) + self.start_compose(constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroy_compose(test.constants.JVM_COMPOSE) + self.destroy_compose(constants.JVM_COMPOSE) def test_bed(self): self.execute() diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 9647fcb7f2455..17c5a5938a85b 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 controller.listener.names=CONTROLLER controller.quorum.voters=3@broker-ssl-file-input:29093 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 5ecd5582f4e69..df1b20f259e3a 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + security.protocol=SSL ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh From 625376e3b4d42f6c0d399a3eaf13177465747a52 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 10 Nov 2023 17:25:14 +0530 Subject: [PATCH 36/46] Add readme file and refactor the python scripts --- docker/README.md | 54 ++++++++++++++++++++++++++++++++++ docker/common.py | 11 +++++++ docker/docker_build_test.py | 18 +++++------- docker/docker_promote.py | 36 ++++++++++++----------- docker/docker_release.py | 32 ++++++++++---------- docker/resources/ub/ub_test.go | 8 ++--- 6 files changed, 110 insertions(+), 49 deletions(-) create mode 100644 docker/README.md create mode 100644 docker/common.py diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000..1b26da8505f80 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,54 @@ +Docker Images +============= + +This directory contains docker image for Kafka. +The scripts take a url containing kafka as input and generate the respective docker image. +There are interactive python scripts to release the docker image and promote a release candidate. + +Bulding image and running tests locally +--------------------------------------- +- `docker_build_test.py` script builds and tests the docker image. +- kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. +- Sanity tests for the docker image are present in test/docker_sanity_test.py. +- By default image will be built and tested, but if only build is required pass `-b` flag and if only testing the given image is required pass `-t` flag. +- An html test report will be generated after the tests are executed containing the results. + +Bulding image and running tests using github actions +---------------------------------------------------- +This is the recommended way to build, test and get a CVE report for the docker image. +Just choose the image type and provide kafka url to Docker build test workflow. It will generate a test report and CVE report that can be shared to the community. + +Creating a release +------------------ +`docker_release.py` provides an interactive way to build multi arch image and publish it a docker registry. + +Promoting a release +------------------- +`docker_promote.py` provides an interactive way to pull an RC Docker image and promote it to required dockerhub repo. + + +Using the image in a docker container +------------------------------------- +- The image uses the kafka downloaded from provided kafka url +- The image can be run in a container in default mode by running +`docker run -p 9092:9092` +- Default configs run kafka in kraft mode with plaintext listners on 9092 port. +- Default configs can be overriden by user using 2 ways:- + - By mounting folder containing property files + - Mount the folder containing kafka property files to `/mnt/shared/config` + - These files will override the default config files + - Using environment variables + - Kafka properties defined via env variables will override properties defined in file input + - Replace . with _ + - Replace _ with __(double underscore) + - Replace - with ___(triple underscore) + - Prefix the result with KAFKA_ + - Examples: + - For abc.def, use KAFKA_ABC_DEF + - For abc-def, use KAFKA_ABC___DEF + - For abc_def, use KAFKA_ABC__DEF +- Hence order of precedence of properties is the follwing:- + - Env variable (highest) + - File input + - Default (lowest) +- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts \ No newline at end of file diff --git a/docker/common.py b/docker/common.py new file mode 100644 index 0000000000000..450d97b33a5e2 --- /dev/null +++ b/docker/common.py @@ -0,0 +1,11 @@ +import subprocess + +def execute(command): + if subprocess.run(command).returncode != 0: + raise SystemError("Failure in executing following command:- ", " ".join(command)) + +def get_input(message): + value = input(message) + if value == "": + raise ValueError("This field cannot be empty") + return value \ No newline at end of file diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 25704711d7617..ddc900670bfcc 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,29 +15,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import subprocess from datetime import date import argparse from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests +from common import execute def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' copy_tree("resources", "jvm/resources") - result = subprocess.run(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + execute(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f'build_date={date.today()}', "jvm"]) - if result.stderr: - print(result.stdout) - return + shutil.rmtree("jvm/resources") def run_jvm_tests(image, tag, kafka_url): - subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - subprocess.run(["mkdir", "./test/fixtures/kafka"]) - subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) + execute(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) + execute(["mkdir", "./test/fixtures/kafka"]) + execute(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) failure_count = run_tests(f"{image}:{tag}", "jvm") - subprocess.run(["rm", "kafka.tgz"]) + execute(["rm", "kafka.tgz"]) shutil.rmtree("./test/fixtures/kafka") if failure_count != 0: raise SystemError("Test Failure. Error count is non 0") @@ -46,7 +44,7 @@ def run_jvm_tests(image, tag, kafka_url): parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", choices=["jvm"], dest="image_type", help="Image type you want to build") + parser.add_argument("-type", "--image-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") diff --git a/docker/docker_promote.py b/docker/docker_promote.py index 45e14932f7d53..75cd2fc06da1f 100644 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -25,13 +25,9 @@ Interactive utility to promote a docker image """ -import subprocess import requests from getpass import getpass - -def execute(command): - if subprocess.run(command).returncode != 0: - raise SystemError("Failure in executing following command:- ", " ".join(command)) +from common import execute, get_input def login(): execute(["docker", "login"]) @@ -58,26 +54,32 @@ def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64 is removed from dockerhub") if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-arm64", headers={"Authorization": f"JWT {token}"}).status_code != 204: raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64 is removed from dockerhub") - subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) - subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) + execute(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) + execute(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) if __name__ == "__main__": login() - username = input("Enter dockerhub username: ") + username = get_input("Enter dockerhub username: ") password = getpass("Enter dockerhub password: ") token = (requests.post("https://hub.docker.com/v2/users/login/", json={"username": username, "password": password})).json()['token'] if len(token) == 0: raise PermissionError("Dockerhub login failed") - rc_image = input("Enter the RC docker image that you want to pull (in the format ::): ") - promotion_image_namespace = input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") - promotion_image_name = input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") - promotion_image_tag = input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: 4.0.0]: ") + rc_image = get_input("Enter the RC docker image that you want to pull (in the format //:): ") + promotion_image_namespace = get_input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") + promotion_image_name = get_input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") + promotion_image_tag = get_input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: latest]: ") promotion_image = f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}" - pull(rc_image, promotion_image) - push(promotion_image) - push_manifest(promotion_image) - remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) - print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") \ No newline at end of file + print(f"Docker image {rc_image} will be pulled and pushed to {promotion_image}") + + proceed = input("Should we proceed? [y/N]: ") + if proceed == "y": + pull(rc_image, promotion_image) + push(promotion_image) + push_manifest(promotion_image) + remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) + print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") + else: + print("Image promotion aborted") \ No newline at end of file diff --git a/docker/docker_release.py b/docker/docker_release.py index b92aa64bd85e4..1c55d0779bd3b 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -22,30 +22,34 @@ Interactive utility to push the docker image to dockerhub """ -import subprocess from distutils.dir_util import copy_tree from datetime import date import shutil +from common import execute, get_input + def push_jvm(image, kafka_url): copy_tree("resources", "jvm/resources") - subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", + execute(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", "--push", "--platform", "linux/amd64,linux/arm64", "--tag", image, "jvm"]) shutil.rmtree("jvm/resources") def login(): - status = subprocess.run(["docker", "login"]) - if status.returncode != 0: - print("Docker login failed, aborting the docker release") - raise PermissionError + execute(["docker", "login"]) def create_builder(): - subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) + execute(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) def remove_builder(): - subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) + execute(["docker", "buildx", "rm", "kafka-builder"]) + +def get_input(message): + value = input(message) + if value == "": + raise ValueError("This field cannot be empty") + return value if __name__ == "__main__": print("\ @@ -56,15 +60,9 @@ def remove_builder(): if docker_registry == "": docker_registry = "docker.io" docker_namespace = input("Enter the docker namespace you want to push the image to: ") - image_name = input("Enter the image name: ") - if image_name == "": - raise ValueError("image name cannot be empty") - image_tag = input("Enter the image tag for the image: ") - if image_tag == "": - raise ValueError("image tag cannot be empty") - kafka_url = input("Enter the url for kafka binary tarball: ") - if kafka_url == "": - raise ValueError("kafka url cannot be empty") + image_name = get_input("Enter the image name: ") + image_tag = get_input("Enter the image tag for the image: ") + kafka_url = get_input("Enter the url for kafka binary tarball: ") image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") proceed = input("Should we proceed? [y/N]: ") diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index b5031a2b86692..9d09c5c413e27 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -261,7 +261,6 @@ func Test_buildProperties(t *testing.T) { environment: map[string]string{ "PATH": "thePath", "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "CONFLUENT_METRICS": "metricsValue", "KAFKA_IGNORED": "ignored", "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", }, @@ -272,7 +271,7 @@ func Test_buildProperties(t *testing.T) { name: "server properties", args: args{ spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Prefixes: map[string]bool{"KAFKA": false}, Excludes: []string{"KAFKA_IGNORED"}, Renamed: map[string]string{}, Defaults: map[string]string{ @@ -284,18 +283,17 @@ func Test_buildProperties(t *testing.T) { environment: map[string]string{ "PATH": "thePath", "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "CONFLUENT_METRICS": "metricsValue", "KAFKA_IGNORED": "ignored", "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", }, }, - want: map[string]string{"bootstrap.servers": "localhost:9092", "confluent.metrics": "metricsValue", "default.property.key": "default.property.value"}, + want: map[string]string{"bootstrap.servers": "localhost:9092", "default.property.key": "default.property.value"}, }, { name: "kafka properties", args: args{ spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Prefixes: map[string]bool{"KAFKA": false}, Excludes: []string{"KAFKA_IGNORED"}, Renamed: map[string]string{}, Defaults: map[string]string{ From d9cbdd85480b7cfb856b84ecd5e42cffa68e2af1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 15 Nov 2023 10:40:27 +0530 Subject: [PATCH 37/46] Add local setup section in Readme --- docker/README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docker/README.md b/docker/README.md index 1b26da8505f80..98730bb70c3a0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,22 +5,29 @@ This directory contains docker image for Kafka. The scripts take a url containing kafka as input and generate the respective docker image. There are interactive python scripts to release the docker image and promote a release candidate. +Local Setup +----------- +Make sure you have python (>= 3.7.x) and java (>= 17) installed before running the tests and scripts. + +Run `pip install -r requirements.txt` to get all the requirements for running the scripts. + + Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. - kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. - Sanity tests for the docker image are present in test/docker_sanity_test.py. -- By default image will be built and tested, but if only build is required pass `-b` flag and if only testing the given image is required pass `-t` flag. +- By default image will be built and tested, but if you only want to build the image, pass `-b` flag and if you only want to test the given image pass `-t` flag. - An html test report will be generated after the tests are executed containing the results. Bulding image and running tests using github actions ---------------------------------------------------- This is the recommended way to build, test and get a CVE report for the docker image. -Just choose the image type and provide kafka url to Docker build test workflow. It will generate a test report and CVE report that can be shared to the community. +Just choose the image type and provide kafka url to `Docker build test` workflow. It will generate a test report and CVE report that can be shared to the community. Creating a release ------------------ -`docker_release.py` provides an interactive way to build multi arch image and publish it a docker registry. +`docker_release.py` provides an interactive way to build multi arch image and publish it to a docker registry. Promoting a release ------------------- @@ -51,4 +58,4 @@ Using the image in a docker container - Env variable (highest) - File input - Default (lowest) -- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts \ No newline at end of file +- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts From 3dcdf2d22892f7c21f6da4a2f257aa1364c30fa9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 15 Nov 2023 15:50:57 +0530 Subject: [PATCH 38/46] Update file input test and readme file --- .gitignore | 2 ++ docker/README.md | 2 -- docker/test/fixtures/file-input/server.properties | 3 +-- docker/test/fixtures/jvm/docker-compose.yml | 4 ++++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f466af2c59828..7bf18c57cc6b0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,5 @@ jmh-benchmarks/src/main/generated **/src/generated-test storage/kafka-tiered-storage/ + +docker/report_*.html diff --git a/docker/README.md b/docker/README.md index 98730bb70c3a0..2b20cc6164101 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,6 @@ Make sure you have python (>= 3.7.x) and java (>= 17) installed before running t Run `pip install -r requirements.txt` to get all the requirements for running the scripts. - Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. @@ -33,7 +32,6 @@ Promoting a release ------------------- `docker_promote.py` provides an interactive way to pull an RC Docker image and promote it to required dockerhub repo. - Using the image in a docker container ------------------------------------- - The image uses the kafka downloaded from provided kafka url diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 17c5a5938a85b..cb5d420c76e6a 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -22,9 +22,8 @@ listener.name.internal.ssl.endpoint.identification.algorithm= listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 log.dirs=/tmp/kraft-combined-logs -node.id=3 offsets.topic.replication.factor=1 -process.roles=broker,controller +process.roles=invalid,value ssl.client.auth=required ssl.endpoint.identification.algorithm= ssl.key.password=abcdefgh diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml index de89fbaf3206f..73238f951be34 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -80,3 +80,7 @@ services: - ../file-input:/mnt/shared/config environment: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set a property absent from the file + KAFKA_NODE_ID: 3 + # Override an existing property + KAFKA_PROCESS_ROLES: 'broker,controller' \ No newline at end of file From 01619a0510dcaaebc33fa6d994e4c7ca1d21d472 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 16 Nov 2023 19:04:06 +0530 Subject: [PATCH 39/46] Add test for broker metrics --- docker/test/constants.py | 9 ++- docker/test/docker_sanity_test.py | 73 ++++++++++++++++----- docker/test/fixtures/jvm/docker-compose.yml | 3 + 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/docker/test/constants.py b/docker/test/constants.py index 89c27e0c3baf6..4f1fc3ce59ecf 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -16,6 +16,7 @@ KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_RUN_CLASS="./test/fixtures/kafka/bin/kafka-run-class.sh" JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" @@ -32,6 +33,12 @@ BROKER_CONTAINER="broker" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" +BROKER_METRICS_TESTS="Broker Metrics Tests" +BROKER_METRICS_TEST_TOPIC="test-topic-broker-metrics" +JMX_TOOL="org.apache.kafka.tools.JmxTool" +BROKER_METRICS_HEADING='"time","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:Count","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:EventType","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:FifteenMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:FiveMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:MeanRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:OneMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:RateUnit"' + SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" -FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" \ No newline at end of file +FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" +BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 217649d506c18..b35ba0123a198 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -71,6 +71,55 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() + + def get_metrics(self, jmx_tool_config): + command = [constants.KAFKA_RUN_CLASS, constants.JMX_TOOL] + command.extend(jmx_tool_config) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + return message.decode("utf-8").strip().split() + + def broker_metrics_flow(self): + print(f"Running {constants.BROKER_METRICS_TESTS}") + errors = [] + try: + self.assertTrue(self.create_topic(constants.BROKER_METRICS_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + jmx_tool_config = ["--one-time", "--object-name", "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", "--jmx-url", "service:jmx:rmi:///jndi/rmi://:9101/jmxrmi"] + metrics_before_message = self.get_metrics(jmx_tool_config) + try: + self.assertEqual(len(metrics_before_message), 2) + self.assertEqual(metrics_before_message[0], constants.BROKER_METRICS_HEADING) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] + self.produce_message(constants.BROKER_METRICS_TEST_TOPIC, producer_config, "key", "message") + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] + message = self.consume_message(constants.BROKER_METRICS_TEST_TOPIC, consumer_config) + try: + self.assertEqual(message, "key:message") + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + + metrics_after_message = self.get_metrics(jmx_tool_config) + try: + self.assertEqual(len(metrics_before_message), 2) + self.assertEqual(metrics_after_message[0], constants.BROKER_METRICS_HEADING) + before_metrics_data, after_metrics_data = metrics_before_message[1].split(","), metrics_after_message[1].split(",") + self.assertEqual(len(before_metrics_data), len(after_metrics_data)) + for i in range(len(before_metrics_data)): + if after_metrics_data[i].replace(".", "").isnumeric(): + self.assertGreater(float(after_metrics_data[i]), float(before_metrics_data[i])) + else: + self.assertEqual(after_metrics_data[i], before_metrics_data[i]) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + + return errors def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") @@ -91,18 +140,11 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): "--consumer.config", constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(test_error_prefix + str(e)) try: self.assertEqual(message, "key:message") except AssertionError as e: errors.append(test_error_prefix + str(e)) - if errors: - print(f"Errors in {test_name}:- {errors}") - else: - print(f"No errors in {test_name}") + return errors def broker_restart_flow(self): @@ -125,23 +167,20 @@ def broker_restart_flow(self): consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - return errors try: self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - if errors: - print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") - else: - print(f"No errors in {constants.BROKER_RESTART_TESTS}") + return errors def execute(self): total_errors = [] + try: + total_errors.extend(self.broker_metrics_flow()) + except Exception as e: + print(constants.BROKER_METRICS_ERROR_PREFIX, str(e)) + total_errors.append(str(e)) try: total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml index 73238f951be34..64a89967fdf28 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -22,6 +22,7 @@ services: container_name: broker ports: - "9092:9092" + - "9101:9101" environment: KAFKA_NODE_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' @@ -37,6 +38,8 @@ services: KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost broker-ssl: image: {$IMAGE} From 6a1f038ae7a49481b06034f9a9215738c979fc78 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 16 Nov 2023 19:17:42 +0530 Subject: [PATCH 40/46] Remove potential flakiness from the test --- docker/test/docker_sanity_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index b35ba0123a198..522927545de78 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -113,7 +113,7 @@ def broker_metrics_flow(self): self.assertEqual(len(before_metrics_data), len(after_metrics_data)) for i in range(len(before_metrics_data)): if after_metrics_data[i].replace(".", "").isnumeric(): - self.assertGreater(float(after_metrics_data[i]), float(before_metrics_data[i])) + self.assertGreaterEqual(float(after_metrics_data[i]), float(before_metrics_data[i])) else: self.assertEqual(after_metrics_data[i], before_metrics_data[i]) except AssertionError as e: From c81e63a5ea255afb40e8ccb3b7ee25d53149881e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 09:53:40 +0530 Subject: [PATCH 41/46] Remove redundant paths in chown and chmod --- docker/jvm/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 0e0fd6ff623db..e6135ef249f76 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -75,12 +75,12 @@ RUN set -eux ; \ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ gpg --import KEYS; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ - mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ - chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka; \ - chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /etc/kafka; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \ rm kafka.tgz kafka.tgz.asc KEYS; \ apk del curl wget gpg dirmngr gpg-agent; \ apk cache clean; From 3677b08039387594308314493475d709a372b002 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 11:57:57 +0530 Subject: [PATCH 42/46] Fix build by adding license and excluding files where it cannot be added --- .github/workflows/docker_build_and_test.yml | 15 +++++++++++ build.gradle | 4 ++- docker/common.py | 17 +++++++++++++ docker/jvm/jsa_launch | 14 +++++++++++ docker/jvm/launch | 25 ++++++++----------- docker/requirements.txt | 14 +++++++++++ .../kafka-log4j.properties.template | 14 +++++++++++ .../kafka-tools-log4j.properties.template | 14 +++++++++++ docker/resources/ub/testResources/sampleFile | 14 +++++++++++ docker/resources/ub/testResources/sampleFile2 | 14 +++++++++++ .../ub/testResources/sampleLog4j.template | 14 +++++++++++ docker/test/__init__.py | 16 ++++++++++++ 12 files changed, 160 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index af2aa121a3679..47fe4d0929a9e 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Docker build test on: diff --git a/build.gradle b/build.gradle index 3b7878a3e8955..1e1ba0b546e3a 100644 --- a/build.gradle +++ b/build.gradle @@ -207,7 +207,9 @@ if (repo != null) { 'streams/streams-scala/logs/*', 'licenses/*', '**/generated/**', - 'clients/src/test/resources/serializedData/*' + 'clients/src/test/resources/serializedData/*', + 'docker/resources/ub/go.sum', + 'docker/test/fixtures/secrets/*' ]) } } else { diff --git a/docker/common.py b/docker/common.py index 450d97b33a5e2..66c4888c17853 100644 --- a/docker/common.py +++ b/docker/common.py @@ -1,3 +1,20 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import subprocess def execute(command): diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 7f40f86f97058..eac2188f272aa 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties diff --git a/docker/jvm/launch b/docker/jvm/launch index 5d821d1e43773..8c5fec1bfc568 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### # Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. diff --git a/docker/requirements.txt b/docker/requirements.txt index bc4fdd44eaf21..f2854bc69ced5 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,2 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requests HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index f9da46cc7443d..7bbc37353e9de 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. {{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}{{ if ne $value "INFO" }} log4j.rootLogger={{ $value }}, stdout {{ end }}{{ end }} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index 4c55b9bb9a2ae..9a7acb94c0438 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. {{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} {{if ne $value "WARN"}} log4j.rootLogger={{ $value }}, stderr {{ end }}{{ end }} diff --git a/docker/resources/ub/testResources/sampleFile b/docker/resources/ub/testResources/sampleFile index e69de29bb2d1d..91eacc92e8be9 100755 --- a/docker/resources/ub/testResources/sampleFile +++ b/docker/resources/ub/testResources/sampleFile @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleFile2 b/docker/resources/ub/testResources/sampleFile2 index e69de29bb2d1d..91eacc92e8be9 100755 --- a/docker/resources/ub/testResources/sampleFile2 +++ b/docker/resources/ub/testResources/sampleFile2 @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template index 6a6a8630b2c10..8bc1f5e3dbd4d 100644 --- a/docker/resources/ub/testResources/sampleLog4j.template +++ b/docker/resources/ub/testResources/sampleLog4j.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout {{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} diff --git a/docker/test/__init__.py b/docker/test/__init__.py index e69de29bb2d1d..977976beec24a 100644 --- a/docker/test/__init__.py +++ b/docker/test/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file From 5193ca1d2872438f086412790f554682a70752cb Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 17:40:46 +0530 Subject: [PATCH 43/46] Update documentation to include docker image in the release process --- docker/README.md | 11 +++++++++++ docker/docker_release.py | 2 ++ release.py | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/docker/README.md b/docker/README.md index 2b20cc6164101..005ce4bec1741 100644 --- a/docker/README.md +++ b/docker/README.md @@ -57,3 +57,14 @@ Using the image in a docker container - File input - Default (lowest) - Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts + +Steps to release docker image +----------------------------- +- Make sure you have executed release.py script to prepare RC tarball in apache sftp server. +- Use the RC tarball url as input kafka url to build docker image and run sanity tests. +- Trigger github actions workflow using the RC branch, provide RC tarball url as kafka url. +- This will generate test report and CVE report for docker images. +- If the reports look fine, RC docker image can be built and published. +- Execute `docker_release.py` script to build and publish RC docker image in your own dockerhub account. +- Share the RC docker image, test report and CVE report with the community in RC vote email. +- Once approved and ready, take help from someone in PMC to trigger `docker_promote.py` script and promote the RC docker image to apache/kafka dockerhub repo diff --git a/docker/docker_release.py b/docker/docker_release.py index 1c55d0779bd3b..3a774ef11f9e4 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -17,6 +17,8 @@ """ Python script to build and push docker image +This script is used to prepare and publish docker release candidate + Usage: docker_release.py Interactive utility to push the docker image to dockerhub diff --git a/release.py b/release.py index 8458323ea9d5e..ca9af6c86b7d7 100755 --- a/release.py +++ b/release.py @@ -759,6 +759,10 @@ def select_gpg_key(): * Release artifacts to be voted upon (source and binary): https://home.apache.org/~%(apache_id)s/kafka-%(rc_tag)s/ + +* Docker release artifact to be voted upon: +/: + * Maven artifacts to be voted upon: https://repository.apache.org/content/groups/staging/org/apache/kafka/ @@ -777,6 +781,7 @@ def select_gpg_key(): * Successful Jenkins builds for the %(dev_branch)s branch: Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s// System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s// +Docker Build Test Pipeline: https://github.com/apache/kafka/actions/runs/ /************************************** From da0604312215d80229991a2ede00203bde3b53a5 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 23 Nov 2023 15:39:04 +0530 Subject: [PATCH 44/46] Add CDS for storage format --- docker/jvm/Dockerfile | 1 + docker/jvm/jsa_launch | 6 ++++-- docker/jvm/launch | 16 +++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e6135ef249f76..e5c9d7a47cb9a 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -86,6 +86,7 @@ RUN set -eux ; \ apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa +COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index eac2188f272aa..50d620d6724ac 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -14,8 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" -opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties +KAFKA_CLUSTER_ID="5L6g3nShT-eMCtK--X86sw" + +KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=storage.jsa" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties + KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & check_timeout() { diff --git a/docker/jvm/launch b/docker/jvm/launch index 8c5fec1bfc568..ccab3e02cda7a 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -34,17 +34,23 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi +if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then + export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="" +else + export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS" +fi + +export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/storage.jsa" + echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } -if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then - export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" -else - export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" -fi +export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS" + +export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties From 12297ee60388e9287c1740e22c2db254ef7a70c7 Mon Sep 17 00:00:00 2001 From: Krishna Agarwal <62741600+kagarwal06@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:04:25 +0530 Subject: [PATCH 45/46] Merge branch 'trunk' into native-docker-image --- .github/workflows/docker_build_and_test.yml | 73 +- .gitignore | 3 +- LICENSE-binary | 30 +- NOTICE-binary | 9 +- bin/connect-standalone.sh | 2 +- build.gradle | 85 +- checkstyle/import-control-core.xml | 1 + checkstyle/import-control-metadata.xml | 5 - checkstyle/import-control-server-common.xml | 4 - checkstyle/import-control-server.xml | 72 + checkstyle/import-control-storage.xml | 4 - checkstyle/import-control.xml | 12 + checkstyle/suppressions.xml | 26 +- .../org/apache/kafka/clients/ClientUtils.java | 14 +- .../clients/ClusterConnectionStates.java | 36 +- .../kafka/clients/CommonClientConfigs.java | 13 + .../org/apache/kafka/clients/Metadata.java | 7 + .../apache/kafka/clients/MetadataCache.java | 11 +- .../apache/kafka/clients/NetworkClient.java | 153 +- .../org/apache/kafka/clients/admin/Admin.java | 20 + .../admin/ClientMetricsResourceListing.java | 54 + .../kafka/clients/admin/ForwardingAdmin.java | 5 + .../kafka/clients/admin/KafkaAdminClient.java | 91 +- .../ListClientMetricsResourcesOptions.java | 29 + .../ListClientMetricsResourcesResult.java | 57 + .../clients/consumer/ConsumerConfig.java | 13 +- .../clients/consumer/ConsumerRecords.java | 2 +- .../kafka/clients/consumer/KafkaConsumer.java | 971 +----------- .../internals/AbstractCoordinator.java | 17 + ...cConsumer.java => AsyncKafkaConsumer.java} | 1015 +++++++++---- .../internals/CommitRequestManager.java | 257 +++- .../internals/ConsumerCoordinator.java | 7 +- .../consumer/internals/ConsumerDelegate.java | 42 + .../internals/ConsumerDelegateCreator.java | 115 ++ .../internals/ConsumerNetworkThread.java | 146 +- .../consumer/internals/ConsumerUtils.java | 19 +- .../internals/CoordinatorRequestManager.java | 2 +- .../consumer/internals/FetchBuffer.java | 15 +- .../consumer/internals/FetchCollector.java | 6 +- .../internals/HeartbeatRequestManager.java | 270 +++- .../internals/LegacyKafkaConsumer.java | 1276 ++++++++++++++++ .../consumer/internals/MemberState.java | 96 +- .../consumer/internals/MembershipManager.java | 70 +- .../internals/MembershipManagerImpl.java | 938 +++++++++++- .../internals/NetworkClientDelegate.java | 35 +- .../consumer/internals/RequestManager.java | 15 + .../consumer/internals/RequestManagers.java | 24 +- .../consumer/internals/RequestState.java | 10 +- .../clients/consumer/internals/Utils.java | 23 +- .../consumer/internals/WakeupTrigger.java | 56 +- .../internals/events/ApplicationEvent.java | 3 +- .../events/ApplicationEventHandler.java | 12 + .../events/ApplicationEventProcessor.java | 43 +- .../internals/events/BackgroundEvent.java | 1 + .../events/BackgroundEventProcessor.java | 75 - .../events/GroupMetadataUpdateEvent.java | 79 + .../SubscriptionChangeApplicationEvent.java | 30 + .../events/UnsubscribeApplicationEvent.java | 32 + .../kafka/clients/producer/KafkaProducer.java | 31 +- .../internals/BrokerSecurityConfigs.java | 10 + .../errors/TelemetryTooLargeException.java | 28 + .../UnknownSubscriptionIdException.java | 28 + .../apache/kafka/common/protocol/ApiKeys.java | 3 +- .../apache/kafka/common/protocol/Errors.java | 6 +- .../kafka/common/record/ConvertedRecords.java | 10 +- .../kafka/common/record/FileRecords.java | 2 +- .../record/LazyDownConversionRecordsSend.java | 10 +- .../kafka/common/record/MultiRecordsSend.java | 4 +- ...nStats.java => RecordValidationStats.java} | 17 +- .../kafka/common/record/RecordsUtil.java | 2 +- .../requests/AbstractControlRequest.java | 28 + .../common/requests/AbstractRequest.java | 2 + .../common/requests/AbstractResponse.java | 2 + .../common/requests/ApiVersionsResponse.java | 26 +- .../requests/AssignReplicasToDirsRequest.java | 7 + .../ConsumerGroupDescribeRequest.java | 13 + .../ConsumerGroupHeartbeatRequest.java | 6 + .../common/requests/FetchSnapshotRequest.java | 2 + .../common/requests/JoinGroupResponse.java | 6 + .../common/requests/LeaderAndIsrRequest.java | 24 - .../common/requests/LeaveGroupResponse.java | 6 +- .../ListClientMetricsResourcesRequest.java | 77 + .../ListClientMetricsResourcesResponse.java | 77 + .../common/requests/ProduceResponse.java | 26 +- .../common/requests/PushTelemetryRequest.java | 34 +- .../requests/UpdateMetadataRequest.java | 18 +- .../kafka/common/security/ssl/SslFactory.java | 34 +- .../internals/ClientTelemetryEmitter.java | 60 + .../internals/ClientTelemetryProvider.java | 149 ++ .../internals/ClientTelemetryReporter.java | 984 ++++++++++++ .../internals/ClientTelemetrySender.java | 35 + .../internals/ClientTelemetryUtils.java | 207 +++ .../internals/KafkaMetricsCollector.java | 346 +++++ .../telemetry/internals/LastValueTracker.java | 87 ++ .../telemetry/internals/MetricsEmitter.java | 11 +- .../internals/SinglePointMetric.java | 115 +- .../kafka/common/utils/ConfigUtils.java | 21 + .../kafka/common/utils/FlattenedIterator.java | 2 +- .../org/apache/kafka/common/utils/Utils.java | 13 +- .../message/BrokerRegistrationRequest.json | 12 +- .../ConsumerGroupDescribeResponse.json | 2 +- .../ConsumerGroupHeartbeatRequest.json | 4 - .../GetTelemetrySubscriptionsRequest.json | 4 - .../ListClientMetricsResourcesRequest.json | 26 + .../ListClientMetricsResourcesResponse.json | 30 + .../common/message/OffsetCommitRequest.json | 3 - .../common/message/OffsetFetchRequest.json | 4 - .../common/message/PushTelemetryRequest.json | 4 - .../common/message/UpdateMetadataRequest.json | 3 + .../clients/ClusterConnectionStatesTest.java | 19 + .../kafka/clients/MetadataCacheTest.java | 36 + .../kafka/clients/NetworkClientTest.java | 119 +- .../clients/admin/KafkaAdminClientTest.java | 115 +- .../kafka/clients/admin/MockAdminClient.java | 76 +- .../clients/consumer/ConsumerConfigTest.java | 31 +- .../clients/consumer/KafkaConsumerTest.java | 1328 +++++++++++------ .../internals/AsyncKafkaConsumerTest.java | 1108 ++++++++++++++ .../internals/CommitRequestManagerTest.java | 95 +- .../internals/ConsumerCoordinatorTest.java | 10 +- .../internals/ConsumerNetworkThreadTest.java | 116 +- .../internals/ConsumerTestBuilder.java | 77 +- .../consumer/internals/FetchBufferTest.java | 16 + .../consumer/internals/FetcherTest.java | 84 ++ .../HeartbeatRequestManagerTest.java | 313 +++- .../internals/MembershipManagerImplTest.java | 1042 +++++++++++-- .../internals/NetworkClientDelegateTest.java | 25 +- .../internals/PrototypeAsyncConsumerTest.java | 447 ------ .../TopicMetadataRequestManagerTest.java | 3 +- .../consumer/internals/WakeupTriggerTest.java | 76 +- .../events/BackgroundEventHandlerTest.java | 141 -- .../clients/producer/KafkaProducerTest.java | 91 +- .../kafka/common/network/NioEchoServer.java | 53 +- .../kafka/common/network/SelectorTest.java | 7 +- .../common/network/SslTransportLayerTest.java | 16 +- .../network/SslTransportTls12Tls13Test.java | 3 + .../SslVersionsTransportLayerTest.java | 2 + .../record/MemoryRecordsBuilderTest.java | 2 +- .../requests/ApiVersionsResponseTest.java | 59 +- .../BrokerRegistrationRequestTest.java | 76 + .../ConsumerGroupDescribeRequestTest.java | 23 + .../requests/JoinGroupResponseTest.java | 15 + .../requests/LeaveGroupResponseTest.java | 57 + .../common/requests/RequestResponseTest.java | 15 + .../authenticator/SaslAuthenticatorTest.java | 1 + .../secured/RefreshingHttpsJwksTest.java | 47 +- .../common/security/ssl/SslFactoryTest.java | 39 +- .../internals/ClientTelemetryEmitterTest.java | 103 ++ .../ClientTelemetryReporterTest.java | 619 ++++++++ .../internals/ClientTelemetryUtilsTest.java | 115 ++ .../internals/KafkaMetricsCollectorTest.java | 605 ++++++++ .../internals/LastValueTrackerTest.java | 106 ++ .../internals/SinglePointMetricTest.java | 170 +++ .../telemetry/internals/TestEmitter.java | 70 + .../common/utils/AbstractIteratorTest.java | 2 +- .../kafka/common/utils/ConfigUtilsTest.java | 26 + .../java/org/apache/kafka/test/TestUtils.java | 4 +- .../connect/mirror/MirrorSourceTask.java | 33 +- .../connect/mirror/MirrorSourceTaskTest.java | 63 + .../MirrorConnectorsIntegrationBaseTest.java | 29 + .../kafka/connect/cli/ConnectStandalone.java | 84 +- .../apache/kafka/connect/runtime/Herder.java | 13 + .../distributed/DistributedHerder.java | 22 +- .../rest/entities/CreateConnectorRequest.java | 49 +- .../rest/resources/ConnectorsResource.java | 2 +- .../runtime/standalone/StandaloneHerder.java | 11 +- .../connect/storage/ConfigBackingStore.java | 4 +- .../storage/KafkaConfigBackingStore.java | 36 +- .../storage/MemoryConfigBackingStore.java | 19 +- .../connect/cli/ConnectStandaloneTest.java | 127 ++ .../ConnectWorkerIntegrationTest.java | 233 ++- .../RestForwardingIntegrationTest.java | 5 +- .../StandaloneWorkerIntegrationTest.java | 53 + .../distributed/DistributedHerderTest.java | 51 +- .../entities/CreateConnectorRequestTest.java | 53 + .../resources/ConnectorsResourceTest.java | 90 +- .../standalone/StandaloneHerderTest.java | 31 + .../storage/KafkaConfigBackingStoreTest.java | 71 +- .../storage/MemoryConfigBackingStoreTest.java | 32 +- .../util/clusters/ConnectAssertions.java | 4 +- .../util/clusters/EmbeddedConnect.java | 33 +- .../kafka/log/remote/RemoteLogManager.java | 74 +- .../kafka/server/ClientMetricsManager.java | 46 - .../server/builders/KafkaApisBuilder.java | 2 +- .../builders/ReplicaManagerBuilder.java | 10 +- core/src/main/scala/kafka/Kafka.scala | 1 - .../scala/kafka/admin/ConfigCommand.scala | 38 +- .../main/scala/kafka/cluster/Partition.scala | 6 + .../controller/ControllerChannelManager.scala | 21 +- .../group/CoordinatorLoaderImpl.scala | 6 +- .../group/CoordinatorPartitionWriter.scala | 116 +- .../group/GroupCoordinatorAdapter.scala | 11 +- .../transaction/ProducerIdManager.scala | 2 +- core/src/main/scala/kafka/log/LocalLog.scala | 34 +- .../src/main/scala/kafka/log/UnifiedLog.scala | 92 +- .../kafka/migration/MigrationPropagator.scala | 5 +- .../kafka/network/RequestConvertToJson.scala | 46 +- .../kafka/server/AlterPartitionManager.scala | 3 +- .../kafka/server/ApiVersionManager.scala | 18 +- .../server/AutoTopicCreationManager.scala | 5 +- .../kafka/server/BrokerLifecycleManager.scala | 22 +- .../scala/kafka/server/BrokerServer.scala | 54 +- .../scala/kafka/server/ConfigHandler.scala | 1 + .../scala/kafka/server/ControllerApis.scala | 45 +- .../ControllerConfigurationValidator.scala | 3 +- .../ControllerRegistrationManager.scala | 1 + .../scala/kafka/server/ControllerServer.scala | 17 +- .../kafka/server/DynamicBrokerConfig.scala | 23 +- .../scala/kafka/server/DynamicConfig.scala | 7 + .../kafka/server/ForwardingManager.scala | 4 +- .../main/scala/kafka/server/KafkaApis.scala | 208 ++- .../main/scala/kafka/server/KafkaBroker.scala | 1 + .../main/scala/kafka/server/KafkaConfig.scala | 35 +- .../main/scala/kafka/server/KafkaServer.scala | 23 +- .../scala/kafka/server/MetadataCache.scala | 5 +- .../NodeToControllerChannelManager.scala | 47 +- .../scala/kafka/server/ReplicaManager.scala | 70 +- .../kafka/server/RequestHandlerHelper.scala | 11 +- .../scala/kafka/server/SharedServer.scala | 5 +- .../scala/kafka/server/ZkConfigManager.scala | 3 +- .../checkpoints/OffsetCheckpointFile.scala | 2 +- .../metadata/DynamicConfigPublisher.scala | 13 +- .../server/metadata/KRaftMetadataCache.scala | 13 +- .../server/metadata/ZkConfigRepository.scala | 2 + .../server/metadata/ZkMetadataCache.scala | 89 +- .../main/scala/kafka/tools/StorageTool.scala | 14 +- .../log/remote/RemoteLogManagerTest.java | 189 ++- .../kafka/metrics/ClientMetricsTestUtils.java | 41 - .../kafka/test/ClusterTestExtensionsTest.java | 2 +- .../kafka/test/annotation/ClusterTest.java | 2 +- .../test/java/kafka/test/annotation/Type.java | 24 +- .../test/junit/ClusterTestExtensions.java | 7 +- .../junit/RaftClusterInvocationContext.java | 6 +- .../junit/ZkClusterInvocationContext.java | 6 +- .../kafka/testkit/KafkaClusterTestKit.java | 1 + .../admin/BrokerApiVersionsCommandTest.scala | 4 +- .../kafka/api/AbstractConsumerTest.scala | 2 +- .../kafka/api/BaseAsyncConsumerTest.scala | 66 - .../kafka/api/BaseConsumerTest.scala | 55 +- .../kafka/api/IntegrationTestHarness.scala | 18 +- .../kafka/api/PlaintextConsumerTest.scala | 625 ++++++-- .../api/SaslMultiMechanismConsumerTest.scala | 3 +- .../api/SaslPlainPlaintextConsumerTest.scala | 4 +- .../kafka/api/SaslPlaintextConsumerTest.scala | 3 +- .../kafka/api/SaslSslConsumerTest.scala | 3 +- .../kafka/api/SslConsumerTest.scala | 3 +- .../FetchFromFollowerIntegrationTest.scala | 3 +- .../kafka/server/KRaftClusterTest.scala | 5 + .../kafka/server/QuorumTestHarness.scala | 37 +- .../kafka/zk/ZkMigrationIntegrationTest.scala | 70 +- .../NodeToControllerRequestThreadTest.scala | 1 + .../scala/kafka/utils/TestInfoUtils.scala | 18 +- .../unit/kafka/admin/ConfigCommandTest.scala | 117 ++ .../unit/kafka/cluster/PartitionTest.scala | 3 +- .../AbstractCoordinatorConcurrencyTest.scala | 4 +- .../group/CoordinatorLoaderImplTest.scala | 54 +- .../CoordinatorPartitionWriterTest.scala | 121 +- .../group/GroupCoordinatorAdapterTest.scala | 13 + .../transaction/ProducerIdManagerTest.scala | 2 +- .../scala/unit/kafka/log/LocalLogTest.scala | 68 - .../scala/unit/kafka/log/LogSegmentTest.scala | 2 +- .../scala/unit/kafka/log/LogTestUtils.scala | 4 + .../unit/kafka/log/LogValidatorTest.scala | 64 +- .../scala/unit/kafka/log/UnifiedLogTest.scala | 141 +- .../AbstractApiVersionsRequestTest.scala | 4 +- .../server/AlterPartitionManagerTest.scala | 1 + .../server/AutoTopicCreationManagerTest.scala | 4 +- .../server/BrokerLifecycleManagerTest.scala | 29 +- .../BrokerRegistrationRequestTest.scala | 3 +- .../ConsumerGroupHeartbeatRequestTest.scala | 141 +- ...ControllerConfigurationValidatorTest.scala | 4 +- .../server/DeleteGroupsRequestTest.scala | 11 +- .../server/DescribeClusterRequestTest.scala | 2 + .../server/DescribeGroupsRequestTest.scala | 4 +- .../server/DynamicBrokerConfigTest.scala | 40 +- .../GroupCoordinatorBaseRequestTest.scala | 184 ++- .../kafka/server/HeartbeatRequestTest.scala | 200 +++ .../kafka/server/JoinGroupRequestTest.scala | 401 +++++ .../unit/kafka/server/KafkaApisTest.scala | 417 +++++- .../kafka/server/LeaveGroupRequestTest.scala | 151 ++ .../kafka/server/ListGroupsRequestTest.scala | 23 +- .../unit/kafka/server/MetadataCacheTest.scala | 257 +++- .../unit/kafka/server/MockFetcherThread.scala | 2 +- .../MockNodeToControllerChannelManager.scala | 9 +- .../server/OffsetCommitRequestTest.scala | 3 - .../server/OffsetDeleteRequestTest.scala | 6 +- .../kafka/server/OffsetFetchRequestTest.scala | 9 - .../server/ReplicaFetcherThreadTest.scala | 6 +- .../ReplicaManagerConcurrencyTest.scala | 13 +- .../kafka/server/ReplicaManagerTest.scala | 8 +- .../unit/kafka/server/RequestQuotaTest.scala | 3 + .../unit/kafka/server/ServerStartupTest.scala | 21 + .../kafka/server/StopReplicaRequestTest.scala | 2 +- .../kafka/server/SyncGroupRequestTest.scala | 272 ++++ ...CheckpointFileWithFailureHandlerTest.scala | 2 +- .../epoch/LeaderEpochFileCacheTest.scala | 21 +- .../BrokerMetadataPublisherTest.scala | 3 +- .../metadata/ZkConfigRepositoryTest.scala | 5 +- .../unit/kafka/tools/StorageToolTest.scala | 47 +- .../scala/unit/kafka/utils/TestUtils.scala | 39 +- .../zk/migration/ZkMigrationClientTest.scala | 112 +- .../kafka/zookeeper/ZooKeeperClientTest.scala | 4 +- docker/README.md | 78 +- docker/common.py | 20 +- docker/docker_build_test.py | 78 +- docker/docker_promote.py | 2 +- docker/docker_release.py | 83 +- docker/jvm/Dockerfile | 28 +- docker/jvm/jsa_launch | 7 +- docker/jvm/launch | 8 +- docker/native-image/Dockerfile2 | 63 + docker/report_jvm.html | 276 ++++ docker/resources/common-scripts/configure | 93 +- docker/resources/utility/go.mod | 29 + docker/resources/utility/go.sum | 14 + .../utility/testResources/sampleFile | 14 + .../utility/testResources/sampleFile2 | 14 + .../testResources/sampleLog4j.template | 20 + docker/resources/utility/utility.go | 323 ++++ docker/resources/utility/utility_test.go | 355 +++++ docker/test/__init__.py | 2 +- docker/test/constants.py | 19 +- docker/test/docker_sanity_test.py | 104 +- .../fixtures/file-input/server.properties | 14 +- .../fixtures/jvm/combined/docker-compose.yml | 101 ++ .../fixtures/jvm/isolated/docker-compose.yml | 170 +++ .../fixtures/secrets/client-ssl.properties | 7 +- .../fixtures/secrets/kafka_keystore_creds | 2 +- .../test/fixtures/secrets/kafka_ssl_key_creds | 2 +- .../fixtures/secrets/kafka_truststore_creds | 2 +- docker/test/report.html | 278 ++++ docs/configuration.html | 2 +- docs/connect.html | 10 +- docs/design.html | 36 +- docs/implementation.html | 2 +- docs/ops.html | 104 +- docs/quickstart.html | 2 +- docs/security.html | 204 +-- docs/streams/developer-guide/dsl-api.html | 4 - docs/streams/upgrade-guide.html | 31 +- docs/toc.html | 12 +- docs/upgrade.html | 6 +- docs/uses.html | 6 +- gradle/dependencies.gradle | 4 +- gradle/spotbugs-exclude.xml | 16 + gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 5 +- .../coordinator/group/GroupCoordinator.java | 18 +- .../group/GroupCoordinatorService.java | 90 +- .../group/GroupCoordinatorShard.java | 99 +- .../group/GroupMetadataManager.java | 351 ++++- .../group/OffsetMetadataManager.java | 44 +- .../AbstractUniformAssignmentBuilder.java | 21 +- .../GeneralUniformAssignmentBuilder.java | 918 +++++++++++- .../group/assignor/UniformAssignor.java | 5 +- .../group/consumer/ConsumerGroup.java | 120 +- .../group/consumer/ConsumerGroupMember.java | 35 + .../consumer/TargetAssignmentBuilder.java | 28 +- .../group/generic/GenericGroup.java | 14 +- .../group/metrics/CoordinatorMetrics.java | 40 + .../metrics/CoordinatorMetricsShard.java | 97 ++ .../metrics/GroupCoordinatorMetrics.java | 380 +++++ .../metrics/GroupCoordinatorMetricsShard.java | 325 ++++ .../group/runtime/CoordinatorPlayback.java | 11 +- .../group/runtime/CoordinatorRuntime.java | 139 +- .../runtime/CoordinatorShardBuilder.java | 24 + .../group/runtime/CoordinatorTimer.java | 14 + .../group/runtime/PartitionWriter.java | 8 +- .../group/GroupCoordinatorServiceTest.java | 290 +++- .../group/GroupCoordinatorShardTest.java | 276 +++- .../group/GroupMetadataManagerTest.java | 1225 ++++++++++++++- .../group/MockCoordinatorTimer.java | 12 + .../group/OffsetMetadataManagerTest.java | 135 ++ .../coordinator/group/RecordHelpersTest.java | 14 +- .../GeneralUniformAssignmentBuilderTest.java | 1033 +++++++++++++ .../consumer/ConsumerGroupMemberTest.java | 84 ++ .../group/consumer/ConsumerGroupTest.java | 110 +- .../consumer/TargetAssignmentBuilderTest.java | 120 +- .../group/generic/GenericGroupTest.java | 17 +- .../GroupCoordinatorMetricsShardTest.java | 249 ++++ .../metrics/GroupCoordinatorMetricsTest.java | 169 +++ .../group/metrics/MetricsTestUtils.java | 57 + .../group/runtime/CoordinatorRuntimeTest.java | 136 +- .../runtime/InMemoryPartitionWriter.java | 2 + .../MultiThreadedEventProcessorTest.java | 2 - .../ReplicaFetcherThreadBenchmark.java | 2 +- .../metadata/MetadataRequestBenchmark.java | 2 +- .../kafka/jmh/server/CheckpointBench.java | 2 +- .../jmh/server/PartitionCreationBench.java | 2 +- licenses/checker-qual-MIT | 17 + licenses/jsr305-BSD-3-clause | 28 + licenses/paranamer-BSD-3-clause | 2 +- licenses/protobuf-java-BSD-3-clause | 34 + .../controller/ClusterControlManager.java | 4 +- .../controller/PartitionChangeBuilder.java | 33 +- .../kafka/controller/QuorumFeatures.java | 6 +- .../controller/ReplicationControlManager.java | 134 +- .../org/apache/kafka/image/ClusterImage.java | 15 +- .../image/publisher/SnapshotEmitter.java | 2 +- .../image/writer/ImageWriterOptions.java | 16 +- .../kafka/metadata/BrokerRegistration.java | 41 +- .../metadata/ControllerRegistration.java | 2 +- .../kafka/metadata/PartitionRegistration.java | 46 +- .../metadata/placement/ClusterDescriber.java | 8 +- .../placement/DefaultDirProvider.java | 32 + .../placement/PartitionAssignment.java | 40 +- .../placement/StripedReplicaPlacer.java | 2 +- .../metadata/PartitionChangeRecord.json | 16 +- .../common/metadata/PartitionRecord.json | 14 +- .../controller/ClusterControlManagerTest.java | 38 +- .../controller/FeatureControlManagerTest.java | 2 +- .../PartitionChangeBuilderTest.java | 255 +++- .../PartitionReassignmentReplicasTest.java | 23 + .../PartitionReassignmentRevertTest.java | 32 + .../ProducerIdControlManagerTest.java | 2 +- .../QuorumControllerIntegrationTestUtils.java | 2 +- .../controller/QuorumControllerTest.java | 137 +- .../controller/QuorumControllerTestEnv.java | 2 +- .../kafka/controller/QuorumFeaturesTest.java | 18 + .../ReplicationControlManagerTest.java | 69 +- ...ontrollerMetadataMetricsPublisherTest.java | 3 +- .../metrics/ControllerMetricsChangesTest.java | 3 +- .../metrics/ControllerMetricsTestUtils.java | 2 + .../apache/kafka/image/ClusterImageTest.java | 30 +- .../kafka/image/ImageDowngradeTest.java | 6 +- .../apache/kafka/image/MetadataImageTest.java | 3 +- .../apache/kafka/image/TopicsImageTest.java | 18 +- .../image/loader/MetadataLoaderTest.java | 2 +- .../node/ClusterImageBrokersNodeTest.java | 5 +- .../metadata/BrokerRegistrationTest.java | 47 +- .../metadata/PartitionRegistrationTest.java | 123 +- .../migration/KRaftMigrationDriverTest.java | 2 +- .../placement/PartitionAssignmentTest.java | 11 +- .../placement/StripedReplicaPlacerTest.java | 15 +- .../placement/TopicAssignmentTest.java | 11 +- .../metadata/util/MetadataFeatureUtil.java | 32 + .../apache/kafka/raft/KafkaRaftClient.java | 14 +- .../org/apache/kafka/raft/LeaderState.java | 59 + .../org/apache/kafka/raft/QuorumState.java | 2 + .../org/apache/kafka/raft/RaftConfig.java | 4 +- .../raft/KafkaRaftClientSnapshotTest.java | 100 +- .../kafka/raft/KafkaRaftClientTest.java | 54 + .../apache/kafka/raft/LeaderStateTest.java | 44 +- .../kafka/raft/RaftClientTestContext.java | 2 + .../org/apache/kafka/common/DirectoryId.java | 54 +- .../org/apache/kafka/queue/EventQueue.java | 19 + .../kafka/server/common/CheckpointFile.java | 8 +- .../server/common/DirectoryEventHandler.java | 44 + .../kafka/server/common/MetadataVersion.java | 61 +- .../apache/kafka/common/DirectoryIdTest.java | 63 +- .../server/common/MetadataVersionTest.java | 69 +- .../kafka/server/AssignmentsManager.java | 394 +++++ .../kafka/server/ClientMetricsManager.java | 432 ++++++ .../ControllerRequestCompletionHandler.java | 30 + .../NodeToControllerChannelManager.java | 37 + .../server}/metrics/ClientMetricsConfigs.java | 76 +- .../server/metrics/ClientMetricsInstance.java | 123 ++ .../ClientMetricsInstanceMetadata.java | 72 + .../metrics/ClientMetricsReceiverPlugin.java | 57 + .../DefaultClientTelemetryPayload.java | 61 + .../org/apache/kafka/server/package-info.java | 20 + .../kafka/server/AssignmentsManagerTest.java | 251 ++++ .../server/ClientMetricsManagerTest.java | 922 ++++++++++++ .../ClientMetricsInstanceMetadataTest.java | 134 ++ .../metrics/ClientMetricsInstanceTest.java | 89 ++ .../ClientMetricsReceiverPluginTest.java | 61 + .../metrics/ClientMetricsTestUtils.java | 90 ++ settings.gradle | 1 + .../storage/CommittedOffsetsFile.java | 4 +- .../TopicBasedRemoteLogMetadataManager.java | 58 +- .../CheckpointFileWithFailureHandler.java | 4 +- .../InMemoryLeaderEpochCheckpoint.java | 4 +- .../checkpoint/LeaderEpochCheckpoint.java | 7 +- .../checkpoint/LeaderEpochCheckpointFile.java | 8 +- .../internals/epoch/LeaderEpochFileCache.java | 46 +- .../storage/internals/log/LogAppendInfo.java | 34 +- .../storage/internals/log/LogValidator.java | 77 +- .../internals/log/ProducerStateManager.java | 22 +- .../storage/internals/log/SnapshotFile.java | 8 +- ...opicBasedRemoteLogMetadataManagerTest.java | 27 + .../tiered/storage/actions/ProduceAction.java | 9 +- .../RollAndOffloadActiveSegmentTest.java | 75 + .../kafka/streams/ClientInstanceIds.java | 50 + .../apache/kafka/streams/KafkaStreams.java | 127 ++ .../apache/kafka/streams/StreamsConfig.java | 35 +- .../apache/kafka/streams/TopologyConfig.java | 41 +- .../internals/ClientInstanceIdsImpl.java | 62 + .../apache/kafka/streams/kstream/KTable.java | 21 +- .../kafka/streams/kstream/Materialized.java | 69 +- .../kafka/streams/kstream/StreamJoined.java | 66 + .../AbstractConfigurableStoreFactory.java | 54 + .../internals/InternalStreamsBuilder.java | 5 + .../kstream/internals/KStreamImpl.java | 2 +- .../kstream/internals/KStreamImplJoin.java | 138 +- .../internals/KeyValueStoreMaterializer.java | 17 +- .../internals/MaterializedInternal.java | 15 +- .../internals/MaterializedStoreFactory.java | 27 +- .../OuterStreamJoinStoreFactory.java | 212 +++ .../internals/SessionStoreMaterializer.java | 33 +- .../SlidingWindowStoreMaterializer.java | 44 +- .../internals/StreamJoinedInternal.java | 27 +- .../internals/StreamJoinedStoreFactory.java | 163 ++ .../internals/WindowStoreMaterializer.java | 43 +- .../internals/foreignkeyjoin/CombinedKey.java | 1 - .../SubscriptionJoinProcessorSupplier.java | 5 +- .../SubscriptionReceiveProcessorSupplier.java | 56 +- .../SubscriptionSendProcessorSupplier.java | 122 +- .../internals/graph/StreamStreamJoinNode.java | 48 +- .../processor/StandbyUpdateListener.java | 85 ++ .../processor/internals/AbstractTask.java | 2 +- .../internals/DefaultStateUpdater.java | 4 +- .../internals/PendingUpdateAction.java | 6 +- .../processor/internals/ProcessingThread.java | 28 + .../internals/ProcessorStateManager.java | 27 +- .../internals/RepartitionTopics.java | 2 +- .../internals/StoreChangelogReader.java | 85 +- .../processor/internals/StreamThread.java | 7 +- .../processor/internals/TaskManager.java | 37 +- .../streams/processor/internals/Tasks.java | 8 +- .../processor/internals/TasksRegistry.java | 4 +- .../processor/internals/TopologyMetadata.java | 18 +- .../BalanceSubtopologyGraphConstructor.java | 254 ++++ .../processor/internals/assignment/Graph.java | 185 ++- .../MinTrafficGraphConstructor.java | 145 ++ .../assignment/RackAwareGraphConstructor.java | 142 ++ .../RackAwareGraphConstructorFactory.java | 39 + .../assignment/RackAwareTaskAssignor.java | 211 +-- .../internals/tasks/DefaultTaskExecutor.java | 3 +- .../apache/kafka/streams/query/KeyQuery.java | 9 +- .../kafka/streams/query/RangeQuery.java | 3 +- .../kafka/streams/query/StateQueryResult.java | 1 + .../streams/query/TimestampedKeyQuery.java | 76 + .../streams/query/TimestampedRangeQuery.java | 130 ++ .../streams/query/VersionedKeyQuery.java | 87 ++ .../state/BuiltInDslStoreSuppliers.java | 98 ++ .../streams/state/DslKeyValueParams.java | 64 + .../kafka/streams/state/DslSessionParams.java | 90 ++ .../streams/state/DslStoreSuppliers.java | 62 + .../kafka/streams/state/DslWindowParams.java | 129 ++ ...ToTimestampedKeyValueByteStoreAdapter.java | 67 +- ...eToTimestampedKeyValueIteratorAdapter.java | 3 +- .../state/internals/ListValueStore.java | 2 +- .../state/internals/MeteredKeyValueStore.java | 28 +- .../MeteredTimestampedKeyValueStore.java | 251 +++- .../MeteredVersionedKeyValueStore.java | 92 +- .../state/internals/RocksDBRangeIterator.java | 2 +- .../streams/state/internals/RocksDBStore.java | 71 +- .../internals/RocksDBTimestampedStore.java | 40 +- .../internals/RocksDBVersionedStore.java | 27 +- .../state/internals/RocksDbIterator.java | 2 +- .../state/internals/StoreQueryUtils.java | 47 + .../TimestampedKeyValueStoreBuilder.java | 70 +- .../TimestampedWindowStoreBuilder.java | 67 +- .../kafka/streams/KafkaStreamsTest.java | 129 +- .../kafka/streams/StreamsBuilderTest.java | 420 +++++- .../kafka/streams/StreamsConfigTest.java | 132 ++ .../apache/kafka/streams/TopologyTest.java | 48 + .../ConsistencyVectorIntegrationTest.java | 15 +- .../integration/IQv2IntegrationTest.java | 2 + .../integration/IQv2StoreIntegrationTest.java | 240 ++- .../IQv2VersionedStoreIntegrationTest.java | 166 +++ ...leKTableForeignKeyJoinIntegrationTest.java | 103 +- .../MetricsReporterIntegrationTest.java | 12 +- .../NamedTopologyIntegrationTest.java | 7 + .../PurgeRepartitionTopicIntegrationTest.java | 3 + .../RegexSourceIntegrationTest.java | 2 +- .../integration/RestoreIntegrationTest.java | 8 +- .../TaskAssignorIntegrationTest.java | 11 +- .../utils/IntegrationTestUtils.java | 24 + .../internals/KStreamKStreamJoinTest.java | 65 + .../internals/MaterializedInternalTest.java | 64 +- .../InternalTopologyBuilderTest.java | 1 + .../KeyValueStoreMaterializerTest.java | 14 +- .../internals/ProcessorStateManagerTest.java | 110 +- .../internals/StoreChangelogReaderTest.java | 92 +- .../processor/internals/StreamTaskTest.java | 9 +- .../processor/internals/StreamThreadTest.java | 490 ++---- .../processor/internals/TaskManagerTest.java | 276 ++-- .../processor/internals/TasksTest.java | 36 +- .../internals/TopologyMetadataTest.java | 29 +- .../assignment/AssignmentTestUtils.java | 34 +- .../internals/assignment/GraphTest.java | 107 ++ .../RackAwareGraphConstructorFactoryTest.java | 46 + .../RackAwareGraphConstructorTest.java | 301 ++++ .../assignment/RackAwareTaskAssignorTest.java | 85 +- .../TaskAssignorConvergenceTest.java | 2 +- .../streams/query/VersionedKeyQueryTest.java | 38 + .../StreamThreadStateStoreProviderTest.java | 4 +- .../kafka/test/MockStandbyUpdateListener.java | 59 + .../scala/kstream/StreamJoinedTest.scala | 24 +- .../client/consumer_rolling_upgrade_test.py | 11 +- tests/kafkatest/tests/client/consumer_test.py | 133 +- .../tests/connect/connect_distributed_test.py | 161 +- .../tests/core/consumer_group_command_test.py | 28 +- .../tests/core/fetch_from_follower_test.py | 11 +- .../tests/core/kraft_upgrade_test.py | 24 +- .../tests/core/reassign_partitions_test.py | 17 +- .../tests/core/replica_scale_test.py | 36 +- .../core/replication_replica_failure_test.py | 11 +- tests/kafkatest/tests/core/security_test.py | 39 +- tests/kafkatest/tests/core/snapshot_test.py | 14 +- .../kafkatest/tests/core/transactions_test.py | 22 +- .../streams_broker_down_resilience_test.py | 16 +- .../streams/streams_standby_replica_test.py | 4 +- .../streams/streams_static_membership_test.py | 4 +- .../org/apache/kafka/tools/TopicCommand.java | 10 +- .../reassign/ReassignPartitionsCommand.java | 6 +- .../kafka/tools/FeatureCommandTest.java | 6 +- .../apache/kafka/tools/ToolsTestUtils.java | 2 +- .../tools/other/ReplicationQuotasTestRig.java | 480 ++++++ 609 files changed, 39271 insertions(+), 7305 deletions(-) create mode 100644 checkstyle/import-control-server.xml create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ClientMetricsResourceListing.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesOptions.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesResult.java rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/{PrototypeAsyncConsumer.java => AsyncKafkaConsumer.java} (50%) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegateCreator.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/GroupMetadataUpdateEvent.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/SubscriptionChangeApplicationEvent.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/UnsubscribeApplicationEvent.java create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/TelemetryTooLargeException.java create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/UnknownSubscriptionIdException.java rename clients/src/main/java/org/apache/kafka/common/record/{RecordConversionStats.java => RecordValidationStats.java} (78%) create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesResponse.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitter.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryProvider.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporter.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtils.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollector.java create mode 100644 clients/src/main/java/org/apache/kafka/common/telemetry/internals/LastValueTracker.java create mode 100644 clients/src/main/resources/common/message/ListClientMetricsResourcesRequest.json create mode 100644 clients/src/main/resources/common/message/ListClientMetricsResourcesResponse.json create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java delete mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java delete mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/requests/BrokerRegistrationRequestTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitterTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporterTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtilsTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollectorTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/LastValueTrackerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/SinglePointMetricTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/telemetry/internals/TestEmitter.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/cli/ConnectStandaloneTest.java create mode 100644 connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequestTest.java delete mode 100644 core/src/main/java/kafka/server/ClientMetricsManager.java delete mode 100644 core/src/test/java/kafka/metrics/ClientMetricsTestUtils.java delete mode 100644 core/src/test/scala/integration/kafka/api/BaseAsyncConsumerTest.scala create mode 100644 core/src/test/scala/unit/kafka/server/HeartbeatRequestTest.scala create mode 100644 core/src/test/scala/unit/kafka/server/JoinGroupRequestTest.scala create mode 100644 core/src/test/scala/unit/kafka/server/LeaveGroupRequestTest.scala create mode 100644 core/src/test/scala/unit/kafka/server/SyncGroupRequestTest.scala create mode 100644 docker/native-image/Dockerfile2 create mode 100644 docker/report_jvm.html create mode 100644 docker/resources/utility/go.mod create mode 100644 docker/resources/utility/go.sum create mode 100755 docker/resources/utility/testResources/sampleFile create mode 100755 docker/resources/utility/testResources/sampleFile2 create mode 100644 docker/resources/utility/testResources/sampleLog4j.template create mode 100644 docker/resources/utility/utility.go create mode 100644 docker/resources/utility/utility_test.go create mode 100644 docker/test/fixtures/jvm/combined/docker-compose.yml create mode 100644 docker/test/fixtures/jvm/isolated/docker-compose.yml create mode 100644 docker/test/report.html create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/metrics/CoordinatorMetrics.java create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/metrics/CoordinatorMetricsShard.java create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/metrics/GroupCoordinatorMetrics.java create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/metrics/GroupCoordinatorMetricsShard.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/assignor/GeneralUniformAssignmentBuilderTest.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/metrics/GroupCoordinatorMetricsShardTest.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/metrics/GroupCoordinatorMetricsTest.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/metrics/MetricsTestUtils.java create mode 100644 licenses/checker-qual-MIT create mode 100644 licenses/jsr305-BSD-3-clause create mode 100644 licenses/protobuf-java-BSD-3-clause create mode 100644 metadata/src/main/java/org/apache/kafka/metadata/placement/DefaultDirProvider.java create mode 100644 metadata/src/test/java/org/apache/kafka/metadata/util/MetadataFeatureUtil.java create mode 100644 server-common/src/main/java/org/apache/kafka/server/common/DirectoryEventHandler.java create mode 100644 server/src/main/java/org/apache/kafka/server/AssignmentsManager.java create mode 100644 server/src/main/java/org/apache/kafka/server/ClientMetricsManager.java create mode 100644 server/src/main/java/org/apache/kafka/server/ControllerRequestCompletionHandler.java create mode 100644 server/src/main/java/org/apache/kafka/server/NodeToControllerChannelManager.java rename {core/src/main/java/kafka => server/src/main/java/org/apache/kafka/server}/metrics/ClientMetricsConfigs.java (76%) create mode 100644 server/src/main/java/org/apache/kafka/server/metrics/ClientMetricsInstance.java create mode 100644 server/src/main/java/org/apache/kafka/server/metrics/ClientMetricsInstanceMetadata.java create mode 100644 server/src/main/java/org/apache/kafka/server/metrics/ClientMetricsReceiverPlugin.java create mode 100644 server/src/main/java/org/apache/kafka/server/metrics/DefaultClientTelemetryPayload.java create mode 100644 server/src/main/java/org/apache/kafka/server/package-info.java create mode 100644 server/src/test/java/org/apache/kafka/server/AssignmentsManagerTest.java create mode 100644 server/src/test/java/org/apache/kafka/server/ClientMetricsManagerTest.java create mode 100644 server/src/test/java/org/apache/kafka/server/metrics/ClientMetricsInstanceMetadataTest.java create mode 100644 server/src/test/java/org/apache/kafka/server/metrics/ClientMetricsInstanceTest.java create mode 100644 server/src/test/java/org/apache/kafka/server/metrics/ClientMetricsReceiverPluginTest.java create mode 100644 server/src/test/java/org/apache/kafka/server/metrics/ClientMetricsTestUtils.java create mode 100644 storage/src/test/java/org/apache/kafka/tiered/storage/integration/RollAndOffloadActiveSegmentTest.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/ClientInstanceIds.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/internals/ClientInstanceIdsImpl.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/AbstractConfigurableStoreFactory.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/OuterStreamJoinStoreFactory.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/StreamJoinedStoreFactory.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/StandbyUpdateListener.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/ProcessingThread.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/BalanceSubtopologyGraphConstructor.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/MinTrafficGraphConstructor.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/RackAwareGraphConstructor.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/RackAwareGraphConstructorFactory.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/query/TimestampedKeyQuery.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/query/TimestampedRangeQuery.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/query/VersionedKeyQuery.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/BuiltInDslStoreSuppliers.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/DslKeyValueParams.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/DslSessionParams.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/DslStoreSuppliers.java create mode 100644 streams/src/main/java/org/apache/kafka/streams/state/DslWindowParams.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/integration/IQv2VersionedStoreIntegrationTest.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/RackAwareGraphConstructorFactoryTest.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/RackAwareGraphConstructorTest.java create mode 100644 streams/src/test/java/org/apache/kafka/streams/query/VersionedKeyQueryTest.java create mode 100644 streams/src/test/java/org/apache/kafka/test/MockStandbyUpdateListener.java create mode 100644 tools/src/test/java/org/apache/kafka/tools/other/ReplicationQuotasTestRig.java diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 47fe4d0929a9e..5721333b14623 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Docker build test +name: Docker Build Test on: workflow_dispatch: @@ -21,7 +21,7 @@ on: image_type: type: choice description: Docker image type to build and test - options: + options: - "jvm" kafka_url: description: Kafka url to be used to build the docker image @@ -31,39 +31,36 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r docker/requirements.txt - - name: Build image and run tests - working-directory: ./docker - run: | - python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} - - name: Run CVE scan - if: always() - uses: aquasecurity/trivy-action@master - with: - image-ref: 'kafka/test:test' - format: 'table' - ignore-unfixed: true - vuln-type: 'os,library' - severity: 'CRITICAL,HIGH' - output: scan_report_${{ github.event.inputs.image_type }}.txt - exit-code: '1' - - name: Upload test report - if: always() - uses: actions/upload-artifact@v3 - with: - name: report_${{ github.event.inputs.image_type }}.html - path: docker/report_${{ github.event.inputs.image_type }}.html - - name: Upload CVE scan report - if: always() - uses: actions/upload-artifact@v3 - with: - name: scan_report_${{ github.event.inputs.image_type }}.txt - path: scan_report_${{ github.event.inputs.image_type }}.txt + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/requirements.txt + - name: Build image and run tests + working-directory: ./docker + run: | + python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} + - name: Run CVE scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'kafka/test:test' + format: 'table' + severity: 'CRITICAL,HIGH' + output: scan_report_${{ github.event.inputs.image_type }}.txt + exit-code: '1' + - name: Upload test report + if: always() + uses: actions/upload-artifact@v3 + with: + name: report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html + - name: Upload CVE scan report + if: always() + uses: actions/upload-artifact@v3 + with: + name: scan_report_${{ github.event.inputs.image_type }}.txt + path: scan_report_${{ github.event.inputs.image_type }}.txt diff --git a/.gitignore b/.gitignore index 7bf18c57cc6b0..4ac36a815ba54 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,5 @@ jmh-benchmarks/src/main/generated storage/kafka-tiered-storage/ -docker/report_*.html +docker/test/report_*.html +__pycache__ diff --git a/LICENSE-binary b/LICENSE-binary index 0a9f626d7c600..f669a86fed094 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -205,15 +205,17 @@ This project bundles some components that are also licensed under the Apache License Version 2.0: -audience-annotations-0.13.0 +audience-annotations-0.12.0 caffeine-2.9.3 commons-beanutils-1.9.4 commons-cli-1.4 commons-collections-3.2.2 commons-digester-2.1 +commons-io-2.11.0 commons-lang3-3.8.1 commons-logging-1.2 commons-validator-1.7 +error_prone_annotations-2.10.0 jackson-annotations-2.13.5 jackson-core-2.13.5 jackson-databind-2.13.5 @@ -226,16 +228,16 @@ jackson-module-scala_2.13-2.13.5 jackson-module-scala_2.12-2.13.5 jakarta.validation-api-2.0.2 javassist-3.29.2-GA -jetty-client-9.4.52.v20230823 -jetty-continuation-9.4.52.v20230823 -jetty-http-9.4.52.v20230823 -jetty-io-9.4.52.v20230823 -jetty-security-9.4.52.v20230823 -jetty-server-9.4.52.v20230823 -jetty-servlet-9.4.52.v20230823 -jetty-servlets-9.4.52.v20230823 -jetty-util-9.4.52.v20230823 -jetty-util-ajax-9.4.52.v20230823 +jetty-client-9.4.53.v20231009 +jetty-continuation-9.4.53.v20231009 +jetty-http-9.4.53.v20231009 +jetty-io-9.4.53.v20231009 +jetty-security-9.4.53.v20231009 +jetty-server-9.4.53.v20231009 +jetty-servlet-9.4.53.v20231009 +jetty-servlets-9.4.53.v20231009 +jetty-util-9.4.53.v20231009 +jetty-util-ajax-9.4.53.v20231009 jose4j-0.9.3 lz4-java-1.8.0 maven-artifact-3.8.8 @@ -250,7 +252,8 @@ netty-transport-4.1.100.Final netty-transport-classes-epoll-4.1.100.Final netty-transport-native-epoll-4.1.100.Final netty-transport-native-unix-common-4.1.100.Final -plexus-utils-3.3.0 +opentelemetry-proto-1.0.0-alpha +plexus-utils-3.3.1 reflections-0.10.2 reload4j-1.2.25 rocksdbjni-7.9.2 @@ -310,6 +313,7 @@ activation-1.1.1 MIT License argparse4j-0.7.0, see: licenses/argparse-MIT +checker-qual-3.19.0, see: licenses/checker-qual-MIT jopt-simple-5.0.4, see: licenses/jopt-simple-MIT slf4j-api-1.7.36, see: licenses/slf4j-MIT slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT @@ -324,7 +328,9 @@ zstd-jni-1.5.5-6 see: licenses/zstd-jni-BSD-2-clause BSD 3-Clause jline-3.22.0, see: licenses/jline-BSD-3-clause +jsr305-3.0.2, see: licenses/jsr305-BSD-3-clause paranamer-2.8, see: licenses/paranamer-BSD-3-clause +protobuf-java-3.23.4, see: licenses/protobuf-java-BSD-3-clause --------------------------------------- Do What The F*ck You Want To Public License diff --git a/NOTICE-binary b/NOTICE-binary index a50c86d84b7d7..82920af2fc951 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -98,6 +98,13 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +Apache Commons IO +Copyright 2002-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + + Apache Commons Lang Copyright 2001-2018 The Apache Software Foundation @@ -853,4 +860,4 @@ This private header is also used by Apple's open source * LICENSE: * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) * HOMEPAGE: - * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h \ No newline at end of file + * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h diff --git a/bin/connect-standalone.sh b/bin/connect-standalone.sh index 441069fed3139..bef78d658fda9 100755 --- a/bin/connect-standalone.sh +++ b/bin/connect-standalone.sh @@ -16,7 +16,7 @@ if [ $# -lt 1 ]; then - echo "USAGE: $0 [-daemon] connect-standalone.properties" + echo "USAGE: $0 [-daemon] connect-standalone.properties [connector1.properties connector2.json ...]" exit 1 fi diff --git a/build.gradle b/build.gradle index 1e1ba0b546e3a..080daf28ac9fd 100644 --- a/build.gradle +++ b/build.gradle @@ -208,7 +208,7 @@ if (repo != null) { 'licenses/*', '**/generated/**', 'clients/src/test/resources/serializedData/*', - 'docker/resources/ub/go.sum', + 'docker/resources/utility/go.sum', 'docker/test/fixtures/secrets/*' ]) } @@ -844,6 +844,62 @@ tasks.create(name: "jarConnect", dependsOn: connectPkgs.collect { it + ":jar" }) tasks.create(name: "testConnect", dependsOn: connectPkgs.collect { it + ":test" }) {} +project(':server') { + archivesBaseName = "kafka-server" + + dependencies { + implementation project(':clients') + implementation project(':server-common') + + implementation libs.slf4jApi + + compileOnly libs.log4j + + testImplementation project(':clients').sourceSets.test.output + + testImplementation libs.mockitoCore + testImplementation libs.junitJupiter + testImplementation libs.slf4jlog4j + } + + task createVersionFile() { + def receiptFile = file("$buildDir/kafka/$buildVersionFileName") + inputs.property "commitId", commitId + inputs.property "version", version + outputs.file receiptFile + + doLast { + def data = [ + commitId: commitId, + version: version, + ] + + receiptFile.parentFile.mkdirs() + def content = data.entrySet().collect { "$it.key=$it.value" }.sort().join("\n") + receiptFile.setText(content, "ISO-8859-1") + } + } + + jar { + dependsOn createVersionFile + from("$buildDir") { + include "kafka/$buildVersionFileName" + } + } + + clean.doFirst { + delete "$buildDir/kafka/" + } + + checkstyle { + configProperties = checkstyleConfigProperties("import-control-server.xml") + } + + javadoc { + enabled = false + } +} + project(':core') { apply plugin: 'scala' @@ -875,7 +931,7 @@ project(':core') { implementation project(':tools:tools-api') implementation project(':raft') implementation project(':storage') - + implementation project(':server') implementation libs.argparse4j implementation libs.commonsValidator @@ -914,6 +970,7 @@ project(':core') { testImplementation project(':raft').sourceSets.test.output testImplementation project(':server-common').sourceSets.test.output testImplementation project(':storage:storage-api').sourceSets.test.output + testImplementation project(':server').sourceSets.test.output testImplementation libs.bcpkix testImplementation libs.mockitoCore testImplementation(libs.apacheda) { @@ -932,9 +989,6 @@ project(':core') { testImplementation libs.apachedsJdbmPartition testImplementation libs.junitJupiter testImplementation libs.slf4jlog4j - testImplementation(libs.jfreechart) { - exclude group: 'junit', module: 'junit' - } testImplementation libs.caffeine generator project(':generator') @@ -1368,6 +1422,7 @@ project(':clients') { testImplementation libs.junitJupiter testImplementation libs.log4j testImplementation libs.mockitoCore + testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testRuntimeOnly libs.slf4jlog4j testRuntimeOnly libs.jacksonDatabind @@ -1645,19 +1700,6 @@ project(':server-common') { } } - sourceSets { - main { - java { - srcDirs = ["src/main/java"] - } - } - test { - java { - srcDirs = ["src/test/java"] - } - } - } - jar { dependsOn createVersionFile from("$buildDir") { @@ -1672,6 +1714,10 @@ project(':server-common') { checkstyle { configProperties = checkstyleConfigProperties("import-control-server-common.xml") } + + javadoc { + enabled = false + } } project(':storage:storage-api') { @@ -1949,6 +1995,9 @@ project(':tools') { testImplementation libs.mockitoCore testImplementation libs.mockitoJunitJupiter // supports MockitoExtension testImplementation libs.bcpkix // required by the clients test module, but we have to specify it explicitly as gradle does not include the transitive test dependency automatically + testImplementation(libs.jfreechart) { + exclude group: 'junit', module: 'junit' + } testRuntimeOnly libs.slf4jlog4j } diff --git a/checkstyle/import-control-core.xml b/checkstyle/import-control-core.xml index 849c45e5b192d..4430b8ec9ddb2 100644 --- a/checkstyle/import-control-core.xml +++ b/checkstyle/import-control-core.xml @@ -36,6 +36,7 @@ + diff --git a/checkstyle/import-control-metadata.xml b/checkstyle/import-control-metadata.xml index d4643c19979ef..897932492b14f 100644 --- a/checkstyle/import-control-metadata.xml +++ b/checkstyle/import-control-metadata.xml @@ -27,16 +27,11 @@ - - - - - diff --git a/checkstyle/import-control-server-common.xml b/checkstyle/import-control-server-common.xml index 79873703d849d..622e7c5e0525f 100644 --- a/checkstyle/import-control-server-common.xml +++ b/checkstyle/import-control-server-common.xml @@ -27,15 +27,11 @@ - - - - diff --git a/checkstyle/import-control-server.xml b/checkstyle/import-control-server.xml new file mode 100644 index 0000000000000..765f6faf4c15d --- /dev/null +++ b/checkstyle/import-control-server.xml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/checkstyle/import-control-storage.xml b/checkstyle/import-control-storage.xml index 42d170ec31536..d8c5e287d31b7 100644 --- a/checkstyle/import-control-storage.xml +++ b/checkstyle/import-control-storage.xml @@ -27,15 +27,11 @@ - - - - diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 0db1700c6122b..4379136bacc6a 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -198,6 +198,8 @@ + + @@ -249,6 +251,7 @@ + @@ -321,6 +324,14 @@ + + + + + + + + @@ -534,6 +545,7 @@ + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 0ba81cf87980a..3777d8714a2d9 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -48,7 +48,7 @@ + files="(AbstractFetch|Sender|SenderTest|ConsumerCoordinator|KafkaConsumer|KafkaProducer|Utils|TransactionManager|TransactionManagerTest|KafkaAdminClient|NetworkClient|Admin|KafkaRaftClient|KafkaRaftClientTest|RaftClientTestContext).java"/> - + files="(NetworkClient|FieldSpec|KafkaRaftClient|KafkaProducer).java"/> + files="(KafkaConsumer|ConsumerCoordinator).java"/> + + files="(KafkaConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|KafkaRaftClientTest).java"/> + files="(Errors|SaslAuthenticatorTest|AgentTest|CoordinatorTest|NetworkClientTest).java"/> + files="(AbstractFetch|ConsumerCoordinator|FetchCollector|OffsetFetcherUtils|KafkaProducer|Sender|ConfigDef|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor|DefaultSslEngineFactory|Authorizer|RecordAccumulator|MemoryRecords|FetchSessionHandler|MockAdminClient).java"/> @@ -110,7 +110,7 @@ + files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> @@ -236,13 +236,13 @@ files=".*[/\\]streams[/\\].*test[/\\].*.java"/> + files="(EosV2UpgradeIntegrationTest|KStreamKStreamJoinTest|KTableKTableForeignKeyJoinIntegrationTest|KTableKTableForeignKeyVersionedJoinIntegrationTest|RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest|RelationalSmokeTest|MockProcessorContextStateStoreTest|IQv2StoreIntegrationTest).java"/> + files="(EosV2UpgradeIntegrationTest|EosTestDriver|KStreamKStreamJoinTest|KTableKTableForeignKeyJoinIntegrationTest|KTableKTableForeignKeyVersionedJoinIntegrationTest|RelationalSmokeTest|MockProcessorContextStateStoreTest|TopologyTestDriverTest|IQv2StoreIntegrationTest).java"/> @@ -328,9 +328,9 @@ + files="(ConsumerGroupMember|GroupMetadataManager|GeneralUniformAssignmentBuilder).java"/> + files="(GroupMetadataManager|ConsumerGroupTest|GroupMetadataManagerTest|GeneralUniformAssignmentBuilder).java"/> 1 && addresses.get(addressIndex).equals(lastAttemptedAddress)) { + addressIndex++; + } } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java index 872c72100962d..a65e2467f1b92 100644 --- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java +++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java @@ -22,6 +22,8 @@ import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.MetricsReporter; import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Configurations shared by Kafka client applications: producer, consumer, connect, etc. @@ -294,4 +297,14 @@ public static List metricsReporters(Map clientI } return reporters; } + + public static Optional telemetryReporter(String clientId, AbstractConfig config) { + if (!config.getBoolean(CommonClientConfigs.ENABLE_METRICS_PUSH_CONFIG)) { + return Optional.empty(); + } + + ClientTelemetryReporter telemetryReporter = new ClientTelemetryReporter(Time.SYSTEM); + telemetryReporter.configure(config.originals(Collections.singletonMap(CommonClientConfigs.CLIENT_ID_CONFIG, clientId))); + return Optional.of(telemetryReporter); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/Metadata.java b/clients/src/main/java/org/apache/kafka/clients/Metadata.java index c0ebcb704f71e..a23303423ee5e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/Metadata.java +++ b/clients/src/main/java/org/apache/kafka/clients/Metadata.java @@ -729,6 +729,13 @@ protected MetadataRequest.Builder newMetadataRequestBuilderForNewTopics() { return null; } + /** + * @return Mapping from topic IDs to topic names for all topics in the cache. + */ + public synchronized Map topicNames() { + return cache.topicNames(); + } + protected boolean retainTopic(String topic, boolean isInternal, long nowMs) { return true; } diff --git a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java index 38a039a0a388f..45574c3549c56 100644 --- a/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java +++ b/clients/src/main/java/org/apache/kafka/clients/MetadataCache.java @@ -51,7 +51,7 @@ public class MetadataCache { private final Node controller; private final Map metadataByPartition; private final Map topicIds; - + private final Map topicNames; private Cluster clusterInstance; MetadataCache(String clusterId, @@ -80,7 +80,10 @@ private MetadataCache(String clusterId, this.invalidTopics = invalidTopics; this.internalTopics = internalTopics; this.controller = controller; - this.topicIds = topicIds; + this.topicIds = Collections.unmodifiableMap(topicIds); + this.topicNames = Collections.unmodifiableMap( + topicIds.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)) + ); this.metadataByPartition = new HashMap<>(partitions.size()); for (PartitionMetadata p : partitions) { @@ -102,6 +105,10 @@ Map topicIds() { return topicIds; } + Map topicNames() { + return topicNames; + } + Optional nodeById(int id) { return Optional.ofNullable(nodes.get(id)); } diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index ad81d1faed49b..a596f660d0261 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -38,10 +38,13 @@ import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.CorrelationIdMismatchException; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.PushTelemetryResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; @@ -128,6 +131,8 @@ private enum State { private final AtomicReference state; + private final TelemetrySender telemetrySender; + public NetworkClient(Selectable selector, Metadata metadata, String clientId, @@ -194,7 +199,8 @@ public NetworkClient(Selectable selector, apiVersions, throttleTimeSensor, logContext, - new DefaultHostResolver()); + new DefaultHostResolver(), + null); } public NetworkClient(Selectable selector, @@ -229,7 +235,8 @@ public NetworkClient(Selectable selector, apiVersions, null, logContext, - new DefaultHostResolver()); + new DefaultHostResolver(), + null); } public NetworkClient(MetadataUpdater metadataUpdater, @@ -249,7 +256,8 @@ public NetworkClient(MetadataUpdater metadataUpdater, ApiVersions apiVersions, Sensor throttleTimeSensor, LogContext logContext, - HostResolver hostResolver) { + HostResolver hostResolver, + ClientTelemetrySender clientTelemetrySender) { /* It would be better if we could pass `DefaultMetadataUpdater` from the public constructor, but it's not * possible because `DefaultMetadataUpdater` is an inner class and it can only be instantiated after the * super constructor is invoked. @@ -279,6 +287,7 @@ public NetworkClient(MetadataUpdater metadataUpdater, this.throttleTimeSensor = throttleTimeSensor; this.log = logContext.logger(NetworkClient.class); this.state = new AtomicReference<>(State.ACTIVE); + this.telemetrySender = (clientTelemetrySender != null) ? new TelemetrySender(clientTelemetrySender) : null; } /** @@ -361,6 +370,8 @@ private void cancelInFlightRequests(String nodeId, } } else if (request.header.apiKey() == ApiKeys.METADATA) { metadataUpdater.handleFailedRequest(now, Optional.empty()); + } else if (isTelemetryApi(request.header.apiKey()) && telemetrySender != null) { + telemetrySender.handleFailedRequest(request.header.apiKey(), null); } } } @@ -522,6 +533,8 @@ private void doSend(ClientRequest clientRequest, boolean isInternalRequest, long abortedSends.add(clientResponse); else if (clientRequest.apiKey() == ApiKeys.METADATA) metadataUpdater.handleFailedRequest(now, Optional.of(unsupportedVersionException)); + else if (isTelemetryApi(clientRequest.apiKey()) && telemetrySender != null) + telemetrySender.handleFailedRequest(clientRequest.apiKey(), unsupportedVersionException); } } @@ -567,8 +580,9 @@ public List poll(long timeout, long now) { } long metadataTimeout = metadataUpdater.maybeUpdate(now); + long telemetryTimeout = telemetrySender != null ? telemetrySender.maybeUpdate(now) : Integer.MAX_VALUE; try { - this.selector.poll(Utils.min(timeout, metadataTimeout, defaultRequestTimeoutMs)); + this.selector.poll(Utils.min(timeout, metadataTimeout, telemetryTimeout, defaultRequestTimeoutMs)); } catch (IOException e) { log.error("Unexpected error during I/O", e); } @@ -663,6 +677,8 @@ public void close() { if (state.compareAndSet(State.CLOSING, State.CLOSED)) { this.selector.close(); this.metadataUpdater.close(); + if (telemetrySender != null) + telemetrySender.close(); } else { log.warn("Attempting to close NetworkClient that has already been closed."); } @@ -925,6 +941,10 @@ private void handleCompletedReceives(List responses, long now) { metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) response); else if (req.isInternalRequest && response instanceof ApiVersionsResponse) handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) response); + else if (req.isInternalRequest && response instanceof GetTelemetrySubscriptionsResponse) + telemetrySender.handleResponse((GetTelemetrySubscriptionsResponse) response); + else if (req.isInternalRequest && response instanceof PushTelemetryResponse) + telemetrySender.handleResponse((PushTelemetryResponse) response); else responses.add(req.completed(response, now)); } @@ -1042,6 +1062,25 @@ private void initiateConnect(Node node, long now) { } } + /** + * Return true if there's at least one connection establishment is currently underway + */ + private boolean isAnyNodeConnecting() { + for (Node node : metadataUpdater.fetchNodes()) { + if (connectionStates.isConnecting(node.idString())) { + return true; + } + } + return false; + } + + /** + * Return true if the ApiKey belongs to the Telemetry API. + */ + private boolean isTelemetryApi(ApiKeys apiKey) { + return apiKey == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS || apiKey == ApiKeys.PUSH_TELEMETRY; + } + class DefaultMetadataUpdater implements MetadataUpdater { /* the current cluster metadata */ @@ -1161,18 +1200,6 @@ public void close() { this.metadata.close(); } - /** - * Return true if there's at least one connection establishment is currently underway - */ - private boolean isAnyNodeConnecting() { - for (Node node : fetchNodes()) { - if (connectionStates.isConnecting(node.idString())) { - return true; - } - } - return false; - } - /** * Add a metadata request to the list of sends if we can make one */ @@ -1222,6 +1249,95 @@ private InProgressData(int requestVersion, boolean isPartialUpdate) { } + class TelemetrySender { + + private final ClientTelemetrySender clientTelemetrySender; + private Node stickyNode; + + public TelemetrySender(ClientTelemetrySender clientTelemetrySender) { + this.clientTelemetrySender = clientTelemetrySender; + } + + public long maybeUpdate(long now) { + long timeToNextUpdate = clientTelemetrySender.timeToNextUpdate(defaultRequestTimeoutMs); + if (timeToNextUpdate > 0) + return timeToNextUpdate; + + // Per KIP-714, let's continue to re-use the same broker for as long as possible. + if (stickyNode == null) { + stickyNode = leastLoadedNode(now); + if (stickyNode == null) { + log.debug("Give up sending telemetry request since no node is available"); + return reconnectBackoffMs; + } + } + + return maybeUpdate(now, stickyNode); + } + + private long maybeUpdate(long now, Node node) { + String nodeConnectionId = node.idString(); + + if (canSendRequest(nodeConnectionId, now)) { + Optional> requestOpt = clientTelemetrySender.createRequest(); + + if (!requestOpt.isPresent()) + return Long.MAX_VALUE; + + AbstractRequest.Builder request = requestOpt.get(); + ClientRequest clientRequest = newClientRequest(nodeConnectionId, request, now, true); + doSend(clientRequest, true, now); + return defaultRequestTimeoutMs; + } else { + // Per KIP-714, if we can't issue a request to this broker node, let's clear it out + // and try another broker on the next loop. + stickyNode = null; + } + + // If there's any connection establishment underway, wait until it completes. This prevents + // the client from unnecessarily connecting to additional nodes while a previous connection + // attempt has not been completed. + if (isAnyNodeConnecting()) + return reconnectBackoffMs; + + if (connectionStates.canConnect(nodeConnectionId, now)) { + // We don't have a connection to this node right now, make one + log.debug("Initialize connection to node {} for sending telemetry request", node); + initiateConnect(node, now); + return reconnectBackoffMs; + } + + // In either case, we just need to wait for a network event to let us know the selected + // connection might be usable again. + return Long.MAX_VALUE; + } + + public void handleResponse(GetTelemetrySubscriptionsResponse response) { + clientTelemetrySender.handleResponse(response); + } + + public void handleResponse(PushTelemetryResponse response) { + clientTelemetrySender.handleResponse(response); + } + + public void handleFailedRequest(ApiKeys apiKey, KafkaException maybeFatalException) { + if (apiKey == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS) + clientTelemetrySender.handleFailedGetTelemetrySubscriptionsRequest(maybeFatalException); + else if (apiKey == ApiKeys.PUSH_TELEMETRY) + clientTelemetrySender.handleFailedPushTelemetryRequest(maybeFatalException); + else + throw new IllegalStateException("Invalid api key for failed telemetry request"); + } + + public void close() { + try { + clientTelemetrySender.close(); + } catch (Exception exception) { + log.error("Failed to close client telemetry sender", exception); + } + } + } + @Override public ClientRequest newClientRequest(String nodeId, AbstractRequest.Builder requestBuilder, @@ -1239,6 +1355,11 @@ int nextCorrelationId() { return correlation++; } + // visible for testing + Node telemetryConnectedNode() { + return telemetrySender.stickyNode; + } + @Override public ClientRequest newClientRequest(String nodeId, AbstractRequest.Builder requestBuilder, diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java index ed6615ee17eec..ff7f4e661d692 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java @@ -1663,6 +1663,26 @@ default FenceProducersResult fenceProducers(Collection transactionalIds) FenceProducersResult fenceProducers(Collection transactionalIds, FenceProducersOptions options); + /** + * List the client metrics configuration resources available in the cluster. + * + * @param options The options to use when listing the client metrics resources. + * @return The ListClientMetricsResourcesResult. + */ + ListClientMetricsResourcesResult listClientMetricsResources(ListClientMetricsResourcesOptions options); + + /** + * List the client metrics configuration resources available in the cluster with the default options. + *

+ * This is a convenience method for {@link #listClientMetricsResources(ListClientMetricsResourcesOptions)} + * with default options. See the overload for more details. + * + * @return The ListClientMetricsResourcesResult. + */ + default ListClientMetricsResourcesResult listClientMetricsResources() { + return listClientMetricsResources(new ListClientMetricsResourcesOptions()); + } + /** * Determines the client's unique client instance ID used for telemetry. This ID is unique to * this specific client instance and will not change after it is initially generated. diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ClientMetricsResourceListing.java b/clients/src/main/java/org/apache/kafka/clients/admin/ClientMetricsResourceListing.java new file mode 100644 index 0000000000000..873175db34c00 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ClientMetricsResourceListing.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +@InterfaceStability.Evolving +public class ClientMetricsResourceListing { + private final String name; + + public ClientMetricsResourceListing(String name) { + this.name = name; + } + + public String name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClientMetricsResourceListing that = (ClientMetricsResourceListing) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return "ClientMetricsResourceListing(" + + "name='" + name + + ')'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java b/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java index 157bd70656406..9fc809dbddd81 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java @@ -278,6 +278,11 @@ public FenceProducersResult fenceProducers(Collection transactionalIds, return delegate.fenceProducers(transactionalIds, options); } + @Override + public ListClientMetricsResourcesResult listClientMetricsResources(ListClientMetricsResourcesOptions options) { + return delegate.listClientMetricsResources(options); + } + @Override public Uuid clientInstanceId(Duration timeout) { return delegate.clientInstanceId(timeout); diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 9329b29b9785c..f663d6efc60f4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -140,7 +140,9 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData.UserName; import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData; import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsRequestData; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; +import org.apache.kafka.common.message.ListClientMetricsResourcesRequestData; import org.apache.kafka.common.message.ListGroupsRequestData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.message.ListPartitionReassignmentsRequestData; @@ -207,9 +209,13 @@ import org.apache.kafka.common.requests.ElectLeadersResponse; import org.apache.kafka.common.requests.ExpireDelegationTokenRequest; import org.apache.kafka.common.requests.ExpireDelegationTokenResponse; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; import org.apache.kafka.common.requests.IncrementalAlterConfigsRequest; import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse; import org.apache.kafka.common.requests.JoinGroupRequest; +import org.apache.kafka.common.requests.ListClientMetricsResourcesRequest; +import org.apache.kafka.common.requests.ListClientMetricsResourcesResponse; import org.apache.kafka.common.requests.ListGroupsRequest; import org.apache.kafka.common.requests.ListGroupsResponse; import org.apache.kafka.common.requests.ListOffsetsRequest; @@ -376,6 +382,12 @@ public class KafkaAdminClient extends AdminClient { private final long retryBackoffMs; private final long retryBackoffMaxMs; private final ExponentialBackoff retryBackoff; + private final boolean clientTelemetryEnabled; + + /** + * The telemetry requests client instance id. + */ + private Uuid clientInstanceId; /** * Get or create a list value from a map. @@ -530,6 +542,7 @@ static KafkaAdminClient createInternal( } } + // Visible for tests static KafkaAdminClient createInternal(AdminClientConfig config, AdminMetadataManager metadataManager, KafkaClient client, @@ -582,6 +595,7 @@ private KafkaAdminClient(AdminClientConfig config, CommonClientConfigs.RETRY_BACKOFF_EXP_BASE, retryBackoffMaxMs, CommonClientConfigs.RETRY_BACKOFF_JITTER); + this.clientTelemetryEnabled = config.getBoolean(AdminClientConfig.ENABLE_METRICS_PUSH_CONFIG); config.logUnused(); AppInfoParser.registerAppInfo(JMX_PREFIX, clientId, metrics, time.milliseconds()); log.debug("Kafka admin client initialized"); @@ -4385,9 +4399,84 @@ public FenceProducersResult fenceProducers(Collection transactionalIds, return new FenceProducersResult(future.all()); } + @Override + public ListClientMetricsResourcesResult listClientMetricsResources(ListClientMetricsResourcesOptions options) { + final long now = time.milliseconds(); + final KafkaFutureImpl> future = new KafkaFutureImpl<>(); + runnable.call(new Call("listClientMetricsResources", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + ListClientMetricsResourcesRequest.Builder createRequest(int timeoutMs) { + return new ListClientMetricsResourcesRequest.Builder(new ListClientMetricsResourcesRequestData()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + ListClientMetricsResourcesResponse response = (ListClientMetricsResourcesResponse) abstractResponse; + if (response.error().isFailure()) { + future.completeExceptionally(response.error().exception()); + } else { + future.complete(response.clientMetricsResources()); + } + } + + @Override + void handleFailure(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, now); + return new ListClientMetricsResourcesResult(future); + } + @Override public Uuid clientInstanceId(Duration timeout) { - throw new UnsupportedOperationException(); + if (timeout.isNegative()) { + throw new IllegalArgumentException("The timeout cannot be negative."); + } + + if (!clientTelemetryEnabled) { + throw new IllegalStateException("Telemetry is not enabled. Set config `" + AdminClientConfig.ENABLE_METRICS_PUSH_CONFIG + "` to `true`."); + } + + if (clientInstanceId != null) { + return clientInstanceId; + } + + final long now = time.milliseconds(); + final KafkaFutureImpl future = new KafkaFutureImpl<>(); + runnable.call(new Call("getTelemetrySubscriptions", calcDeadlineMs(now, (int) timeout.toMillis()), + new LeastLoadedNodeProvider()) { + + @Override + GetTelemetrySubscriptionsRequest.Builder createRequest(int timeoutMs) { + return new GetTelemetrySubscriptionsRequest.Builder(new GetTelemetrySubscriptionsRequestData(), true); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + GetTelemetrySubscriptionsResponse response = (GetTelemetrySubscriptionsResponse) abstractResponse; + if (response.error() != Errors.NONE) { + future.completeExceptionally(response.error().exception()); + } else { + future.complete(response.data().clientInstanceId()); + } + } + + @Override + void handleFailure(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, now); + + try { + clientInstanceId = future.get(); + } catch (Exception e) { + log.error("Error occurred while fetching client instance id", e); + throw new KafkaException("Error occurred while fetching client instance id", e); + } + + return clientInstanceId; } private void invokeDriver( diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesOptions.java new file mode 100644 index 0000000000000..ca45c58bde10c --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesOptions.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link Admin#listClientMetricsResources()}. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class ListClientMetricsResourcesOptions extends AbstractOptions { +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesResult.java new file mode 100644 index 0000000000000..b939a2c5e214e --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListClientMetricsResourcesResult.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.internals.KafkaFutureImpl; + +import java.util.Collection; + +/** + * The result of the {@link Admin#listClientMetricsResources()} call. + *

+ * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class ListClientMetricsResourcesResult { + private final KafkaFuture> future; + + ListClientMetricsResourcesResult(KafkaFuture> future) { + this.future = future; + } + + /** + * Returns a future that yields either an exception, or the full set of client metrics + * listings. + * + * In the event of a failure, the future yields nothing but the first exception which + * occurred. + */ + public KafkaFuture> all() { + final KafkaFutureImpl> result = new KafkaFutureImpl<>(); + future.whenComplete((listings, throwable) -> { + if (throwable != null) { + result.completeExceptionally(throwable); + } else { + result.complete(listings); + } + }); + return result; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index 43bd2eb174113..213fa3ee52bbf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -662,6 +662,7 @@ protected Map postProcessParsedConfig(final Map CommonClientConfigs.warnDisablingExponentialBackoff(this); Map refinedConfigs = CommonClientConfigs.postProcessReconnectBackoffConfigs(this, parsedValues); maybeOverrideClientId(refinedConfigs); + maybeOverrideEnableAutoCommit(refinedConfigs); return refinedConfigs; } @@ -695,17 +696,17 @@ else if (newConfigs.get(VALUE_DESERIALIZER_CLASS_CONFIG) == null) return newConfigs; } - boolean maybeOverrideEnableAutoCommit() { + private void maybeOverrideEnableAutoCommit(Map configs) { Optional groupId = Optional.ofNullable(getString(CommonClientConfigs.GROUP_ID_CONFIG)); - boolean enableAutoCommit = getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + Map originals = originals(); + boolean enableAutoCommit = originals.containsKey(ENABLE_AUTO_COMMIT_CONFIG) ? getBoolean(ENABLE_AUTO_COMMIT_CONFIG) : false; if (!groupId.isPresent()) { // overwrite in case of default group id where the config is not explicitly provided - if (!originals().containsKey(ENABLE_AUTO_COMMIT_CONFIG)) { - enableAutoCommit = false; + if (!originals.containsKey(ENABLE_AUTO_COMMIT_CONFIG)) { + configs.put(ENABLE_AUTO_COMMIT_CONFIG, false); } else if (enableAutoCommit) { - throw new InvalidConfigurationException(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + " cannot be set to true when default group id (null) is used."); + throw new InvalidConfigurationException(ENABLE_AUTO_COMMIT_CONFIG + " cannot be set to true when default group id (null) is used."); } } - return enableAutoCommit; } public ConsumerConfig(Properties props) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java index 92390e91907e3..17ad8123e2dc7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRecords.java @@ -104,7 +104,7 @@ public Iterator> iterator() { Iterator>> iters = iterables.iterator(); Iterator> current; - public ConsumerRecord makeNext() { + protected ConsumerRecord makeNext() { while (current == null || !current.hasNext()) { if (iters.hasNext()) current = iters.next().iterator(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index bd795e033ab2c..82e1f8b93da3c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -16,26 +16,12 @@ */ package org.apache.kafka.clients.consumer; -import org.apache.kafka.clients.ApiVersions; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; +import org.apache.kafka.clients.consumer.internals.ConsumerDelegateCreator; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; -import org.apache.kafka.clients.consumer.internals.Deserializers; -import org.apache.kafka.clients.consumer.internals.Fetch; -import org.apache.kafka.clients.consumer.internals.FetchConfig; -import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; -import org.apache.kafka.clients.consumer.internals.Fetcher; import org.apache.kafka.clients.consumer.internals.KafkaConsumerMetrics; -import org.apache.kafka.clients.consumer.internals.OffsetFetcher; import org.apache.kafka.clients.consumer.internals.SubscriptionState; -import org.apache.kafka.clients.consumer.internals.TopicMetadataFetcher; -import org.apache.kafka.common.Cluster; -import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; @@ -43,52 +29,23 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.InvalidGroupIdException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; -import org.slf4j.Logger; -import org.slf4j.event.Level; -import java.net.InetSocketAddress; import java.time.Duration; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.OptionalLong; import java.util.Properties; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createConsumerNetworkClient; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; -import static org.apache.kafka.common.utils.Utils.closeQuietly; -import static org.apache.kafka.common.utils.Utils.isBlank; -import static org.apache.kafka.common.utils.Utils.join; import static org.apache.kafka.common.utils.Utils.propsToMap; -import static org.apache.kafka.common.utils.Utils.swallow; /** * A client that consumes records from a Kafka cluster. @@ -468,8 +425,7 @@ * *

Multi-threaded Processing

* - * The Kafka consumer is NOT thread-safe. All network I/O happens in the thread of the application - * making the call. It is the responsibility of the user to ensure that multi-threaded access + * The Kafka consumer is NOT thread-safe. It is the responsibility of the user to ensure that multi-threaded access * is properly synchronized. Un-synchronized access will result in {@link ConcurrentModificationException}. * *

@@ -567,43 +523,9 @@ */ public class KafkaConsumer implements Consumer { - private static final long NO_CURRENT_THREAD = -1L; - static final String DEFAULT_REASON = "rebalance enforced by user"; - - // Visible for testing - final Metrics metrics; - final KafkaConsumerMetrics kafkaConsumerMetrics; - - private Logger log; - private final String clientId; - private final Optional groupId; - private final ConsumerCoordinator coordinator; - private final Deserializers deserializers; - private final Fetcher fetcher; - private final OffsetFetcher offsetFetcher; - private final TopicMetadataFetcher topicMetadataFetcher; - private final ConsumerInterceptors interceptors; - private final IsolationLevel isolationLevel; - - private final Time time; - private final ConsumerNetworkClient client; - private final SubscriptionState subscriptions; - private final ConsumerMetadata metadata; - private final long retryBackoffMs; - private final long retryBackoffMaxMs; - private final long requestTimeoutMs; - private final int defaultApiTimeoutMs; - private volatile boolean closed = false; - private final List assignors; - - // currentThread holds the threadId of the current thread accessing KafkaConsumer - // and is used to prevent multi-threaded access - private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); - // refcount is used to allow reentrant access by the thread who has acquired currentThread - private final AtomicInteger refcount = new AtomicInteger(0); - - // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates - private boolean cachedSubscriptionHasAllFetchPositions; + private final static ConsumerDelegateCreator CREATOR = new ConsumerDelegateCreator(); + + private final ConsumerDelegate delegate; /** * A consumer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings @@ -674,165 +596,30 @@ public KafkaConsumer(Map configs, keyDeserializer, valueDeserializer); } - @SuppressWarnings("unchecked") KafkaConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { - try { - GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, - GroupRebalanceConfig.ProtocolType.CONSUMER); - - this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); - this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); - LogContext logContext = createLogContext(config, groupRebalanceConfig); - this.log = logContext.logger(getClass()); - boolean enableAutoCommit = config.maybeOverrideEnableAutoCommit(); - groupId.ifPresent(groupIdStr -> { - if (groupIdStr.isEmpty()) { - log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release."); - } - }); - - log.debug("Initializing the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); - this.time = Time.SYSTEM; - this.metrics = createMetrics(config, time); - this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - this.retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); - - List> interceptorList = configuredConsumerInterceptors(config); - this.interceptors = new ConsumerInterceptors<>(interceptorList); - this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); - this.subscriptions = createSubscriptionState(config, logContext); - ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners( - metrics.reporters(), - interceptorList, - Arrays.asList(this.deserializers.keyDeserializer, this.deserializers.valueDeserializer)); - this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); - List addresses = ClientUtils.parseAndValidateAddresses(config); - this.metadata.bootstrap(addresses); - - FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); - FetchConfig fetchConfig = new FetchConfig(config); - this.isolationLevel = fetchConfig.isolationLevel; - - ApiVersions apiVersions = new ApiVersions(); - this.client = createConsumerNetworkClient(config, - metrics, - logContext, - apiVersions, - time, - metadata, - fetchMetricsManager.throttleTimeSensor(), - retryBackoffMs); - - this.assignors = ConsumerPartitionAssignor.getAssignorInstances( - config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), - config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) - ); - - // no coordinator will be constructed for the default (null) group id - if (!groupId.isPresent()) { - config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); - config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); - this.coordinator = null; - } else { - this.coordinator = new ConsumerCoordinator(groupRebalanceConfig, - logContext, - this.client, - assignors, - this.metadata, - this.subscriptions, - metrics, - CONSUMER_METRIC_GROUP_PREFIX, - this.time, - enableAutoCommit, - config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), - this.interceptors, - config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED), - config.getString(ConsumerConfig.CLIENT_RACK_CONFIG)); - } - this.fetcher = new Fetcher<>( - logContext, - this.client, - this.metadata, - this.subscriptions, - fetchConfig, - this.deserializers, - fetchMetricsManager, - this.time, - apiVersions); - this.offsetFetcher = new OffsetFetcher(logContext, - client, - metadata, - subscriptions, - time, - retryBackoffMs, - requestTimeoutMs, - isolationLevel, - apiVersions); - this.topicMetadataFetcher = new TopicMetadataFetcher(logContext, - client, - retryBackoffMs, - retryBackoffMaxMs); - - this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); - - config.logUnused(); - AppInfoParser.registerAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics, time.milliseconds()); - log.debug("Kafka consumer initialized"); - } catch (Throwable t) { - // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 - // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. - if (this.log != null) { - close(Duration.ZERO, true); - } - // now propagate the exception - throw new KafkaException("Failed to construct kafka consumer", t); - } + delegate = CREATOR.create(config, keyDeserializer, valueDeserializer); } - // visible for testing KafkaConsumer(LogContext logContext, - String clientId, - ConsumerCoordinator coordinator, + Time time, + ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer, - Fetcher fetcher, - OffsetFetcher offsetFetcher, - TopicMetadataFetcher topicMetadataFetcher, - ConsumerInterceptors interceptors, - Time time, - ConsumerNetworkClient client, - Metrics metrics, + KafkaClient client, SubscriptionState subscriptions, ConsumerMetadata metadata, - long retryBackoffMs, - long retryBackoffMaxMs, - long requestTimeoutMs, - int defaultApiTimeoutMs, - List assignors, - String groupId) { - this.log = logContext.logger(getClass()); - this.clientId = clientId; - this.coordinator = coordinator; - this.deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); - this.fetcher = fetcher; - this.offsetFetcher = offsetFetcher; - this.topicMetadataFetcher = topicMetadataFetcher; - this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; - this.interceptors = Objects.requireNonNull(interceptors); - this.time = time; - this.client = client; - this.metrics = metrics; - this.subscriptions = subscriptions; - this.metadata = metadata; - this.retryBackoffMs = retryBackoffMs; - this.retryBackoffMaxMs = retryBackoffMaxMs; - this.requestTimeoutMs = requestTimeoutMs; - this.defaultApiTimeoutMs = defaultApiTimeoutMs; - this.assignors = assignors; - this.groupId = Optional.ofNullable(groupId); - this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); + List assignors) { + delegate = CREATOR.create( + logContext, + time, + config, + keyDeserializer, + valueDeserializer, + client, + subscriptions, + metadata, + assignors + ); } /** @@ -844,12 +631,7 @@ public KafkaConsumer(Map configs, * @return The set of partitions currently assigned to this consumer */ public Set assignment() { - acquireAndEnsureOpen(); - try { - return Collections.unmodifiableSet(this.subscriptions.assignedPartitions()); - } finally { - release(); - } + return delegate.assignment(); } /** @@ -858,12 +640,7 @@ public Set assignment() { * @return The set of topics currently subscribed to */ public Set subscription() { - acquireAndEnsureOpen(); - try { - return Collections.unmodifiableSet(new HashSet<>(this.subscriptions.subscription())); - } finally { - release(); - } + return delegate.subscription(); } /** @@ -903,10 +680,7 @@ public Set subscription() { */ @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { - if (listener == null) - throw new IllegalArgumentException("RebalanceListener cannot be null"); - - subscribe(topics, Optional.of(listener)); + delegate.subscribe(topics, listener); } /** @@ -932,63 +706,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener liste */ @Override public void subscribe(Collection topics) { - subscribe(topics, Optional.empty()); - } - - /** - * Internal helper method for {@link #subscribe(Collection)} and - * {@link #subscribe(Collection, ConsumerRebalanceListener)} - *

- * Subscribe to the given list of topics to get dynamically assigned partitions. - * Topic subscriptions are not incremental. This list will replace the current - * assignment (if there is one). It is not possible to combine topic subscription with group management - * with manual partition assignment through {@link #assign(Collection)}. - * - * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. - * - *

- * @param topics The list of topics to subscribe to - * @param listener {@link Optional} listener instance to get notifications on partition assignment/revocation - * for the subscribed topics - * @throws IllegalArgumentException If topics is null or contains null or empty elements - * @throws IllegalStateException If {@code subscribe()} is called previously with pattern, or assign is called - * previously (without a subsequent call to {@link #unsubscribe()}), or if not - * configured at-least one partition assignment strategy - */ - private void subscribe(Collection topics, Optional listener) { - acquireAndEnsureOpen(); - try { - maybeThrowInvalidGroupIdException(); - if (topics == null) - throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); - if (topics.isEmpty()) { - // treat subscribing to empty topic list as the same as unsubscribing - this.unsubscribe(); - } else { - for (String topic : topics) { - if (isBlank(topic)) - throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); - } - - throwIfNoAssignorsConfigured(); - - // Clear the buffered data which are not a part of newly assigned topics - final Set currentTopicPartitions = new HashSet<>(); - - for (TopicPartition tp : subscriptions.assignedPartitions()) { - if (topics.contains(tp.topic())) - currentTopicPartitions.add(tp); - } - - fetcher.clearBufferedDataForUnassignedPartitions(currentTopicPartitions); - - log.info("Subscribed to topic(s): {}", join(topics, ", ")); - if (this.subscriptions.subscribe(new HashSet<>(topics), listener)) - metadata.requestUpdateForNewTopics(); - } - } finally { - release(); - } + delegate.subscribe(topics); } /** @@ -1012,10 +730,7 @@ private void subscribe(Collection topics, Optional - * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. - * The pattern matching will be done periodically against all topics existing at the time of check. - * This can be controlled through the {@code metadata.max.age.ms} configuration: by lowering - * the max metadata age, the consumer will refresh metadata more often and check for matching topics. - *

- * See {@link #subscribe(Collection, ConsumerRebalanceListener)} for details on the - * use of the {@link ConsumerRebalanceListener}. Generally rebalances are triggered when there - * is a change to the topics matching the provided pattern and when consumer group membership changes. - * Group rebalances only take place during an active call to {@link #poll(Duration)}. - * - * @param pattern Pattern to subscribe to - * @param listener {@link Optional} listener instance to get notifications on partition assignment/revocation - * for the subscribed topics - * @throws IllegalArgumentException If pattern or listener is null - * @throws IllegalStateException If {@code subscribe()} is called previously with topics, or assign is called - * previously (without a subsequent call to {@link #unsubscribe()}), or if not - * configured at-least one partition assignment strategy - */ - private void subscribe(Pattern pattern, Optional listener) { - maybeThrowInvalidGroupIdException(); - if (pattern == null || pattern.toString().equals("")) - throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? - "null" : "empty")); - - acquireAndEnsureOpen(); - try { - throwIfNoAssignorsConfigured(); - log.info("Subscribed to pattern: '{}'", pattern); - this.subscriptions.subscribe(pattern, listener); - this.coordinator.updatePatternSubscription(metadata.fetch()); - this.metadata.requestUpdateForNewTopics(); - } finally { - release(); - } + delegate.subscribe(pattern); } /** @@ -1086,18 +761,7 @@ private void subscribe(Pattern pattern, Optional list * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. rebalance callback errors) */ public void unsubscribe() { - acquireAndEnsureOpen(); - try { - fetcher.clearBufferedDataForUnassignedPartitions(Collections.emptySet()); - if (this.coordinator != null) { - this.coordinator.onLeavePrepare(); - this.coordinator.maybeLeaveGroup("the consumer unsubscribed from all topics"); - } - this.subscriptions.unsubscribe(); - log.info("Unsubscribed all topics or patterns and assigned partitions"); - } finally { - release(); - } + delegate.unsubscribe(); } /** @@ -1121,32 +785,7 @@ public void unsubscribe() { */ @Override public void assign(Collection partitions) { - acquireAndEnsureOpen(); - try { - if (partitions == null) { - throw new IllegalArgumentException("Topic partition collection to assign to cannot be null"); - } else if (partitions.isEmpty()) { - this.unsubscribe(); - } else { - for (TopicPartition tp : partitions) { - String topic = (tp != null) ? tp.topic() : null; - if (isBlank(topic)) - throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); - } - fetcher.clearBufferedDataForUnassignedPartitions(partitions); - - // make sure the offsets of topic partitions the consumer is unsubscribing from - // are committed since there will be no following rebalance - if (coordinator != null) - this.coordinator.maybeAutoCommitOffsetsAsync(time.milliseconds()); - - log.info("Assigned to partition(s): {}", join(partitions, ", ")); - if (this.subscriptions.assignFromUser(new HashSet<>(partitions))) - metadata.requestUpdateForNewTopics(); - } - } finally { - release(); - } + delegate.assign(partitions); } /** @@ -1185,7 +824,7 @@ public void assign(Collection partitions) { @Deprecated @Override public ConsumerRecords poll(final long timeoutMs) { - return poll(time.timer(timeoutMs), false); + return delegate.poll(timeoutMs); } /** @@ -1232,110 +871,7 @@ public ConsumerRecords poll(final long timeoutMs) { */ @Override public ConsumerRecords poll(final Duration timeout) { - return poll(time.timer(timeout), true); - } - - /** - * @throws KafkaException if the rebalance callback throws exception - */ - private ConsumerRecords poll(final Timer timer, final boolean includeMetadataInTimeout) { - acquireAndEnsureOpen(); - try { - this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); - - if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { - throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); - } - - do { - client.maybeTriggerWakeup(); - - if (includeMetadataInTimeout) { - // try to update assignment metadata BUT do not need to block on the timer for join group - updateAssignmentMetadataIfNeeded(timer, false); - } else { - while (!updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE), true)) { - log.warn("Still waiting for metadata"); - } - } - - final Fetch fetch = pollForFetches(timer); - if (!fetch.isEmpty()) { - // before returning the fetched records, we can send off the next round of fetches - // and avoid block waiting for their responses to enable pipelining while the user - // is handling the fetched records. - // - // NOTE: since the consumed position has already been updated, we must not allow - // wakeups or any other errors to be triggered prior to returning the fetched records. - if (sendFetches() > 0 || client.hasPendingRequests()) { - client.transmitSends(); - } - - if (fetch.records().isEmpty()) { - log.trace("Returning empty records from `poll()` " - + "since the consumer's position has advanced for at least one topic partition"); - } - - return this.interceptors.onConsume(new ConsumerRecords<>(fetch.records())); - } - } while (timer.notExpired()); - - return ConsumerRecords.empty(); - } finally { - release(); - this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); - } - } - - private int sendFetches() { - offsetFetcher.validatePositionsOnMetadataChange(); - return fetcher.sendFetches(); - } - - boolean updateAssignmentMetadataIfNeeded(final Timer timer, final boolean waitForJoinGroup) { - if (coordinator != null && !coordinator.poll(timer, waitForJoinGroup)) { - return false; - } - - return updateFetchPositions(timer); - } - - /** - * @throws KafkaException if the rebalance callback throws exception - */ - private Fetch pollForFetches(Timer timer) { - long pollTimeout = coordinator == null ? timer.remainingMs() : - Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); - - // if data is available already, return it immediately - final Fetch fetch = fetcher.collectFetch(); - if (!fetch.isEmpty()) { - return fetch; - } - - // send any new fetches (won't resend pending fetches) - sendFetches(); - - // We do not want to be stuck blocking in poll if we are missing some positions - // since the offset lookup may be backing off after a failure - - // NOTE: the use of cachedSubscriptionHasAllFetchPositions means we MUST call - // updateAssignmentMetadataIfNeeded before this method. - if (!cachedSubscriptionHasAllFetchPositions && pollTimeout > retryBackoffMs) { - pollTimeout = retryBackoffMs; - } - - log.trace("Polling for fetches with timeout {}", pollTimeout); - - Timer pollTimer = time.timer(pollTimeout); - client.poll(pollTimer, () -> { - // since a fetch might be completed by the background thread, we need this poll condition - // to ensure that we do not block unnecessarily in poll() - return !fetcher.hasAvailableFetches(); - }); - timer.update(pollTimer.currentTimeMs()); - - return fetcher.collectFetch(); + return delegate.poll(timeout); } /** @@ -1379,7 +915,7 @@ private Fetch pollForFetches(Timer timer) { */ @Override public void commitSync() { - commitSync(Duration.ofMillis(defaultApiTimeoutMs)); + delegate.commitSync(); } /** @@ -1422,7 +958,7 @@ public void commitSync() { */ @Override public void commitSync(Duration timeout) { - commitSync(subscriptions.allConsumed(), timeout); + delegate.commitSync(timeout); } /** @@ -1470,7 +1006,7 @@ public void commitSync(Duration timeout) { */ @Override public void commitSync(final Map offsets) { - commitSync(offsets, Duration.ofMillis(defaultApiTimeoutMs)); + delegate.commitSync(offsets); } /** @@ -1518,19 +1054,7 @@ public void commitSync(final Map offsets) { */ @Override public void commitSync(final Map offsets, final Duration timeout) { - acquireAndEnsureOpen(); - long commitStart = time.nanoseconds(); - try { - maybeThrowInvalidGroupIdException(); - offsets.forEach(this::updateLastSeenEpochIfNewer); - if (!coordinator.commitOffsetsSync(new HashMap<>(offsets), time.timer(timeout))) { - throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before successfully " + - "committing offsets " + offsets); - } - } finally { - kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart); - release(); - } + delegate.commitSync(offsets, timeout); } /** @@ -1540,7 +1064,7 @@ public void commitSync(final Map offsets, fin */ @Override public void commitAsync() { - commitAsync(null); + delegate.commitAsync(); } /** @@ -1563,7 +1087,7 @@ public void commitAsync() { */ @Override public void commitAsync(OffsetCommitCallback callback) { - commitAsync(subscriptions.allConsumed(), callback); + delegate.commitAsync(callback); } /** @@ -1590,15 +1114,7 @@ public void commitAsync(OffsetCommitCallback callback) { */ @Override public void commitAsync(final Map offsets, OffsetCommitCallback callback) { - acquireAndEnsureOpen(); - try { - maybeThrowInvalidGroupIdException(); - log.debug("Committing offsets: {}", offsets); - offsets.forEach(this::updateLastSeenEpochIfNewer); - coordinator.commitOffsetsAsync(new HashMap<>(offsets), callback); - } finally { - release(); - } + delegate.commitAsync(offsets, callback); } /** @@ -1632,20 +1148,7 @@ public void commitAsync(final Map offsets, Of */ @Override public void seek(TopicPartition partition, long offset) { - if (offset < 0) - throw new IllegalArgumentException("seek offset must not be a negative number"); - - acquireAndEnsureOpen(); - try { - log.info("Seeking to offset {} for partition {}", offset, partition); - SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( - offset, - Optional.empty(), // This will ensure we skip validation - this.metadata.currentLeader(partition)); - this.subscriptions.seekUnvalidated(partition, newPosition); - } finally { - release(); - } + delegate.seek(partition, offset); } /** @@ -1659,29 +1162,7 @@ public void seek(TopicPartition partition, long offset) { */ @Override public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { - long offset = offsetAndMetadata.offset(); - if (offset < 0) { - throw new IllegalArgumentException("seek offset must not be a negative number"); - } - - acquireAndEnsureOpen(); - try { - if (offsetAndMetadata.leaderEpoch().isPresent()) { - log.info("Seeking to offset {} for partition {} with epoch {}", - offset, partition, offsetAndMetadata.leaderEpoch().get()); - } else { - log.info("Seeking to offset {} for partition {}", offset, partition); - } - Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.currentLeader(partition); - SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( - offsetAndMetadata.offset(), - offsetAndMetadata.leaderEpoch(), - currentLeaderAndEpoch); - this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); - this.subscriptions.seekUnvalidated(partition, newPosition); - } finally { - release(); - } + delegate.seek(partition, offsetAndMetadata); } /** @@ -1694,16 +1175,7 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) */ @Override public void seekToBeginning(Collection partitions) { - if (partitions == null) - throw new IllegalArgumentException("Partitions collection cannot be null"); - - acquireAndEnsureOpen(); - try { - Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; - subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); - } finally { - release(); - } + delegate.seekToBeginning(partitions); } /** @@ -1719,16 +1191,7 @@ public void seekToBeginning(Collection partitions) { */ @Override public void seekToEnd(Collection partitions) { - if (partitions == null) - throw new IllegalArgumentException("Partitions collection cannot be null"); - - acquireAndEnsureOpen(); - try { - Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; - subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); - } finally { - release(); - } + delegate.seekToEnd(partitions); } /** @@ -1759,7 +1222,7 @@ public void seekToEnd(Collection partitions) { */ @Override public long position(TopicPartition partition) { - return position(partition, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.position(partition); } /** @@ -1789,26 +1252,7 @@ public long position(TopicPartition partition) { */ @Override public long position(TopicPartition partition, final Duration timeout) { - acquireAndEnsureOpen(); - try { - if (!this.subscriptions.isAssigned(partition)) - throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); - - Timer timer = time.timer(timeout); - do { - SubscriptionState.FetchPosition position = this.subscriptions.validPosition(partition); - if (position != null) - return position.offset; - - updateFetchPositions(timer); - client.poll(timer); - } while (timer.notExpired()); - - throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + - "for partition " + partition + " could be determined"); - } finally { - release(); - } + return delegate.position(partition, timeout); } /** @@ -1838,7 +1282,7 @@ public long position(TopicPartition partition, final Duration timeout) { @Deprecated @Override public OffsetAndMetadata committed(TopicPartition partition) { - return committed(partition, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.committed(partition); } /** @@ -1867,7 +1311,7 @@ public OffsetAndMetadata committed(TopicPartition partition) { @Deprecated @Override public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) { - return committed(Collections.singleton(partition), timeout).get(partition); + return delegate.committed(partition, timeout); } /** @@ -1899,7 +1343,7 @@ public OffsetAndMetadata committed(TopicPartition partition, final Duration time */ @Override public Map committed(final Set partitions) { - return committed(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.committed(partitions); } /** @@ -1927,24 +1371,7 @@ public Map committed(final Set committed(final Set partitions, final Duration timeout) { - acquireAndEnsureOpen(); - long start = time.nanoseconds(); - try { - maybeThrowInvalidGroupIdException(); - final Map offsets; - offsets = coordinator.fetchCommittedOffsets(partitions, time.timer(timeout)); - if (offsets == null) { - throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " + - "committed offset for partitions " + partitions + " could be determined. Try tuning default.api.timeout.ms " + - "larger to relax the threshold."); - } else { - offsets.forEach(this::updateLastSeenEpochIfNewer); - return offsets; - } - } finally { - kafkaConsumerMetrics.recordCommitted(time.nanoseconds() - start); - release(); - } + return delegate.committed(partitions, timeout); } /** @@ -1974,7 +1401,7 @@ public Map committed(final Set metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + return delegate.metrics(); } /** @@ -2004,7 +1431,7 @@ public Uuid clientInstanceId(Duration timeout) { */ @Override public List partitionsFor(String topic) { - return partitionsFor(topic, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.partitionsFor(topic); } /** @@ -2028,19 +1455,7 @@ public List partitionsFor(String topic) { */ @Override public List partitionsFor(String topic, Duration timeout) { - acquireAndEnsureOpen(); - try { - Cluster cluster = this.metadata.fetch(); - List parts = cluster.partitionsForTopic(topic); - if (!parts.isEmpty()) - return parts; - - Timer timer = time.timer(timeout); - List topicMetadata = topicMetadataFetcher.getTopicMetadata(topic, metadata.allowAutoTopicCreation(), timer); - return topicMetadata != null ? topicMetadata : Collections.emptyList(); - } finally { - release(); - } + return delegate.partitionsFor(topic, timeout); } /** @@ -2059,7 +1474,7 @@ public List partitionsFor(String topic, Duration timeout) { */ @Override public Map> listTopics() { - return listTopics(Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.listTopics(); } /** @@ -2079,12 +1494,7 @@ public Map> listTopics() { */ @Override public Map> listTopics(Duration timeout) { - acquireAndEnsureOpen(); - try { - return topicMetadataFetcher.getAllTopicMetadata(time.timer(timeout)); - } finally { - release(); - } + return delegate.listTopics(timeout); } /** @@ -2099,15 +1509,7 @@ public Map> listTopics(Duration timeout) { */ @Override public void pause(Collection partitions) { - acquireAndEnsureOpen(); - try { - log.debug("Pausing partitions {}", partitions); - for (TopicPartition partition: partitions) { - subscriptions.pause(partition); - } - } finally { - release(); - } + delegate.pause(partitions); } /** @@ -2119,15 +1521,7 @@ public void pause(Collection partitions) { */ @Override public void resume(Collection partitions) { - acquireAndEnsureOpen(); - try { - log.debug("Resuming partitions {}", partitions); - for (TopicPartition partition: partitions) { - subscriptions.resume(partition); - } - } finally { - release(); - } + delegate.resume(partitions); } /** @@ -2137,12 +1531,7 @@ public void resume(Collection partitions) { */ @Override public Set paused() { - acquireAndEnsureOpen(); - try { - return Collections.unmodifiableSet(subscriptions.pausedPartitions()); - } finally { - release(); - } + return delegate.paused(); } /** @@ -2168,7 +1557,7 @@ public Set paused() { */ @Override public Map offsetsForTimes(Map timestampsToSearch) { - return offsetsForTimes(timestampsToSearch, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.offsetsForTimes(timestampsToSearch); } /** @@ -2195,19 +1584,7 @@ public Map offsetsForTimes(Map offsetsForTimes(Map timestampsToSearch, Duration timeout) { - acquireAndEnsureOpen(); - try { - for (Map.Entry entry : timestampsToSearch.entrySet()) { - // we explicitly exclude the earliest and latest offset here so the timestamp in the returned - // OffsetAndTimestamp is always positive. - if (entry.getValue() < 0) - throw new IllegalArgumentException("The target time for partition " + entry.getKey() + " is " + - entry.getValue() + ". The target time cannot be negative."); - } - return offsetFetcher.offsetsForTimes(timestampsToSearch, time.timer(timeout)); - } finally { - release(); - } + return delegate.offsetsForTimes(timestampsToSearch, timeout); } /** @@ -2226,7 +1603,7 @@ public Map offsetsForTimes(Map beginningOffsets(Collection partitions) { - return beginningOffsets(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.beginningOffsets(partitions); } /** @@ -2247,12 +1624,7 @@ public Map beginningOffsets(Collection par */ @Override public Map beginningOffsets(Collection partitions, Duration timeout) { - acquireAndEnsureOpen(); - try { - return offsetFetcher.beginningOffsets(partitions, time.timer(timeout)); - } finally { - release(); - } + return delegate.beginningOffsets(partitions, timeout); } /** @@ -2276,7 +1648,7 @@ public Map beginningOffsets(Collection par */ @Override public Map endOffsets(Collection partitions) { - return endOffsets(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + return delegate.endOffsets(partitions); } /** @@ -2302,12 +1674,7 @@ public Map endOffsets(Collection partition */ @Override public Map endOffsets(Collection partitions, Duration timeout) { - acquireAndEnsureOpen(); - try { - return offsetFetcher.endOffsets(partitions, time.timer(timeout)); - } finally { - release(); - } + return delegate.endOffsets(partitions, timeout); } /** @@ -2326,30 +1693,7 @@ public Map endOffsets(Collection partition */ @Override public OptionalLong currentLag(TopicPartition topicPartition) { - acquireAndEnsureOpen(); - try { - final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel); - - // if the log end offset is not known and hence cannot return lag and there is - // no in-flight list offset requested yet, - // issue a list offset request for that partition so that next time - // we may get the answer; we do not need to wait for the return value - // since we would not try to poll the network client synchronously - if (lag == null) { - if (subscriptions.partitionEndOffset(topicPartition, isolationLevel) == null && - !subscriptions.partitionEndOffsetRequested(topicPartition)) { - log.info("Requesting the log end offset for {} in order to compute lag", topicPartition); - subscriptions.requestPartitionEndOffset(topicPartition); - offsetFetcher.endOffsets(Collections.singleton(topicPartition), time.timer(0L)); - } - - return OptionalLong.empty(); - } - - return OptionalLong.of(lag); - } finally { - release(); - } + return delegate.currentLag(topicPartition); } /** @@ -2360,13 +1704,7 @@ public OptionalLong currentLag(TopicPartition topicPartition) { */ @Override public ConsumerGroupMetadata groupMetadata() { - acquireAndEnsureOpen(); - try { - maybeThrowInvalidGroupIdException(); - return coordinator.groupMetadata(); - } finally { - release(); - } + return delegate.groupMetadata(); } /** @@ -2393,15 +1731,7 @@ public ConsumerGroupMetadata groupMetadata() { */ @Override public void enforceRebalance(final String reason) { - acquireAndEnsureOpen(); - try { - if (coordinator == null) { - throw new IllegalStateException("Tried to force a rebalance but consumer does not have a group."); - } - coordinator.requestRejoin(reason == null || reason.isEmpty() ? DEFAULT_REASON : reason); - } finally { - release(); - } + delegate.enforceRebalance(reason); } /** @@ -2409,7 +1739,7 @@ public void enforceRebalance(final String reason) { */ @Override public void enforceRebalance() { - enforceRebalance(null); + delegate.enforceRebalance(); } /** @@ -2424,7 +1754,7 @@ public void enforceRebalance() { */ @Override public void close() { - close(Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS)); + delegate.close(); } /** @@ -2444,19 +1774,7 @@ public void close() { */ @Override public void close(Duration timeout) { - if (timeout.toMillis() < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - acquire(); - try { - if (!closed) { - // need to close before setting the flag since the close function - // itself may trigger rebalance callback that needs the consumer to be open still - close(timeout, false); - } - } finally { - closed = true; - release(); - } + delegate.close(timeout); } /** @@ -2466,152 +1784,23 @@ public void close(Duration timeout) { */ @Override public void wakeup() { - this.client.wakeup(); - } - - private Timer createTimerForRequest(final Duration timeout) { - // this.time could be null if an exception occurs in constructor prior to setting the this.time field - final Time localTime = (time == null) ? Time.SYSTEM : time; - return localTime.timer(Math.min(timeout.toMillis(), requestTimeoutMs)); - } - - private void close(Duration timeout, boolean swallowException) { - log.trace("Closing the Kafka consumer"); - AtomicReference firstException = new AtomicReference<>(); - - final Timer closeTimer = createTimerForRequest(timeout); - // Close objects with a timeout. The timeout is required because the coordinator & the fetcher send requests to - // the server in the process of closing which may not respect the overall timeout defined for closing the - // consumer. - if (coordinator != null) { - // This is a blocking call bound by the time remaining in closeTimer - swallow(log, Level.ERROR, "Failed to close coordinator with a timeout(ms)=" + closeTimer.timeoutMs(), () -> coordinator.close(closeTimer), firstException); - } - - if (fetcher != null) { - // the timeout for the session close is at-most the requestTimeoutMs - long remainingDurationInTimeout = Math.max(0, timeout.toMillis() - closeTimer.elapsedMs()); - if (remainingDurationInTimeout > 0) { - remainingDurationInTimeout = Math.min(requestTimeoutMs, remainingDurationInTimeout); - } - - closeTimer.reset(remainingDurationInTimeout); - - // This is a blocking call bound by the time remaining in closeTimer - swallow(log, Level.ERROR, "Failed to close fetcher with a timeout(ms)=" + closeTimer.timeoutMs(), () -> fetcher.close(closeTimer), firstException); - } - - closeQuietly(interceptors, "consumer interceptors", firstException); - closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); - closeQuietly(metrics, "consumer metrics", firstException); - closeQuietly(client, "consumer network client", firstException); - closeQuietly(deserializers, "consumer deserializers", firstException); - AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); - log.debug("Kafka consumer has been closed"); - Throwable exception = firstException.get(); - if (exception != null && !swallowException) { - if (exception instanceof InterruptException) { - throw (InterruptException) exception; - } - throw new KafkaException("Failed to close kafka consumer", exception); - } + delegate.wakeup(); } - /** - * Set the fetch position to the committed position (if there is one) - * or reset it using the offset reset policy the user has configured. - * - * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details - * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is - * defined - * @return true iff the operation completed without timing out - */ - private boolean updateFetchPositions(final Timer timer) { - // If any partitions have been truncated due to a leader change, we need to validate the offsets - offsetFetcher.validatePositionsIfNeeded(); - - cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions(); - if (cachedSubscriptionHasAllFetchPositions) return true; - - // If there are any partitions which do not have a valid position and are not - // awaiting reset, then we need to fetch committed offsets. We will only do a - // coordinator lookup if there are partitions which have missing positions, so - // a consumer with manually assigned partitions can avoid a coordinator dependence - // by always ensuring that assigned partitions have an initial position. - if (coordinator != null && !coordinator.initWithCommittedOffsetsIfNeeded(timer)) return false; - - // If there are partitions still needing a position and a reset policy is defined, - // request reset using the default policy. If no reset strategy is defined and there - // are partitions with a missing position, then we will raise an exception. - subscriptions.resetInitializingPositions(); - - // Finally send an asynchronous request to look up and update the positions of any - // partitions which are awaiting reset. - offsetFetcher.resetPositionsIfNeeded(); - - return true; - } - - /** - * Acquire the light lock and ensure that the consumer hasn't been closed. - * @throws IllegalStateException If the consumer has been closed - */ - private void acquireAndEnsureOpen() { - acquire(); - if (this.closed) { - release(); - throw new IllegalStateException("This consumer has already been closed."); - } - } - - /** - * Acquire the light lock protecting this consumer from multi-threaded access. Instead of blocking - * when the lock is not available, however, we just throw an exception (since multi-threaded usage is not - * supported). - * @throws ConcurrentModificationException if another thread already has the lock - */ - private void acquire() { - final Thread thread = Thread.currentThread(); - final long threadId = thread.getId(); - if (threadId != currentThread.get() && !currentThread.compareAndSet(NO_CURRENT_THREAD, threadId)) - throw new ConcurrentModificationException("KafkaConsumer is not safe for multi-threaded access. " + - "currentThread(name: " + thread.getName() + ", id: " + threadId + ")" + - " otherThread(id: " + currentThread.get() + ")" - ); - refcount.incrementAndGet(); - } - - /** - * Release the light lock protecting the consumer from multi-threaded access. - */ - private void release() { - if (refcount.decrementAndGet() == 0) - currentThread.set(NO_CURRENT_THREAD); - } - - private void throwIfNoAssignorsConfigured() { - if (assignors.isEmpty()) - throw new IllegalStateException("Must configure at least one partition assigner class name to " + - ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG + " configuration property"); - } - - private void maybeThrowInvalidGroupIdException() { - if (!groupId.isPresent()) - throw new InvalidGroupIdException("To use the group management or offset commit APIs, you must " + - "provide a valid " + ConsumerConfig.GROUP_ID_CONFIG + " in the consumer configuration."); + // Functions below are for testing only + String clientId() { + return delegate.clientId(); } - private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) { - if (offsetAndMetadata != null) - offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch)); + Metrics metricsRegistry() { + return delegate.metricsRegistry(); } - // Functions below are for testing only - String getClientId() { - return clientId; + KafkaConsumerMetrics kafkaConsumerMetrics() { + return delegate.kafkaConsumerMetrics(); } boolean updateAssignmentMetadataIfNeeded(final Timer timer) { - return updateAssignmentMetadataIfNeeded(timer, true); + return delegate.updateAssignmentMetadataIfNeeded(timer); } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index 24d05e7d63bfb..3ab4f2e7e6d31 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -64,6 +64,8 @@ import org.apache.kafka.common.requests.OffsetCommitRequest; import org.apache.kafka.common.requests.SyncGroupRequest; import org.apache.kafka.common.requests.SyncGroupResponse; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryProvider; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; import org.apache.kafka.common.utils.ExponentialBackoff; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; @@ -79,6 +81,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -130,6 +133,7 @@ public boolean hasNotJoinedGroup() { private final Heartbeat heartbeat; private final GroupCoordinatorMetrics sensors; private final GroupRebalanceConfig rebalanceConfig; + private final Optional clientTelemetryReporter; protected final Time time; protected final ConsumerNetworkClient client; @@ -160,6 +164,16 @@ public AbstractCoordinator(GroupRebalanceConfig rebalanceConfig, Metrics metrics, String metricGrpPrefix, Time time) { + this(rebalanceConfig, logContext, client, metrics, metricGrpPrefix, time, Optional.empty()); + } + + public AbstractCoordinator(GroupRebalanceConfig rebalanceConfig, + LogContext logContext, + ConsumerNetworkClient client, + Metrics metrics, + String metricGrpPrefix, + Time time, + Optional clientTelemetryReporter) { Objects.requireNonNull(rebalanceConfig.groupId, "Expected a non-null group id for coordinator construction"); this.rebalanceConfig = rebalanceConfig; @@ -173,6 +187,7 @@ public AbstractCoordinator(GroupRebalanceConfig rebalanceConfig, CommonClientConfigs.RETRY_BACKOFF_JITTER); this.heartbeat = new Heartbeat(rebalanceConfig, time); this.sensors = new GroupCoordinatorMetrics(metrics, metricGrpPrefix); + this.clientTelemetryReporter = clientTelemetryReporter; } /** @@ -648,6 +663,8 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut joinResponse.data().memberId(), joinResponse.data().protocolName()); log.info("Successfully joined group with generation {}", AbstractCoordinator.this.generation); + clientTelemetryReporter.ifPresent(reporter -> reporter.updateMetricsLabels( + Collections.singletonMap(ClientTelemetryProvider.GROUP_MEMBER_ID, joinResponse.data().memberId()))); if (joinResponse.isLeader()) { onLeaderElected(joinResponse).chain(future); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java similarity index 50% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java index a90d37597a34f..6c67f0bffdc56 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java @@ -20,6 +20,7 @@ import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -28,22 +29,28 @@ import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.NoOffsetForPartitionException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.RetriableCommitFailedException; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.EventProcessor; +import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.SubscriptionChangeApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.UnsubscribeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; @@ -53,14 +60,19 @@ import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; -import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.FencedInstanceIdException; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.InvalidGroupIdException; +import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.ListOffsetsRequest; import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryUtils; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -72,18 +84,20 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.ConcurrentModificationException; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; -import java.util.Properties; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -91,11 +105,10 @@ import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; @@ -105,20 +118,98 @@ import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; -import static org.apache.kafka.common.utils.Utils.propsToMap; /** - * This prototype consumer uses an {@link ApplicationEventHandler event handler} to process - * {@link ApplicationEvent application events} so that the network IO can be processed in a dedicated + * This {@link Consumer} implementation uses an {@link ApplicationEventHandler event handler} to process + * {@link ApplicationEvent application events} so that the network I/O can be processed in a dedicated * {@link ConsumerNetworkThread network thread}. Visit * this document - * for detail implementation. + * for implementation detail. + * + *

+ * + * Note: this {@link Consumer} implementation is part of the revised consumer group protocol from KIP-848. + * This class should not be invoked directly; users should instead create a {@link KafkaConsumer} as before. + * This consumer implements the new consumer group protocol and is intended to be the default in coming releases. */ -public class PrototypeAsyncConsumer implements Consumer { +public class AsyncKafkaConsumer implements ConsumerDelegate { + + private static final long NO_CURRENT_THREAD = -1L; + + /** + * An {@link org.apache.kafka.clients.consumer.internals.events.EventProcessor} that is created and executes in the + * application thread for the purpose of processing {@link BackgroundEvent background events} generated by the + * {@link ConsumerNetworkThread network thread}. + * Those events are generally of two types: + * + *

    + *
  • Errors that occur in the network thread that need to be propagated to the application thread
    • + *
  • {@link ConsumerRebalanceListener} callbacks that are to be executed on the application thread
    • + *
+ */ + public class BackgroundEventProcessor extends EventProcessor { + + public BackgroundEventProcessor(final LogContext logContext, + final BlockingQueue backgroundEventQueue) { + super(logContext, backgroundEventQueue); + } + + /** + * Process the events—if any—that were produced by the {@link ConsumerNetworkThread network thread}. + * It is possible that {@link org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent an error} + * could occur when processing the events. In such cases, the processor will take a reference to the first + * error, continue to process the remaining events, and then throw the first error that occurred. + */ + @Override + public void process() { + AtomicReference firstError = new AtomicReference<>(); + process((event, error) -> firstError.compareAndSet(null, error)); + + if (firstError.get() != null) { + throw firstError.get(); + } + } + + @Override + public void process(final BackgroundEvent event) { + switch (event.type()) { + case ERROR: + process((ErrorBackgroundEvent) event); + break; + case GROUP_METADATA_UPDATE: + process((GroupMetadataUpdateEvent) event); + break; + default: + throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); + + } + } + + @Override + protected Class getEventClass() { + return BackgroundEvent.class; + } + + private void process(final ErrorBackgroundEvent event) { + throw event.error(); + } + + private void process(final GroupMetadataUpdateEvent event) { + if (AsyncKafkaConsumer.this.groupMetadata.isPresent()) { + final ConsumerGroupMetadata currentGroupMetadata = AsyncKafkaConsumer.this.groupMetadata.get(); + AsyncKafkaConsumer.this.groupMetadata = Optional.of(new ConsumerGroupMetadata( + currentGroupMetadata.groupId(), + event.memberEpoch(), + event.memberId(), + currentGroupMetadata.groupInstanceId() + )); + } + } + } private final ApplicationEventHandler applicationEventHandler; private final Time time; - private final Optional groupId; + private Optional groupMetadata; private final KafkaConsumerMetrics kafkaConsumerMetrics; private Logger log; private final String clientId; @@ -140,56 +231,48 @@ public class PrototypeAsyncConsumer implements Consumer { private final ConsumerMetadata metadata; private final Metrics metrics; private final long retryBackoffMs; - private final long defaultApiTimeoutMs; + private final int defaultApiTimeoutMs; private volatile boolean closed = false; private final List assignors; + private final Optional clientTelemetryReporter; // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates private boolean cachedSubscriptionHasAllFetchPositions; private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); + private boolean isFenced = false; + private final OffsetCommitCallbackInvoker invoker = new OffsetCommitCallbackInvoker(); - public PrototypeAsyncConsumer(final Properties properties, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - this(propsToMap(properties), keyDeserializer, valueDeserializer); - } - - public PrototypeAsyncConsumer(final Map configs, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - this(new ConsumerConfig(appendDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), - keyDeserializer, - valueDeserializer); - } + // currentThread holds the threadId of the current thread accessing the AsyncKafkaConsumer + // and is used to prevent multithreaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + private final AtomicInteger refCount = new AtomicInteger(0); - public PrototypeAsyncConsumer(final ConsumerConfig config, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - this(Time.SYSTEM, config, keyDeserializer, valueDeserializer); + AsyncKafkaConsumer(final ConsumerConfig config, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { + this(config, keyDeserializer, valueDeserializer, new LinkedBlockingQueue<>()); } - public PrototypeAsyncConsumer(final Time time, - final ConsumerConfig config, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { + AsyncKafkaConsumer(final ConsumerConfig config, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer, + final LinkedBlockingQueue backgroundEventQueue) { try { - GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, - GroupRebalanceConfig.ProtocolType.CONSUMER); - - this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( + config, + GroupRebalanceConfig.ProtocolType.CONSUMER + ); this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); LogContext logContext = createLogContext(config, groupRebalanceConfig); this.log = logContext.logger(getClass()); - groupId.ifPresent(groupIdStr -> { - if (groupIdStr.isEmpty()) { - log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release."); - } - }); log.debug("Initializing the Kafka consumer"); this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); - this.time = time; - this.metrics = createMetrics(config, time); + this.time = Time.SYSTEM; + List reporters = CommonClientConfigs.metricsReporters(clientId, config); + this.clientTelemetryReporter = CommonClientConfigs.telemetryReporter(clientId, config); + this.clientTelemetryReporter.ifPresent(reporters::add); + this.metrics = createMetrics(config, time, reporters); this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); List> interceptorList = configuredConsumerInterceptors(config); @@ -209,7 +292,6 @@ public PrototypeAsyncConsumer(final Time time, ApiVersions apiVersions = new ApiVersions(); final BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); - final BlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); // This FetchBuffer is shared between the application and network threads. this.fetchBuffer = new FetchBuffer(logContext); @@ -219,7 +301,8 @@ public PrototypeAsyncConsumer(final Time time, config, apiVersions, metrics, - fetchMetricsManager); + fetchMetricsManager, + clientTelemetryReporter.map(ClientTelemetryReporter::telemetrySender).orElse(null)); final Supplier requestManagersSupplier = RequestManagers.supplier(time, logContext, backgroundEventQueue, @@ -247,11 +330,7 @@ public PrototypeAsyncConsumer(final Time time, config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) ); - // no coordinator will be constructed for the default (null) group id - if (!groupId.isPresent()) { - config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); - //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); - } + this.groupMetadata = initializeGroupMetadata(config, groupRebalanceConfig); // The FetchCollector is only used on the application thread. this.fetchCollector = new FetchCollector<>(logContext, @@ -278,22 +357,23 @@ public PrototypeAsyncConsumer(final Time time, } } - public PrototypeAsyncConsumer(LogContext logContext, - String clientId, - Deserializers deserializers, - FetchBuffer fetchBuffer, - FetchCollector fetchCollector, - ConsumerInterceptors interceptors, - Time time, - ApplicationEventHandler applicationEventHandler, - BlockingQueue backgroundEventQueue, - Metrics metrics, - SubscriptionState subscriptions, - ConsumerMetadata metadata, - long retryBackoffMs, - int defaultApiTimeoutMs, - List assignors, - String groupId) { + // Visible for testing + AsyncKafkaConsumer(LogContext logContext, + String clientId, + Deserializers deserializers, + FetchBuffer fetchBuffer, + FetchCollector fetchCollector, + ConsumerInterceptors interceptors, + Time time, + ApplicationEventHandler applicationEventHandler, + BlockingQueue backgroundEventQueue, + Metrics metrics, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + long retryBackoffMs, + int defaultApiTimeoutMs, + List assignors, + String groupId) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; this.clientId = clientId; @@ -304,7 +384,7 @@ public PrototypeAsyncConsumer(LogContext logContext, this.time = time; this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); this.metrics = metrics; - this.groupId = Optional.ofNullable(groupId); + this.groupMetadata = initializeGroupMetadata(groupId, Optional.empty()); this.metadata = metadata; this.retryBackoffMs = retryBackoffMs; this.defaultApiTimeoutMs = defaultApiTimeoutMs; @@ -312,6 +392,118 @@ public PrototypeAsyncConsumer(LogContext logContext, this.applicationEventHandler = applicationEventHandler; this.assignors = assignors; this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); + this.clientTelemetryReporter = Optional.empty(); + } + + // Visible for testing + AsyncKafkaConsumer(LogContext logContext, + Time time, + ConsumerConfig config, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + KafkaClient client, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + List assignors) { + this.log = logContext.logger(getClass()); + this.subscriptions = subscriptions; + this.clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + this.fetchBuffer = new FetchBuffer(logContext); + this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; + this.interceptors = new ConsumerInterceptors<>(Collections.emptyList()); + this.time = time; + this.metrics = new Metrics(time); + this.metadata = metadata; + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); + this.deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + this.assignors = assignors; + this.clientTelemetryReporter = Optional.empty(); + + ConsumerMetrics metricsRegistry = new ConsumerMetrics(CONSUMER_METRIC_GROUP_PREFIX); + FetchMetricsManager fetchMetricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics); + this.fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + new FetchConfig(config), + deserializers, + fetchMetricsManager, + time); + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); + + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( + config, + GroupRebalanceConfig.ProtocolType.CONSUMER + ); + + this.groupMetadata = initializeGroupMetadata(config, groupRebalanceConfig); + + BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); + BlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); + ApiVersions apiVersions = new ApiVersions(); + Supplier networkClientDelegateSupplier = () -> new NetworkClientDelegate( + time, + config, + logContext, + client + ); + Supplier requestManagersSupplier = RequestManagers.supplier( + time, + logContext, + backgroundEventQueue, + metadata, + subscriptions, + fetchBuffer, + config, + groupRebalanceConfig, + apiVersions, + fetchMetricsManager, + networkClientDelegateSupplier + ); + Supplier applicationEventProcessorSupplier = ApplicationEventProcessor.supplier( + logContext, + metadata, + applicationEventQueue, + requestManagersSupplier + ); + this.applicationEventHandler = new ApplicationEventHandler(logContext, + time, + applicationEventQueue, + applicationEventProcessorSupplier, + networkClientDelegateSupplier, + requestManagersSupplier); + } + + private Optional initializeGroupMetadata(final ConsumerConfig config, + final GroupRebalanceConfig groupRebalanceConfig) { + final Optional groupMetadata = initializeGroupMetadata( + groupRebalanceConfig.groupId, + groupRebalanceConfig.groupInstanceId + ); + if (!groupMetadata.isPresent()) { + config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); + config.ignore(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); + } + return groupMetadata; + } + + private Optional initializeGroupMetadata(final String groupId, + final Optional groupInstanceId) { + if (groupId != null) { + if (groupId.isEmpty()) { + throw new InvalidGroupIdException("The configured " + ConsumerConfig.GROUP_ID_CONFIG + + " should not be an empty string or whitespace."); + } else { + return Optional.of(new ConsumerGroupMetadata( + groupId, + JoinGroupRequest.UNKNOWN_GENERATION_ID, + JoinGroupRequest.UNKNOWN_MEMBER_ID, + groupInstanceId + )); + } + } + return Optional.empty(); } /** @@ -323,12 +515,29 @@ public PrototypeAsyncConsumer(LogContext logContext, * * @param timeout timeout of the poll loop * @return ConsumerRecord. It can be empty if time timeout expires. + * + * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this + * function is called + * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while + * this function is called + * @throws org.apache.kafka.common.errors.RecordTooLargeException if the fetched record is larger than the maximum + * allowable size + * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors + * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any + * partitions to consume from or an unexpected error occurred + * @throws org.apache.kafka.clients.consumer.OffsetOutOfRangeException if the fetch position of the consumer is + * out of range and no offset reset policy is configured. + * @throws org.apache.kafka.common.errors.TopicAuthorizationException if the consumer is not authorized to read + * from a partition + * @throws org.apache.kafka.common.errors.SerializationException if the fetched records cannot be deserialized */ @Override public ConsumerRecords poll(final Duration timeout) { Timer timer = time.timer(timeout); + acquireAndEnsureOpen(); try { + wakeupTrigger.setFetchAction(fetchBuffer); kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); if (subscriptions.hasNoSubscriptionOrUserAssignment()) { @@ -336,9 +545,14 @@ public ConsumerRecords poll(final Duration timeout) { } do { + // We must not allow wake-ups between polling for fetches and returning the records. + // If the polled fetches are not empty the consumed position has already been updated in the polling + // of the fetches. A wakeup between returned fetches and returning records would lead to never + // returning the records in the fetches. Thus, we trigger a possible wake-up before we poll fetches. + wakeupTrigger.maybeTriggerWakeup(); + updateAssignmentMetadataIfNeeded(timer); final Fetch fetch = pollForFetches(timer); - if (!fetch.isEmpty()) { if (fetch.records().isEmpty()) { log.trace("Returning empty records from `poll()` " @@ -353,6 +567,8 @@ public ConsumerRecords poll(final Duration timeout) { return ConsumerRecords.empty(); } finally { kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); + wakeupTrigger.clearTask(); + release(); } } @@ -380,22 +596,37 @@ public void commitAsync(OffsetCommitCallback callback) { @Override public void commitAsync(Map offsets, OffsetCommitCallback callback) { - CompletableFuture future = commit(offsets, false); - final OffsetCommitCallback commitCallback = callback == null ? new DefaultOffsetCommitCallback() : callback; - future.whenComplete((r, t) -> { - if (t != null) { - commitCallback.onComplete(offsets, new KafkaException(t)); - } else { - commitCallback.onComplete(offsets, null); - } - }).exceptionally(e -> { - throw new KafkaException(e); - }); + acquireAndEnsureOpen(); + try { + CompletableFuture future = commit(offsets, false); + future.whenComplete((r, t) -> { + if (callback == null) { + if (t != null) { + log.error("Offset commit with offsets {} failed", offsets, t); + } + return; + } + + invoker.submit(new OffsetCommitCallbackTask(callback, offsets, (Exception) t)); + }); + } finally { + release(); + } } // Visible for testing - CompletableFuture commit(Map offsets, final boolean isWakeupable) { + CompletableFuture commit(final Map offsets, final boolean isWakeupable) { + maybeInvokeCommitCallbacks(); + maybeThrowFencedInstanceException(); maybeThrowInvalidGroupIdException(); + + log.debug("Committing offsets: {}", offsets); + offsets.forEach(this::updateLastSeenEpochIfNewer); + + if (offsets.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + final CommitApplicationEvent commitEvent = new CommitApplicationEvent(offsets); if (isWakeupable) { // the task can only be woken up if the top level API call is commitSync @@ -410,12 +641,17 @@ public void seek(TopicPartition partition, long offset) { if (offset < 0) throw new IllegalArgumentException("seek offset must not be a negative number"); - log.info("Seeking to offset {} for partition {}", offset, partition); - SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + acquireAndEnsureOpen(); + try { + log.info("Seeking to offset {} for partition {}", offset, partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( offset, Optional.empty(), // This will ensure we skip validation metadata.currentLeader(partition)); - subscriptions.seekUnvalidated(partition, newPosition); + subscriptions.seekUnvalidated(partition, newPosition); + } finally { + release(); + } } @Override @@ -425,19 +661,24 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) throw new IllegalArgumentException("seek offset must not be a negative number"); } - if (offsetAndMetadata.leaderEpoch().isPresent()) { - log.info("Seeking to offset {} for partition {} with epoch {}", + acquireAndEnsureOpen(); + try { + if (offsetAndMetadata.leaderEpoch().isPresent()) { + log.info("Seeking to offset {} for partition {} with epoch {}", offset, partition, offsetAndMetadata.leaderEpoch().get()); - } else { - log.info("Seeking to offset {} for partition {}", offset, partition); - } - Metadata.LeaderAndEpoch currentLeaderAndEpoch = metadata.currentLeader(partition); - SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + } else { + log.info("Seeking to offset {} for partition {}", offset, partition); + } + Metadata.LeaderAndEpoch currentLeaderAndEpoch = metadata.currentLeader(partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( offsetAndMetadata.offset(), offsetAndMetadata.leaderEpoch(), currentLeaderAndEpoch); - updateLastSeenEpochIfNewer(partition, offsetAndMetadata); - subscriptions.seekUnvalidated(partition, newPosition); + updateLastSeenEpochIfNewer(partition, offsetAndMetadata); + subscriptions.seekUnvalidated(partition, newPosition); + } finally { + release(); + } } @Override @@ -445,8 +686,13 @@ public void seekToBeginning(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; - subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); + acquireAndEnsureOpen(); + try { + Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); + } finally { + release(); + } } @Override @@ -454,8 +700,13 @@ public void seekToEnd(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; - subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); + acquireAndEnsureOpen(); + try { + Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); + } finally { + release(); + } } @Override @@ -465,20 +716,25 @@ public long position(TopicPartition partition) { @Override public long position(TopicPartition partition, Duration timeout) { - if (!subscriptions.isAssigned(partition)) - throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); + acquireAndEnsureOpen(); + try { + if (!subscriptions.isAssigned(partition)) + throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); - Timer timer = time.timer(timeout); - do { - SubscriptionState.FetchPosition position = subscriptions.validPosition(partition); - if (position != null) - return position.offset; + Timer timer = time.timer(timeout); + do { + SubscriptionState.FetchPosition position = subscriptions.validPosition(partition); + if (position != null) + return position.offset; - updateFetchPositions(timer); - } while (timer.notExpired()); + updateFetchPositions(timer); + } while (timer.notExpired()); - throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + "for partition " + partition + " could be determined"); + } finally { + release(); + } } @Override @@ -501,24 +757,36 @@ public Map committed(final Set committed(final Set partitions, final Duration timeout) { - maybeThrowInvalidGroupIdException(); - if (partitions.isEmpty()) { - return new HashMap<>(); - } - - final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(partitions); - wakeupTrigger.setActiveTask(event.future()); + acquireAndEnsureOpen(); try { - return applicationEventHandler.addAndGet(event, time.timer(timeout)); + maybeThrowInvalidGroupIdException(); + if (partitions.isEmpty()) { + return Collections.emptyMap(); + } + + final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(partitions); + wakeupTrigger.setActiveTask(event.future()); + try { + final Map committedOffsets = applicationEventHandler.addAndGet(event, + time.timer(timeout)); + committedOffsets.forEach(this::updateLastSeenEpochIfNewer); + return committedOffsets; + } catch (TimeoutException e) { + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " + + "committed offset for partitions " + partitions + " could be determined. Try tuning " + + ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG + " larger to relax the threshold."); + } finally { + wakeupTrigger.clearTask(); + } } finally { - wakeupTrigger.clearActiveTask(); + release(); } } private void maybeThrowInvalidGroupIdException() { - if (!groupId.isPresent() || groupId.get().isEmpty()) { + if (!groupMetadata.isPresent()) { throw new InvalidGroupIdException("To use the group management or offset commit APIs, you must " + - "provide a valid " + ConsumerConfig.GROUP_ID_CONFIG + " in the consumer configuration."); + "provide a valid " + ConsumerConfig.GROUP_ID_CONFIG + " in the consumer configuration."); } } @@ -549,22 +817,37 @@ public Map> listTopics(Duration timeout) { @Override public Set paused() { - return Collections.unmodifiableSet(subscriptions.pausedPartitions()); + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(subscriptions.pausedPartitions()); + } finally { + release(); + } } @Override public void pause(Collection partitions) { - log.debug("Pausing partitions {}", partitions); - for (TopicPartition partition: partitions) { - subscriptions.pause(partition); + acquireAndEnsureOpen(); + try { + log.debug("Pausing partitions {}", partitions); + for (TopicPartition partition : partitions) { + subscriptions.pause(partition); + } + } finally { + release(); } } @Override public void resume(Collection partitions) { - log.debug("Resuming partitions {}", partitions); - for (TopicPartition partition: partitions) { - subscriptions.resume(partition); + acquireAndEnsureOpen(); + try { + log.debug("Resuming partitions {}", partitions); + for (TopicPartition partition : partitions) { + subscriptions.resume(partition); + } + } finally { + release(); } } @@ -575,30 +858,35 @@ public Map offsetsForTimes(Map offsetsForTimes(Map timestampsToSearch, Duration timeout) { - // Keeping same argument validation error thrown by the current consumer implementation - // to avoid API level changes. - requireNonNull(timestampsToSearch, "Timestamps to search cannot be null"); - for (Map.Entry entry : timestampsToSearch.entrySet()) { - // Exclude the earliest and latest offset here so the timestamp in the returned - // OffsetAndTimestamp is always positive. - if (entry.getValue() < 0) - throw new IllegalArgumentException("The target time for partition " + entry.getKey() + " is " + + acquireAndEnsureOpen(); + try { + // Keeping same argument validation error thrown by the current consumer implementation + // to avoid API level changes. + requireNonNull(timestampsToSearch, "Timestamps to search cannot be null"); + for (Map.Entry entry : timestampsToSearch.entrySet()) { + // Exclude the earliest and latest offset here so the timestamp in the returned + // OffsetAndTimestamp is always positive. + if (entry.getValue() < 0) + throw new IllegalArgumentException("The target time for partition " + entry.getKey() + " is " + entry.getValue() + ". The target time cannot be negative."); - } + } - if (timestampsToSearch.isEmpty()) { - return Collections.emptyMap(); - } - final ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( + if (timestampsToSearch.isEmpty()) { + return Collections.emptyMap(); + } + final ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( timestampsToSearch, true); - // If timeout is set to zero return empty immediately; otherwise try to get the results - // and throw timeout exception if it cannot complete in time. - if (timeout.toMillis() == 0L) - return listOffsetsEvent.emptyResult(); + // If timeout is set to zero return empty immediately; otherwise try to get the results + // and throw timeout exception if it cannot complete in time. + if (timeout.toMillis() == 0L) + return listOffsetsEvent.emptyResult(); - return applicationEventHandler.addAndGet(listOffsetsEvent, time.timer(timeout)); + return applicationEventHandler.addAndGet(listOffsetsEvent, time.timer(timeout)); + } finally { + release(); + } } @Override @@ -624,64 +912,80 @@ public Map endOffsets(Collection partition private Map beginningOrEndOffset(Collection partitions, long timestamp, Duration timeout) { - // Keeping same argument validation error thrown by the current consumer implementation - // to avoid API level changes. - requireNonNull(partitions, "Partitions cannot be null"); + acquireAndEnsureOpen(); + try { + // Keeping same argument validation error thrown by the current consumer implementation + // to avoid API level changes. + requireNonNull(partitions, "Partitions cannot be null"); - if (partitions.isEmpty()) { - return Collections.emptyMap(); - } - Map timestampToSearch = partitions + if (partitions.isEmpty()) { + return Collections.emptyMap(); + } + Map timestampToSearch = partitions .stream() .collect(Collectors.toMap(Function.identity(), tp -> timestamp)); - ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( + ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( timestampToSearch, false); - Map offsetAndTimestampMap = applicationEventHandler.addAndGet( + Map offsetAndTimestampMap = applicationEventHandler.addAndGet( listOffsetsEvent, time.timer(timeout)); - return offsetAndTimestampMap + return offsetAndTimestampMap .entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); + } finally { + release(); + } } @Override public OptionalLong currentLag(TopicPartition topicPartition) { - final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel); - - // if the log end offset is not known and hence cannot return lag and there is - // no in-flight list offset requested yet, - // issue a list offset request for that partition so that next time - // we may get the answer; we do not need to wait for the return value - // since we would not try to poll the network client synchronously - if (lag == null) { - if (subscriptions.partitionEndOffset(topicPartition, isolationLevel) == null && + acquireAndEnsureOpen(); + try { + final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel); + + // if the log end offset is not known and hence cannot return lag and there is + // no in-flight list offset requested yet, + // issue a list offset request for that partition so that next time + // we may get the answer; we do not need to wait for the return value + // since we would not try to poll the network client synchronously + if (lag == null) { + if (subscriptions.partitionEndOffset(topicPartition, isolationLevel) == null && !subscriptions.partitionEndOffsetRequested(topicPartition)) { - log.info("Requesting the log end offset for {} in order to compute lag", topicPartition); - subscriptions.requestPartitionEndOffset(topicPartition); - endOffsets(Collections.singleton(topicPartition), Duration.ofMillis(0)); + log.info("Requesting the log end offset for {} in order to compute lag", topicPartition); + subscriptions.requestPartitionEndOffset(topicPartition); + endOffsets(Collections.singleton(topicPartition), Duration.ofMillis(0)); + } + + return OptionalLong.empty(); } - return OptionalLong.empty(); + return OptionalLong.of(lag); + } finally { + release(); } - - return OptionalLong.of(lag); } @Override public ConsumerGroupMetadata groupMetadata() { - throw new KafkaException("method not implemented"); + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + return groupMetadata.get(); + } finally { + release(); + } } @Override public void enforceRebalance() { - throw new KafkaException("method not implemented"); + log.warn("Operation not supported in new consumer group protocol"); } @Override public void enforceRebalance(String reason) { - throw new KafkaException("method not implemented"); + log.warn("Operation not supported in new consumer group protocol"); } @Override @@ -693,7 +997,7 @@ public void close() { public void close(Duration timeout) { if (timeout.toMillis() < 0) throw new IllegalArgumentException("The timeout cannot be negative."); - + acquire(); try { if (!closed) { // need to close before setting the flag since the close function @@ -702,6 +1006,7 @@ public void close(Duration timeout) { } } finally { closed = true; + release(); } } @@ -709,14 +1014,23 @@ private void close(Duration timeout, boolean swallowException) { log.trace("Closing the Kafka consumer"); AtomicReference firstException = new AtomicReference<>(); + final Timer closeTimer = time.timer(timeout); + clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(timeout.toMillis())); + closeTimer.update(); + if (applicationEventHandler != null) - closeQuietly(() -> applicationEventHandler.close(timeout), "Failed to close application event handler with a timeout(ms)=" + timeout, firstException); + closeQuietly(() -> applicationEventHandler.close(Duration.ofMillis(closeTimer.remainingMs())), "Failed to close application event handler with a timeout(ms)=" + closeTimer.remainingMs(), firstException); + + // Invoke all callbacks after the background thread exists in case if there are unsent async + // commits + maybeInvokeCommitCallbacks(); closeQuietly(fetchBuffer, "Failed to close the fetch buffer", firstException); closeQuietly(interceptors, "consumer interceptors", firstException); closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); closeQuietly(metrics, "consumer metrics", firstException); closeQuietly(deserializers, "consumer deserializers", firstException); + clientTelemetryReporter.ifPresent(reporter -> closeQuietly(reporter, "async consumer telemetry reporter", firstException)); AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); log.debug("Kafka consumer has been closed"); @@ -752,25 +1066,35 @@ public void commitSync(Map offsets) { @Override public void commitSync(Map offsets, Duration timeout) { + acquireAndEnsureOpen(); long commitStart = time.nanoseconds(); try { CompletableFuture commitFuture = commit(offsets, true); - offsets.forEach(this::updateLastSeenEpochIfNewer); ConsumerUtils.getResult(commitFuture, time.timer(timeout)); } finally { - wakeupTrigger.clearActiveTask(); + wakeupTrigger.clearTask(); kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart); + release(); } } @Override public Uuid clientInstanceId(Duration timeout) { - throw new KafkaException("method not implemented"); + if (!clientTelemetryReporter.isPresent()) { + throw new IllegalStateException("Telemetry is not enabled. Set config `" + ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG + "` to `true`."); + } + + return ClientTelemetryUtils.fetchClientInstanceId(clientTelemetryReporter.get(), timeout); } @Override public Set assignment() { - return Collections.unmodifiableSet(subscriptions.assignedPartitions()); + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(subscriptions.assignedPartitions()); + } finally { + release(); + } } /** @@ -780,46 +1104,56 @@ public Set assignment() { */ @Override public Set subscription() { - return Collections.unmodifiableSet(subscriptions.subscription()); + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(subscriptions.subscription()); + } finally { + release(); + } } @Override public void assign(Collection partitions) { - if (partitions == null) { - throw new IllegalArgumentException("Topic partitions collection to assign to cannot be null"); - } + acquireAndEnsureOpen(); + try { + if (partitions == null) { + throw new IllegalArgumentException("Topic partitions collection to assign to cannot be null"); + } - if (partitions.isEmpty()) { - unsubscribe(); - return; - } + if (partitions.isEmpty()) { + unsubscribe(); + return; + } - for (TopicPartition tp : partitions) { - String topic = (tp != null) ? tp.topic() : null; - if (isBlank(topic)) - throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); - } + for (TopicPartition tp : partitions) { + String topic = (tp != null) ? tp.topic() : null; + if (isBlank(topic)) + throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); + } - // Clear the buffered data which are not a part of newly assigned topics - final Set currentTopicPartitions = new HashSet<>(); + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); - for (TopicPartition tp : subscriptions.assignedPartitions()) { - if (partitions.contains(tp)) - currentTopicPartitions.add(tp); - } + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (partitions.contains(tp)) + currentTopicPartitions.add(tp); + } - fetchBuffer.retainAll(currentTopicPartitions); + fetchBuffer.retainAll(currentTopicPartitions); - // assignment change event will trigger autocommit if it is configured and the group id is specified. This is - // to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will - // be no following rebalance. - // - // See the ApplicationEventProcessor.process() method that handles this event for more detail. - applicationEventHandler.add(new AssignmentChangeApplicationEvent(subscriptions.allConsumed(), time.milliseconds())); + // assignment change event will trigger autocommit if it is configured and the group id is specified. This is + // to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will + // be no following rebalance. + // + // See the ApplicationEventProcessor.process() method that handles this event for more detail. + applicationEventHandler.add(new AssignmentChangeApplicationEvent(subscriptions.allConsumed(), time.milliseconds())); - log.info("Assigned to partition(s): {}", join(partitions, ", ")); - if (subscriptions.assignFromUser(new HashSet<>(partitions))) - applicationEventHandler.add(new NewTopicsMetadataUpdateRequestEvent()); + log.info("Assigned to partition(s): {}", join(partitions, ", ")); + if (subscriptions.assignFromUser(new HashSet<>(partitions))) + applicationEventHandler.add(new NewTopicsMetadataUpdateRequestEvent()); + } finally { + release(); + } } /** @@ -842,14 +1176,30 @@ private void updatePatternSubscription(Cluster cluster) { @Override public void unsubscribe() { - fetchBuffer.retainAll(Collections.emptySet()); - subscriptions.unsubscribe(); + acquireAndEnsureOpen(); + try { + fetchBuffer.retainAll(Collections.emptySet()); + if (groupMetadata.isPresent()) { + UnsubscribeApplicationEvent unsubscribeApplicationEvent = new UnsubscribeApplicationEvent(); + applicationEventHandler.add(unsubscribeApplicationEvent); + try { + unsubscribeApplicationEvent.future().get(); + log.info("Unsubscribed all topics or patterns and assigned partitions"); + } catch (InterruptedException | ExecutionException e) { + log.error("Failed while waiting for the unsubscribe event to complete", e); + } + } + subscriptions.unsubscribe(); + } finally { + release(); + } } @Override @Deprecated public ConsumerRecords poll(final long timeoutMs) { - return poll(Duration.ofMillis(timeoutMs)); + throw new UnsupportedOperationException("Consumer.poll(long) is not supported when \"group.protocol\" is \"consumer\". " + + "This method is deprecated and will be removed in the next major release."); } // Visible for testing @@ -858,7 +1208,9 @@ WakeupTrigger wakeupTrigger() { } private Fetch pollForFetches(Timer timer) { - long pollTimeout = timer.remainingMs(); + long pollTimeout = isCommittedOffsetsManagementEnabled() + ? Math.min(applicationEventHandler.maximumTimeToWait(), timer.remainingMs()) + : timer.remainingMs(); // if data is available already, return it immediately final Fetch fetch = collectFetch(); @@ -958,7 +1310,7 @@ private boolean updateFetchPositions(final Timer timer) { * according to config {@link CommonClientConfigs#GROUP_ID_CONFIG} */ private boolean isCommittedOffsetsManagementEnabled() { - return groupId.isPresent(); + return groupMetadata.isPresent(); } /** @@ -970,6 +1322,9 @@ private boolean isCommittedOffsetsManagementEnabled() { private boolean initWithCommittedOffsetsIfNeeded(Timer timer) { final Set initializingPartitions = subscriptions.initializingPartitions(); + if (initializingPartitions.isEmpty()) + return true; + log.debug("Refreshing committed offsets for partitions {}", initializingPartitions); try { final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(initializingPartitions); @@ -982,23 +1337,6 @@ private boolean initWithCommittedOffsetsIfNeeded(Timer timer) { } } - // This is here temporary as we don't have public access to the ConsumerConfig in this module. - public static Map appendDeserializerToConfig(Map configs, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { - // validate deserializer configuration, if the passed deserializer instance is null, the user must explicitly set a valid deserializer configuration value - Map newConfigs = new HashMap<>(configs); - if (keyDeserializer != null) - newConfigs.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass()); - else if (newConfigs.get(KEY_DESERIALIZER_CLASS_CONFIG) == null) - throw new ConfigException(KEY_DESERIALIZER_CLASS_CONFIG, null, "must be non-null."); - if (valueDeserializer != null) - newConfigs.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); - else if (newConfigs.get(VALUE_DESERIALIZER_CLASS_CONFIG) == null) - throw new ConfigException(VALUE_DESERIALIZER_CLASS_CONFIG, null, "must be non-null."); - return newConfigs; - } - private void throwIfNoAssignorsConfigured() { if (assignors.isEmpty()) throw new IllegalStateException("Must configure at least one partition assigner class name to " + @@ -1010,15 +1348,10 @@ private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAnd offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch)); } - private class DefaultOffsetCommitCallback implements OffsetCommitCallback { - @Override - public void onComplete(Map offsets, Exception exception) { - if (exception != null) - log.error("Offset commit with offsets {} failed", offsets, exception); - } - } - - boolean updateAssignmentMetadataIfNeeded(Timer timer) { + @Override + public boolean updateAssignmentMetadataIfNeeded(Timer timer) { + maybeInvokeCommitCallbacks(); + maybeThrowFencedInstanceException(); backgroundEventProcessor.process(); // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as @@ -1053,47 +1386,191 @@ public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { subscribeInternal(pattern, Optional.of(listener)); } + /** + * Acquire the light lock and ensure that the consumer hasn't been closed. + * + * @throws IllegalStateException If the consumer has been closed + */ + private void acquireAndEnsureOpen() { + acquire(); + if (this.closed) { + release(); + throw new IllegalStateException("This consumer has already been closed."); + } + } + + /** + * Acquire the light lock protecting this consumer from multithreaded access. Instead of blocking + * when the lock is not available, however, we just throw an exception (since multithreaded usage is not + * supported). + * + * @throws ConcurrentModificationException if another thread already has the lock + */ + private void acquire() { + final Thread thread = Thread.currentThread(); + final long threadId = thread.getId(); + if (threadId != currentThread.get() && !currentThread.compareAndSet(NO_CURRENT_THREAD, threadId)) + throw new ConcurrentModificationException("KafkaConsumer is not safe for multi-threaded access. " + + "currentThread(name: " + thread.getName() + ", id: " + threadId + ")" + + " otherThread(id: " + currentThread.get() + ")" + ); + refCount.incrementAndGet(); + } + + /** + * Release the light lock protecting the consumer from multithreaded access. + */ + private void release() { + if (refCount.decrementAndGet() == 0) + currentThread.set(NO_CURRENT_THREAD); + } + private void subscribeInternal(Pattern pattern, Optional listener) { - maybeThrowInvalidGroupIdException(); - if (pattern == null || pattern.toString().isEmpty()) - throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + if (pattern == null || pattern.toString().isEmpty()) + throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? "null" : "empty")); - - throwIfNoAssignorsConfigured(); - log.info("Subscribed to pattern: '{}'", pattern); - subscriptions.subscribe(pattern, listener); - updatePatternSubscription(metadata.fetch()); - metadata.requestUpdateForNewTopics(); + throwIfNoAssignorsConfigured(); + log.info("Subscribed to pattern: '{}'", pattern); + subscriptions.subscribe(pattern, listener); + updatePatternSubscription(metadata.fetch()); + metadata.requestUpdateForNewTopics(); + } finally { + release(); + } } private void subscribeInternal(Collection topics, Optional listener) { - maybeThrowInvalidGroupIdException(); - if (topics == null) - throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); - if (topics.isEmpty()) { - // treat subscribing to empty topic list as the same as unsubscribing - unsubscribe(); - } else { - for (String topic : topics) { - if (isBlank(topic)) - throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + if (topics == null) + throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); + if (topics.isEmpty()) { + // treat subscribing to empty topic list as the same as unsubscribing + unsubscribe(); + } else { + for (String topic : topics) { + if (isBlank(topic)) + throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); + } + + throwIfNoAssignorsConfigured(); + + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (topics.contains(tp.topic())) + currentTopicPartitions.add(tp); + } + + fetchBuffer.retainAll(currentTopicPartitions); + log.info("Subscribed to topic(s): {}", join(topics, ", ")); + if (subscriptions.subscribe(new HashSet<>(topics), listener)) + metadata.requestUpdateForNewTopics(); + + // Trigger subscribe event to effectively join the group if not already part of it, + // or just send the new subscription to the broker. + applicationEventHandler.add(new SubscriptionChangeApplicationEvent()); } + } finally { + release(); + } + } - throwIfNoAssignorsConfigured(); + @Override + public String clientId() { + return clientId; + } - // Clear the buffered data which are not a part of newly assigned topics - final Set currentTopicPartitions = new HashSet<>(); + @Override + public Metrics metricsRegistry() { + return metrics; + } - for (TopicPartition tp : subscriptions.assignedPartitions()) { - if (topics.contains(tp.topic())) - currentTopicPartitions.add(tp); + @Override + public KafkaConsumerMetrics kafkaConsumerMetrics() { + return kafkaConsumerMetrics; + } + + private void maybeThrowFencedInstanceException() { + if (isFenced) { + String groupInstanceId = "unknown"; + if (!groupMetadata.isPresent()) { + log.error("No group metadata found although a group ID was provided. This is a bug!"); + } else if (!groupMetadata.get().groupInstanceId().isPresent()) { + log.error("No group instance ID found although the consumer is fenced. This is a bug!"); + } else { + groupInstanceId = groupMetadata.get().groupInstanceId().get(); } + throw new FencedInstanceIdException("Get fenced exception for group.instance.id " + groupInstanceId); + } + } - fetchBuffer.retainAll(currentTopicPartitions); - log.info("Subscribed to topic(s): {}", join(topics, ", ")); - if (subscriptions.subscribe(new HashSet<>(topics), listener)) - metadata.requestUpdateForNewTopics(); + // Visible for testing + void maybeInvokeCommitCallbacks() { + if (callbacks() > 0) { + invoker.executeCallbacks(); } } -} \ No newline at end of file + // Visible for testing + int callbacks() { + return invoker.callbackQueue.size(); + } + + /** + * Utility class that helps the application thread to invoke user registered {@link OffsetCommitCallback}. This is + * achieved by having the background thread register a {@link OffsetCommitCallbackTask} to the invoker upon the + * future completion, and execute the callbacks when user polls/commits/closes the consumer. + */ + private class OffsetCommitCallbackInvoker { + // Thread-safe queue to store callbacks + private final BlockingQueue callbackQueue = new LinkedBlockingQueue<>(); + + public void submit(final OffsetCommitCallbackTask callback) { + try { + callbackQueue.offer(callback); + } catch (Exception e) { + log.error("Unexpected error encountered when adding offset commit callback to the invocation queue", e); + } + } + + public void executeCallbacks() { + while (!callbackQueue.isEmpty()) { + OffsetCommitCallbackTask callback = callbackQueue.poll(); + if (callback != null) { + callback.invoke(); + } + } + } + } + + private class OffsetCommitCallbackTask { + private final Map offsets; + private final Exception exception; + private final OffsetCommitCallback callback; + + public OffsetCommitCallbackTask(final OffsetCommitCallback callback, + final Map offsets, + final Exception e) { + this.offsets = offsets; + this.exception = e; + this.callback = callback; + } + + public void invoke() { + if (exception instanceof RetriableException) { + callback.onComplete(offsets, new RetriableCommitFailedException(exception)); + return; + } + + if (exception instanceof FencedInstanceIdException) + isFenced = true; + callback.onComplete(offsets, exception); + } + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index a04d3fa843718..5caee5e50378a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.RetriableCommitFailedException; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; @@ -54,8 +55,10 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; import java.util.stream.Collectors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; import static org.apache.kafka.common.protocol.Errors.COORDINATOR_LOAD_IN_PROGRESS; import static org.apache.kafka.common.protocol.Errors.COORDINATOR_NOT_AVAILABLE; @@ -64,12 +67,11 @@ public class CommitRequestManager implements RequestManager { - // TODO: current in ConsumerConfig but inaccessible in the internal package. - private static final String THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED = "internal.throw.on.fetch.stable.offset.unsupported"; private final SubscriptionState subscriptions; private final LogContext logContext; private final Logger log; private final Optional autoCommitState; + private final BackgroundEventHandler backgroundEventHandler; private final CoordinatorRequestManager coordinatorRequestManager; private final GroupState groupState; private final long retryBackoffMs; @@ -85,6 +87,7 @@ public CommitRequestManager( final SubscriptionState subscriptions, final ConsumerConfig config, final CoordinatorRequestManager coordinatorRequestManager, + final BackgroundEventHandler backgroundEventHandler, final GroupState groupState) { Objects.requireNonNull(coordinatorRequestManager, "Coordinator is needed upon committing offsets"); this.logContext = logContext; @@ -97,6 +100,7 @@ public CommitRequestManager( } else { this.autoCommitState = Optional.empty(); } + this.backgroundEventHandler = backgroundEventHandler; this.coordinatorRequestManager = coordinatorRequestManager; this.groupState = groupState; this.subscriptions = subscriptions; @@ -113,6 +117,7 @@ public CommitRequestManager( final SubscriptionState subscriptions, final ConsumerConfig config, final CoordinatorRequestManager coordinatorRequestManager, + final BackgroundEventHandler backgroundEventHandler, final GroupState groupState, final long retryBackoffMs, final long retryBackoffMaxMs, @@ -128,6 +133,7 @@ public CommitRequestManager( } else { this.autoCommitState = Optional.empty(); } + this.backgroundEventHandler = backgroundEventHandler; this.coordinatorRequestManager = coordinatorRequestManager; this.groupState = groupState; this.subscriptions = subscriptions; @@ -147,7 +153,7 @@ public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { if (!coordinatorRequestManager.coordinator().isPresent()) return EMPTY; - maybeAutoCommit(this.subscriptions.allConsumed()); + maybeAutoCommitAllConsumed(); if (!pendingRequests.hasUnsentRequests()) return EMPTY; @@ -159,6 +165,17 @@ public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { return new NetworkClientDelegate.PollResult(timeUntilNextPoll, requests); } + /** + * Returns the delay for which the application thread can safely wait before it should be responsive + * to results from the request managers. For example, the subscription state can change when heartbeats + * are sent, so blocking for longer than the heartbeat interval might mean the application thread is not + * responsive to changes. + */ + @Override + public long maximumTimeToWait(long currentTimeMs) { + return autoCommitState.map(ac -> ac.remainingMs(currentTimeMs)).orElse(Long.MAX_VALUE); + } + private static long findMinTime(final Collection requests, final long currentTimeMs) { return requests.stream() .mapToLong(request -> request.remainingBackoffMs(currentTimeMs)) @@ -166,19 +183,75 @@ private static long findMinTime(final Collection request .orElse(Long.MAX_VALUE); } - public void maybeAutoCommit(final Map offsets) { - if (!autoCommitState.isPresent()) { - return; + /** + * Generate a request to commit offsets if auto-commit is enabled. The request will be + * returned to be sent out on the next call to {@link #poll(long)}. This will only generate a + * request if there is no other commit request already in-flight, and if the commit interval + * has elapsed. + * + * @param offsets Offsets to commit + * @return Future that will complete when a response is received for the request, or a + * completed future if no request is generated. + */ + public CompletableFuture maybeAutoCommit(final Map offsets) { + if (!canAutoCommit()) { + return CompletableFuture.completedFuture(null); } AutoCommitState autocommit = autoCommitState.get(); - if (!autocommit.canSendAutocommit()) { - return; + if (!autocommit.shouldAutoCommit()) { + return CompletableFuture.completedFuture(null); } - sendAutoCommit(offsets); + CompletableFuture result = sendAutoCommit(offsets); autocommit.resetTimer(); autocommit.setInflightCommitStatus(true); + return result; + } + + /** + * If auto-commit is enabled, this will generate a commit offsets request for all assigned + * partitions and their current positions. + * + * @return Future that will complete when a response is received for the request, or a + * completed future if no request is generated. + */ + public CompletableFuture maybeAutoCommitAllConsumed() { + return maybeAutoCommit(subscriptions.allConsumed()); + } + + boolean canAutoCommit() { + return autoCommitState.isPresent() && !subscriptions.allConsumed().isEmpty(); + } + + /** + * Returns an OffsetCommitRequest of all assigned topicPartitions and their current positions. + */ + NetworkClientDelegate.UnsentRequest createCommitAllConsumedRequest() { + Map offsets = subscriptions.allConsumed(); + OffsetCommitRequestState request = pendingRequests.createOffsetCommitRequest(offsets, jitter); + log.debug("Sending synchronous auto-commit of offsets {}", offsets); + request.future.whenComplete(autoCommitCallback(subscriptions.allConsumed())); + return request.toUnsentRequest(); + } + + private CompletableFuture sendAutoCommit(final Map allConsumedOffsets) { + log.debug("Enqueuing autocommit offsets: {}", allConsumedOffsets); + return addOffsetCommitRequest(allConsumedOffsets).whenComplete(autoCommitCallback(allConsumedOffsets)); + } + + private BiConsumer autoCommitCallback(final Map allConsumedOffsets) { + return (response, throwable) -> { + autoCommitState.ifPresent(autoCommitState -> autoCommitState.setInflightCommitStatus(false)); + if (throwable == null) { + log.debug("Completed asynchronous auto-commit of offsets {}", allConsumedOffsets); + } else if (throwable instanceof RetriableCommitFailedException) { + log.debug("Asynchronous auto-commit of offsets {} failed due to retriable error: {}", + allConsumedOffsets, throwable.getMessage()); + } else { + log.warn("Asynchronous auto-commit of offsets {} failed", allConsumedOffsets, throwable); + } + }; } /** @@ -210,28 +283,40 @@ private List unsentOffsetFetchRequests() { return pendingRequests.unsentOffsetFetches; } - // Visible for testing - CompletableFuture sendAutoCommit(final Map allConsumedOffsets) { - log.debug("Enqueuing autocommit offsets: {}", allConsumedOffsets); - return addOffsetCommitRequest(allConsumedOffsets).whenComplete((response, throwable) -> { - autoCommitState.ifPresent(autoCommitState -> autoCommitState.setInflightCommitStatus(false)); - if (throwable == null) { - log.debug("Completed asynchronous auto-commit of offsets {}", allConsumedOffsets); - } else if (throwable instanceof RetriableCommitFailedException) { - log.debug("Asynchronous auto-commit of offsets {} failed due to retriable error: {}", - allConsumedOffsets, throwable.getMessage()); - } else { - log.warn("Asynchronous auto-commit of offsets {} failed", allConsumedOffsets, throwable); - } - }); - } - private void handleCoordinatorDisconnect(Throwable exception, long currentTimeMs) { if (exception instanceof DisconnectException) { coordinatorRequestManager.markCoordinatorUnknown(exception.getMessage(), currentTimeMs); } } + /** + * @return True if auto-commit is enabled as defined in the config {@link ConsumerConfig#ENABLE_AUTO_COMMIT_CONFIG} + */ + public boolean autoCommitEnabled() { + return autoCommitState.isPresent(); + } + + /** + * Reset the auto-commit timer so that the next auto-commit is sent out on the interval + * starting from now. If auto-commit is not enabled this will perform no action. + */ + public void resetAutoCommitTimer() { + autoCommitState.ifPresent(AutoCommitState::resetTimer); + } + + /** + * Drains the inflight offsetCommits during shutdown because we want to make sure all pending commits are sent + * before closing. + */ + @Override + public NetworkClientDelegate.PollResult pollOnClose() { + if (!pendingRequests.hasUnsentRequests() || !coordinatorRequestManager.coordinator().isPresent()) + return EMPTY; + + List requests = pendingRequests.drainOnClose(); + return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, requests); + } + private class OffsetCommitRequestState extends RequestState { private final Map offsets; private final String groupId; @@ -240,11 +325,11 @@ private class OffsetCommitRequestState extends RequestState { private final CompletableFuture future; OffsetCommitRequestState(final Map offsets, - final String groupId, - final String groupInstanceId, - final GroupState.Generation generation, - final long retryBackoffMs, - final long retryBackoffMaxMs) { + final String groupId, + final String groupInstanceId, + final GroupState.Generation generation, + final long retryBackoffMs, + final long retryBackoffMaxMs) { super(logContext, CommitRequestManager.class.getSimpleName(), retryBackoffMs, retryBackoffMaxMs); this.offsets = offsets; this.future = new CompletableFuture<>(); @@ -254,12 +339,12 @@ private class OffsetCommitRequestState extends RequestState { } OffsetCommitRequestState(final Map offsets, - final String groupId, - final String groupInstanceId, - final GroupState.Generation generation, - final long retryBackoffMs, - final long retryBackoffMaxMs, - final double jitter) { + final String groupId, + final String groupInstanceId, + final GroupState.Generation generation, + final long retryBackoffMs, + final long retryBackoffMaxMs, + final double jitter) { super(logContext, CommitRequestManager.class.getSimpleName(), retryBackoffMs, 2, retryBackoffMaxMs, jitter); this.offsets = offsets; this.future = new CompletableFuture<>(); @@ -296,7 +381,6 @@ public NetworkClientDelegate.UnsentRequest toUnsentRequest() { .setMemberId(generation.memberId) .setGroupInstanceId(groupInstanceId) .setTopics(new ArrayList<>(requestTopicDataMap.values()))); - NetworkClientDelegate.UnsentRequest resp = new NetworkClientDelegate.UnsentRequest( builder, coordinatorRequestManager.coordinator()); @@ -368,8 +452,8 @@ public void onResponse(final ClientResponse response) { private void handleRetriableError(Errors error, ClientResponse response) { if (error == COORDINATOR_NOT_AVAILABLE || - error == NOT_COORDINATOR || - error == REQUEST_TIMED_OUT) { + error == NOT_COORDINATOR || + error == REQUEST_TIMED_OUT) { coordinatorRequestManager.markCoordinatorUnknown(error.message(), response.receivedTimeMs()); } } @@ -417,6 +501,17 @@ public OffsetFetchRequestState(final Set partitions, this.future = new CompletableFuture<>(); } + public OffsetFetchRequestState(final Set partitions, + final GroupState.Generation generation, + final long retryBackoffMs, + final long retryBackoffMaxMs, + final double jitter) { + super(logContext, CommitRequestManager.class.getSimpleName(), retryBackoffMs, 2, retryBackoffMaxMs, jitter); + this.requestedPartitions = partitions; + this.requestedGeneration = generation; + this.future = new CompletableFuture<>(); + } + public boolean sameRequest(final OffsetFetchRequestState request) { return Objects.equals(requestedGeneration, request.requestedGeneration) && requestedPartitions.equals(request.requestedPartitions); } @@ -446,10 +541,11 @@ private void onFailure(final long currentTimeMs, final Errors responseError) { handleCoordinatorDisconnect(responseError.exception(), currentTimeMs); log.debug("Offset fetch failed: {}", responseError.message()); - // TODO: should we retry on COORDINATOR_NOT_AVAILABLE as well ? - if (responseError == COORDINATOR_LOAD_IN_PROGRESS || - responseError == Errors.NOT_COORDINATOR) { + if (responseError == COORDINATOR_LOAD_IN_PROGRESS) { + retry(currentTimeMs); + } else if (responseError == Errors.NOT_COORDINATOR) { // re-discover the coordinator and retry + coordinatorRequestManager.markCoordinatorUnknown("error response " + responseError.name(), currentTimeMs); retry(currentTimeMs); } else if (responseError == Errors.GROUP_AUTHORIZATION_FAILED) { future.completeExceptionally(GroupAuthorizationException.forGroupId(groupState.groupId)); @@ -562,23 +658,7 @@ boolean hasUnsentRequests() { OffsetCommitRequestState addOffsetCommitRequest(final Map offsets) { // TODO: Dedupe committing the same offsets to the same partitions - OffsetCommitRequestState request = jitter.isPresent() ? - new OffsetCommitRequestState( - offsets, - groupState.groupId, - groupState.groupInstanceId.orElse(null), - groupState.generation, - retryBackoffMs, - retryBackoffMaxMs, - jitter.getAsDouble()) : - new OffsetCommitRequestState( - offsets, - groupState.groupId, - groupState.groupInstanceId.orElse(null), - groupState.generation, - retryBackoffMs, - retryBackoffMaxMs); - return addOffsetCommitRequest(request); + return addOffsetCommitRequest(createOffsetCommitRequest(offsets, jitter)); } OffsetCommitRequestState addOffsetCommitRequest(OffsetCommitRequestState request) { @@ -586,6 +666,26 @@ OffsetCommitRequestState addOffsetCommitRequest(OffsetCommitRequestState request return request; } + OffsetCommitRequestState createOffsetCommitRequest(final Map offsets, + final OptionalDouble jitter) { + return jitter.isPresent() ? + new OffsetCommitRequestState( + offsets, + groupState.groupId, + groupState.groupInstanceId.orElse(null), + groupState.generation, + retryBackoffMs, + retryBackoffMaxMs, + jitter.getAsDouble()) : + new OffsetCommitRequestState( + offsets, + groupState.groupId, + groupState.groupInstanceId.orElse(null), + groupState.generation, + retryBackoffMs, + retryBackoffMaxMs); + } + /** *

Adding an offset fetch request to the outgoing buffer. If the same request was made, we chain the future * to the existing one. @@ -616,11 +716,18 @@ private CompletableFuture> addOffsetFetch } private CompletableFuture> addOffsetFetchRequest(final Set partitions) { - OffsetFetchRequestState request = new OffsetFetchRequestState( - partitions, - groupState.generation, - retryBackoffMs, - retryBackoffMaxMs); + OffsetFetchRequestState request = jitter.isPresent() ? + new OffsetFetchRequestState( + partitions, + groupState.generation, + retryBackoffMs, + retryBackoffMaxMs, + jitter.getAsDouble()) : + new OffsetFetchRequestState( + partitions, + groupState.generation, + retryBackoffMs, + retryBackoffMaxMs); return addOffsetFetchRequest(request); } @@ -659,13 +766,24 @@ List drain(final long currentTimeMs) { } // Clear the unsent offset commit and fetch lists and add all non-sendable offset fetch requests to the unsentOffsetFetches list - unsentOffsetCommits.clear(); - unsentOffsetFetches.clear(); + clearAll(); unsentOffsetFetches.addAll(partitionedBySendability.get(false)); unsentOffsetCommits.addAll(unreadyCommitRequests); return Collections.unmodifiableList(unsentRequests); } + + private void clearAll() { + unsentOffsetCommits.clear(); + unsentOffsetFetches.clear(); + } + + private List drainOnClose() { + ArrayList res = new ArrayList<>(); + res.addAll(unsentOffsetCommits.stream().map(OffsetCommitRequestState::toUnsentRequest).collect(Collectors.toList())); + clearAll(); + return res; + } } /** @@ -684,7 +802,7 @@ public AutoCommitState( this.hasInflightCommit = false; } - public boolean canSendAutocommit() { + public boolean shouldAutoCommit() { return !this.hasInflightCommit && this.timer.isExpired(); } @@ -692,6 +810,11 @@ public void resetTimer() { this.timer.reset(autoCommitInterval); } + public long remainingMs(final long currentTimeMs) { + this.timer.update(currentTimeMs); + return this.timer.remainingMs(); + } + public void ack(final long currentTimeMs) { this.timer.update(currentTimeMs); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 590c8a0976ac7..e3ce79e373f42 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -56,6 +56,7 @@ import org.apache.kafka.common.requests.OffsetCommitResponse; import org.apache.kafka.common.requests.OffsetFetchRequest; import org.apache.kafka.common.requests.OffsetFetchResponse; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; @@ -168,13 +169,15 @@ public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig, int autoCommitIntervalMs, ConsumerInterceptors interceptors, boolean throwOnFetchStableOffsetsUnsupported, - String rackId) { + String rackId, + Optional clientTelemetryReporter) { super(rebalanceConfig, logContext, client, metrics, metricGrpPrefix, - time); + time, + clientTelemetryReporter); this.rebalanceConfig = rebalanceConfig; this.log = logContext.logger(ConsumerCoordinator.class); this.metadata = metadata; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java new file mode 100644 index 0000000000000..612827ebe83df --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegate.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.utils.Timer; + +/** + * This extension interface provides a handful of methods to expose internals of the {@link Consumer} for + * various tests. + * + *

+ * + * Note: this is for internal use only and is not intended for use by end users. Internal users should + * not attempt to determine the underlying implementation to avoid coding to an unstable interface. Rather, it is + * the {@link Consumer} API contract that should serve as the caller's interface. + */ +public interface ConsumerDelegate extends Consumer { + + String clientId(); + + Metrics metricsRegistry(); + + KafkaConsumerMetrics kafkaConsumerMetrics(); + + boolean updateAssignmentMetadataIfNeeded(final Timer timer); +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegateCreator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegateCreator.java new file mode 100644 index 0000000000000..bd95e06c86448 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerDelegateCreator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; +import org.apache.kafka.clients.consumer.GroupProtocol; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; + +import java.util.List; +import java.util.Locale; + +/** + * {@code ConsumerDelegateCreator} implements a quasi-factory pattern to allow the caller to remain unaware of the + * underlying {@link Consumer} implementation that is created. This provides the means by which {@link KafkaConsumer} + * can remain the top-level facade for implementations, but allow different implementations to co-exist under + * the covers. + * + *

+ * + * The current logic for the {@code ConsumerCreator} inspects the incoming configuration and determines if + * it is using the new consumer group protocol (KIP-848) or if it should fall back to the existing, legacy group + * protocol. This is based on the presence and value of the {@link ConsumerConfig#GROUP_PROTOCOL_CONFIG group.protocol} + * configuration. If the value is present and equal to "{@code consumer}", the {@link AsyncKafkaConsumer} + * will be returned. Otherwise, the {@link LegacyKafkaConsumer} will be returned. + * + * + *

+ * + * Note: this is for internal use only and is not intended for use by end users. Internal users should + * not attempt to determine the underlying implementation to avoid coding to an unstable interface. Rather, it is + * the {@link Consumer} API contract that should serve as the caller's interface. + */ +public class ConsumerDelegateCreator { + + public ConsumerDelegate create(ConsumerConfig config, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + try { + GroupProtocol groupProtocol = GroupProtocol.valueOf(config.getString(ConsumerConfig.GROUP_PROTOCOL_CONFIG).toUpperCase(Locale.ROOT)); + + if (groupProtocol == GroupProtocol.CONSUMER) + return new AsyncKafkaConsumer<>(config, keyDeserializer, valueDeserializer); + else + return new LegacyKafkaConsumer<>(config, keyDeserializer, valueDeserializer); + } catch (KafkaException e) { + throw e; + } catch (Throwable t) { + throw new KafkaException("Failed to construct Kafka consumer", t); + } + } + + public ConsumerDelegate create(LogContext logContext, + Time time, + ConsumerConfig config, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + KafkaClient client, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + List assignors) { + try { + GroupProtocol groupProtocol = GroupProtocol.valueOf(config.getString(ConsumerConfig.GROUP_PROTOCOL_CONFIG).toUpperCase(Locale.ROOT)); + + if (groupProtocol == GroupProtocol.CONSUMER) + return new AsyncKafkaConsumer<>( + logContext, + time, + config, + keyDeserializer, + valueDeserializer, + client, + subscriptions, + metadata, + assignors + ); + else + return new LegacyKafkaConsumer<>( + logContext, + time, + config, + keyDeserializer, + valueDeserializer, + client, + subscriptions, + metadata, + assignors + ); + } catch (KafkaException e) { + throw e; + } catch (Throwable t) { + throw new KafkaException("Failed to construct Kafka consumer", t); + } + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index 77a2952d1b2f9..07bb9811b5b6f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -16,12 +16,12 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.Node; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.utils.KafkaThread; @@ -33,9 +33,11 @@ import java.io.Closeable; import java.time.Duration; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -49,7 +51,8 @@ */ public class ConsumerNetworkThread extends KafkaThread implements Closeable { - private static final long MAX_POLL_TIMEOUT_MS = 5000; + // visible for testing + static final long MAX_POLL_TIMEOUT_MS = 5000; private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread"; private final Time time; private final Logger log; @@ -62,6 +65,7 @@ public class ConsumerNetworkThread extends KafkaThread implements Closeable { private volatile boolean running; private final IdempotentCloser closer = new IdempotentCloser(); private volatile Duration closeTimeout = Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS); + private volatile long cachedMaximumTimeToWait = MAX_POLL_TIMEOUT_MS; public ConsumerNetworkThread(LogContext logContext, Time time, @@ -74,13 +78,11 @@ public ConsumerNetworkThread(LogContext logContext, this.applicationEventProcessorSupplier = applicationEventProcessorSupplier; this.networkClientDelegateSupplier = networkClientDelegateSupplier; this.requestManagersSupplier = requestManagersSupplier; + this.running = true; } @Override public void run() { - closer.assertOpen("Consumer network thread is already closed"); - running = true; - try { log.debug("Consumer network thread started"); @@ -90,14 +92,11 @@ public void run() { while (running) { try { runOnce(); - } catch (final WakeupException e) { - log.debug("WakeupException caught, consumer network thread won't be interrupted"); - // swallow the wakeup exception to prevent killing the thread. + } catch (final Throwable e) { + // Swallow the exception and continue + log.error("Unexpected error caught in consumer network thread", e); } } - } catch (final Throwable t) { - log.error("The consumer network thread failed due to unexpected error", t); - throw new KafkaException(t); } finally { cleanup(); } @@ -144,6 +143,12 @@ void runOnce() { .map(networkClientDelegate::addAll) .reduce(MAX_POLL_TIMEOUT_MS, Math::min); networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs); + + cachedMaximumTimeToWait = requestManagers.entries().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .map(rm -> rm.maximumTimeToWait(currentTimeMs)) + .reduce(Long.MAX_VALUE, Math::min); } /** @@ -181,9 +186,6 @@ static void runAtClose(final Collection> requ long pollWaitTimeMs = pollResults.stream() .map(networkClientDelegate::addAll) .reduce(MAX_POLL_TIMEOUT_MS, Math::min); - pollWaitTimeMs = Math.min(pollWaitTimeMs, timer.remainingMs()); - networkClientDelegate.poll(pollWaitTimeMs, timer.currentTimeMs()); - timer.update(); List> requestFutures = pollResults.stream() .flatMap(fads -> fads.unsentRequests.stream()) @@ -192,10 +194,11 @@ static void runAtClose(final Collection> requ // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until // all requests have received a response. - while (timer.notExpired() && !requestFutures.stream().allMatch(Future::isDone)) { - networkClientDelegate.poll(timer.remainingMs(), timer.currentTimeMs()); + do { + pollWaitTimeMs = Math.min(pollWaitTimeMs, timer.remainingMs()); + networkClientDelegate.poll(pollWaitTimeMs, timer.currentTimeMs()); timer.update(); - } + } while (timer.notExpired() && !requestFutures.stream().allMatch(Future::isDone)); } public boolean isRunning() { @@ -208,6 +211,22 @@ public void wakeup() { networkClientDelegate.wakeup(); } + /** + * Returns the delay for which the application thread can safely wait before it should be responsive + * to results from the request managers. For example, the subscription state can change when heartbeats + * are sent, so blocking for longer than the heartbeat interval might mean the application thread is not + * responsive to changes. + * + * Because this method is called by the application thread, it's not allowed to access the request managers + * that actually provide the information. As a result, the consumer network thread periodically caches the + * information from the request managers and this can then be read safely using this method. + * + * @return The maximum delay in milliseconds + */ + public long maximumTimeToWait() { + return cachedMaximumTimeToWait; + } + @Override public void close() { close(closeTimeout); @@ -247,22 +266,87 @@ private void closeInternal(final Duration timeout) { closeTimeout = timeout; wakeup(); - if (timeoutMs > 0) { - try { - join(timeoutMs); - } catch (InterruptedException e) { - log.error("Interrupted while waiting for consumer network thread to complete", e); - } + try { + join(); + } catch (InterruptedException e) { + log.error("Interrupted while waiting for consumer network thread to complete", e); } } void cleanup() { - log.trace("Closing the consumer network thread"); - Timer timer = time.timer(closeTimeout); - runAtClose(requestManagers.entries(), networkClientDelegate, timer); - closeQuietly(requestManagers, "request managers"); - closeQuietly(networkClientDelegate, "network client delegate"); - closeQuietly(applicationEventProcessor, "application event processor"); - log.debug("Closed the consumer network thread"); + try { + log.trace("Closing the consumer network thread"); + Timer timer = time.timer(closeTimeout); + maybeAutocommitOnClose(timer); + runAtClose(requestManagers.entries(), networkClientDelegate, timer); + maybeLeaveGroup(timer); + } catch (Exception e) { + log.error("Unexpected error during shutdown. Proceed with closing.", e); + } finally { + closeQuietly(requestManagers, "request managers"); + closeQuietly(networkClientDelegate, "network client delegate"); + closeQuietly(applicationEventProcessor, "application event processor"); + log.debug("Closed the consumer network thread"); + } + } + + /** + * We need to autocommit before shutting down the consumer. The method needs to first connect to the coordinator + * node to construct the closing requests. Then wait for all closing requests to finish before returning. The + * method is bounded by a closing timer. We will continue closing down the consumer if the requests cannot be + * completed in time. + */ + // Visible for testing + void maybeAutocommitOnClose(final Timer timer) { + if (!requestManagers.coordinatorRequestManager.isPresent()) + return; + + if (!requestManagers.commitRequestManager.isPresent()) { + log.error("Expecting a CommitRequestManager but the object was never initialized. Shutting down."); + return; + } + + if (!requestManagers.commitRequestManager.get().canAutoCommit()) { + return; + } + + ensureCoordinatorReady(timer); + NetworkClientDelegate.UnsentRequest autocommitRequest = + requestManagers.commitRequestManager.get().createCommitAllConsumedRequest(); + networkClientDelegate.add(autocommitRequest); + do { + long currentTimeMs = timer.currentTimeMs(); + ensureCoordinatorReady(timer); + networkClientDelegate.poll(timer.remainingMs(), currentTimeMs); + } while (timer.notExpired() && !autocommitRequest.future().isDone()); + } + + void maybeLeaveGroup(final Timer timer) { + // TODO: Leave group upon closing the consumer + } + + private void ensureCoordinatorReady(final Timer timer) { + while (!coordinatorReady() && timer.notExpired()) { + findCoordinatorSync(timer); + } + } + + private boolean coordinatorReady() { + CoordinatorRequestManager coordinatorRequestManager = requestManagers.coordinatorRequestManager.orElseThrow( + () -> new IllegalStateException("CoordinatorRequestManager uninitialized.")); + Optional coordinator = coordinatorRequestManager.coordinator(); + return coordinator.isPresent() && !networkClientDelegate.isUnavailable(coordinator.get()); + } + + private void findCoordinatorSync(final Timer timer) { + CoordinatorRequestManager coordinatorRequestManager = requestManagers.coordinatorRequestManager.orElseThrow( + () -> new IllegalStateException("CoordinatorRequestManager uninitialized.")); + NetworkClientDelegate.UnsentRequest request = coordinatorRequestManager.makeFindCoordinatorRequest(timer.currentTimeMs()); + networkClientDelegate.addAll(Collections.singletonList(request)); + CompletableFuture findCoordinatorRequest = request.future(); + while (timer.notExpired() && !findCoordinatorRequest.isDone()) { + networkClientDelegate.poll(timer.remainingMs(), timer.currentTimeMs()); + timer.update(); + } } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 92b098213b061..d599d41a245ee 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -38,6 +38,7 @@ import org.apache.kafka.common.metrics.MetricsContext; import org.apache.kafka.common.metrics.MetricsReporter; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; @@ -56,6 +57,12 @@ public final class ConsumerUtils { + /** + * This configuration has only package-level visibility in {@link ConsumerConfig}, so it's inaccessible in the + * internals package where most of its uses live. Attempts were made to move things around, but it was deemed + * better to leave it as is. + */ + static final String THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED = "internal.throw.on.fetch.stable.offset.unsupported"; public static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; public static final String CONSUMER_JMX_PREFIX = "kafka.consumer"; public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; @@ -75,7 +82,8 @@ public static ConsumerNetworkClient createConsumerNetworkClient(ConsumerConfig c Time time, Metadata metadata, Sensor throttleTimeSensor, - long retryBackoffMs) { + long retryBackoffMs, + ClientTelemetrySender clientTelemetrySender) { NetworkClient netClient = ClientUtils.createNetworkClient(config, metrics, CONSUMER_METRIC_GROUP_PREFIX, @@ -84,7 +92,8 @@ public static ConsumerNetworkClient createConsumerNetworkClient(ConsumerConfig c time, CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, metadata, - throttleTimeSensor); + throttleTimeSensor, + clientTelemetrySender); // Will avoid blocking an extended period of time to prevent heartbeat thread starvation int heartbeatIntervalMs = config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG); @@ -124,6 +133,11 @@ public static SubscriptionState createSubscriptionState(ConsumerConfig config, L } public static Metrics createMetrics(ConsumerConfig config, Time time) { + return createMetrics(config, time, CommonClientConfigs.metricsReporters( + config.getString(ConsumerConfig.CLIENT_ID_CONFIG), config)); + } + + public static Metrics createMetrics(ConsumerConfig config, Time time, List reporters) { String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); Map metricsTags = Collections.singletonMap(CONSUMER_CLIENT_ID_METRIC_TAG, clientId); MetricConfig metricConfig = new MetricConfig() @@ -131,7 +145,6 @@ public static Metrics createMetrics(ConsumerConfig config, Time time) { .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) .recordLevel(Sensor.RecordingLevel.forName(config.getString(ConsumerConfig.METRICS_RECORDING_LEVEL_CONFIG))) .tags(metricsTags); - List reporters = CommonClientConfigs.metricsReporters(clientId, config); MetricsContext metricsContext = new KafkaMetricsContext(CONSUMER_JMX_PREFIX, config.originalsWithPrefix(CommonClientConfigs.METRICS_CONTEXT_PREFIX)); return new Metrics(metricConfig, reporters, time, metricsContext); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java index 78f6b3d5e2752..d6a72812a52f9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java @@ -105,7 +105,7 @@ public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { return new NetworkClientDelegate.PollResult(coordinatorRequestState.remainingBackoffMs(currentTimeMs)); } - private NetworkClientDelegate.UnsentRequest makeFindCoordinatorRequest(final long currentTimeMs) { + NetworkClientDelegate.UnsentRequest makeFindCoordinatorRequest(final long currentTimeMs) { coordinatorRequestState.onSendAttempt(currentTimeMs); FindCoordinatorRequestData data = new FindCoordinatorRequestData() .setKeyType(FindCoordinatorRequest.CoordinatorType.GROUP.id()) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java index de7f88ab72596..d9e365e09e4b1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -52,6 +53,8 @@ public class FetchBuffer implements AutoCloseable { private final Condition notEmptyCondition; private final IdempotentCloser idempotentCloser = new IdempotentCloser(); + private final AtomicBoolean wokenup = new AtomicBoolean(false); + private CompletedFetch nextInLineFetch; public FetchBuffer(final LogContext logContext) { @@ -166,7 +169,7 @@ void awaitNotEmpty(Timer timer) { try { lock.lock(); - while (isEmpty()) { + while (isEmpty() && !wokenup.compareAndSet(true, false)) { // Update the timer before we head into the loop in case it took a while to get the lock. timer.update(); @@ -185,6 +188,16 @@ void awaitNotEmpty(Timer timer) { } } + void wakeup() { + wokenup.set(true); + try { + lock.lock(); + notEmptyCondition.signalAll(); + } finally { + lock.unlock(); + } + } + /** * Updates the buffer to retain only the fetch data that corresponds to the given partitions. Any previously * {@link CompletedFetch fetched data} is removed if its partition is not in the given set of partitions. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java index e98441321d399..692a67b9c301e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java @@ -131,7 +131,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { pausedCompletedFetches.add(nextInLineFetch); fetchBuffer.setNextInLineFetch(null); } else { - final Fetch nextFetch = fetchRecords(nextInLineFetch); + final Fetch nextFetch = fetchRecords(nextInLineFetch, recordsRemaining); recordsRemaining -= nextFetch.numRecords(); fetch.add(nextFetch); } @@ -148,7 +148,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { return fetch; } - private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { + private Fetch fetchRecords(final CompletedFetch nextInLineFetch, int maxRecords) { final TopicPartition tp = nextInLineFetch.partition; if (!subscriptions.isAssigned(tp)) { @@ -167,7 +167,7 @@ private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { if (nextInLineFetch.nextFetchOffset() == position.offset) { List> partRecords = nextInLineFetch.fetchRecords(fetchConfig, deserializers, - fetchConfig.maxPollRecords); + maxRecords); log.trace("Returning {} fetched records at offset {} for assigned partition {}", partRecords.size(), position, tp); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index a0a4ca97e16ca..3632098a09b58 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -18,8 +18,12 @@ import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; @@ -33,6 +37,12 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; /** *

Manages the request creation and response handling for the heartbeat. The module creates a @@ -41,7 +51,7 @@ * {@link MembershipManager} and handle any errors.

* *

The manager will try to send a heartbeat when the member is in {@link MemberState#STABLE}, - * {@link MemberState#UNJOINED}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable + * {@link MemberState#JOINING}, or {@link MemberState#RECONCILING}. Which mean the member is either in a stable * group, is trying to join a group, or is in the process of reconciling the assignment changes.

* *

If the member got kick out of a group, it will try to give up the current assignment by invoking {@code @@ -62,7 +72,6 @@ public class HeartbeatRequestManager implements RequestManager { private final Logger logger; - private final Time time; /** * Time that the group coordinator will wait on member to revoke its partitions. This is provided by the group @@ -75,16 +84,16 @@ public class HeartbeatRequestManager implements RequestManager { */ private final CoordinatorRequestManager coordinatorRequestManager; - /** - * SubscriptionState tracks the topic, partition and offset of the member - */ - private final SubscriptionState subscriptions; - /** * HeartbeatRequestState manages heartbeat request timing and retries */ private final HeartbeatRequestState heartbeatRequestState; + /* + * HeartbeatState manages building the heartbeat requests correctly + */ + private final HeartbeatState heartbeatState; + /** * MembershipManager manages member's essential attributes like epoch and id, and its rebalance state */ @@ -95,6 +104,8 @@ public class HeartbeatRequestManager implements RequestManager { */ private final BackgroundEventHandler backgroundEventHandler; + private GroupMetadataUpdateEvent previousGroupMetadataUpdateEvent = null; + public HeartbeatRequestManager( final LogContext logContext, final Time time, @@ -104,14 +115,13 @@ public HeartbeatRequestManager( final MembershipManager membershipManager, final BackgroundEventHandler backgroundEventHandler) { this.coordinatorRequestManager = coordinatorRequestManager; - this.time = time; this.logger = logContext.logger(getClass()); - this.subscriptions = subscriptions; this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.heartbeatState = new HeartbeatState(subscriptions, membershipManager, rebalanceTimeoutMs); this.heartbeatRequestState = new HeartbeatRequestState(logContext, time, 0, retryBackoffMs, retryBackoffMaxMs, rebalanceTimeoutMs); } @@ -119,69 +129,81 @@ public HeartbeatRequestManager( // Visible for testing HeartbeatRequestManager( final LogContext logContext, - final Time time, final ConsumerConfig config, final CoordinatorRequestManager coordinatorRequestManager, - final SubscriptionState subscriptions, final MembershipManager membershipManager, + final HeartbeatState heartbeatState, final HeartbeatRequestState heartbeatRequestState, final BackgroundEventHandler backgroundEventHandler) { this.logger = logContext.logger(this.getClass()); - this.time = time; - this.subscriptions = subscriptions; this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG); this.coordinatorRequestManager = coordinatorRequestManager; this.heartbeatRequestState = heartbeatRequestState; + this.heartbeatState = heartbeatState; this.membershipManager = membershipManager; this.backgroundEventHandler = backgroundEventHandler; } /** - * Determines the maximum wait time until the next poll based on the member's state, and creates a heartbeat - * request. + * This will build a heartbeat request if one must be sent, determined based on the member + * state. A heartbeat is sent in the following situations: *

    - *
  1. If the member is without a coordinator or is in a failed state, the timer is set to Long.MAX_VALUE, as there's no need to send a heartbeat.
    2. - *
  2. If the member cannot send a heartbeat due to either exponential backoff, it will return the remaining time left on the backoff timer.
    3. - *
  3. If the member's heartbeat timer has not expired, It will return the remaining time left on the - * heartbeat timer.
    4. + *
  4. Member is part of the consumer group or wants to join it.
    5. + *
  5. The heartbeat interval has expired, or the member is in a state that indicates + * that it should heartbeat without waiting for the interval.
    6. + *
+ * This will also determine the maximum wait time until the next poll based on the member's + * state. + *
    + *
  1. If the member is without a coordinator or is in a failed state, the timer is set + * to Long.MAX_VALUE, as there's no need to send a heartbeat.
    2. + *
  2. If the member cannot send a heartbeat due to either exponential backoff, it will + * return the remaining time left on the backoff timer.
    3. + *
  3. If the member's heartbeat timer has not expired, It will return the remaining time + * left on the heartbeat timer.
    4. *
  4. If the member can send a heartbeat, the timer is set to the current heartbeat interval.
    5. *
+ * + * @return {@link PollResult} that includes a heartbeat request if one must be sent, and the + * time to wait until the next poll. */ @Override public NetworkClientDelegate.PollResult poll(long currentTimeMs) { - if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) + if (!coordinatorRequestManager.coordinator().isPresent() || membershipManager.shouldSkipHeartbeat()) { + membershipManager.onHeartbeatRequestSkipped(); return NetworkClientDelegate.PollResult.EMPTY; + } - // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be - // implemented either with or after the partition reconciliation logic. - if (!heartbeatRequestState.canSendRequest(currentTimeMs)) + boolean heartbeatNow = membershipManager.shouldHeartbeatNow() && !heartbeatRequestState.requestInFlight(); + + if (!heartbeatRequestState.canSendRequest(currentTimeMs) && !heartbeatNow) { return new NetworkClientDelegate.PollResult(heartbeatRequestState.nextHeartbeatMs(currentTimeMs)); + } - this.heartbeatRequestState.onSendAttempt(currentTimeMs); + heartbeatRequestState.onSendAttempt(currentTimeMs); + membershipManager.onHeartbeatRequestSent(); NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); } - private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { - // TODO: We only need to send the rebalanceTimeoutMs field once unless the first request failed. - ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData() - .setGroupId(membershipManager.groupId()) - .setMemberEpoch(membershipManager.memberEpoch()) - .setMemberId(membershipManager.memberId()) - .setRebalanceTimeoutMs(rebalanceTimeoutMs); - - membershipManager.groupInstanceId().ifPresent(data::setInstanceId); - - if (this.subscriptions.hasPatternSubscription()) { - // TODO: Pass the string to the GC if server side regex is used. - } else { - data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); - } - - this.membershipManager.serverAssignor().ifPresent(data::setServerAssignor); + /** + * Returns the delay for which the application thread can safely wait before it should be responsive + * to results from the request managers. For example, the subscription state can change when heartbeats + * are sent, so blocking for longer than the heartbeat interval might mean the application thread is not + * responsive to changes. + * + *

In the event that heartbeats are currently being skipped, this still returns the next heartbeat + * delay rather than {@code Long.MAX_VALUE} so that the application thread remains responsive. + */ + @Override + public long maximumTimeToWait(long currentTimeMs) { + boolean heartbeatNow = membershipManager.shouldHeartbeatNow() && !heartbeatRequestState.requestInFlight(); + return heartbeatNow ? 0L : heartbeatRequestState.nextHeartbeatMs(currentTimeMs); + } + private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { NetworkClientDelegate.UnsentRequest request = new NetworkClientDelegate.UnsentRequest( - new ConsumerGroupHeartbeatRequest.Builder(data), + new ConsumerGroupHeartbeatRequest.Builder(this.heartbeatState.buildRequestData()), coordinatorRequestManager.coordinator()); return request.whenComplete((response, exception) -> { if (response != null) { @@ -194,6 +216,7 @@ private NetworkClientDelegate.UnsentRequest makeHeartbeatRequest() { private void onFailure(final Throwable exception, final long responseTimeMs) { this.heartbeatRequestState.onFailedAttempt(responseTimeMs); + this.heartbeatState.reset(); if (exception instanceof RetriableException) { String message = String.format("GroupHeartbeatRequest failed because of the retriable exception. " + "Will retry in %s ms: %s", @@ -211,17 +234,36 @@ private void onResponse(final ConsumerGroupHeartbeatResponse response, long curr this.heartbeatRequestState.updateHeartbeatIntervalMs(response.data().heartbeatIntervalMs()); this.heartbeatRequestState.onSuccessfulAttempt(currentTimeMs); this.heartbeatRequestState.resetTimer(); - this.membershipManager.updateState(response.data()); + this.membershipManager.onHeartbeatResponseReceived(response.data()); + maybeSendGroupMetadataUpdateEvent(); return; } onErrorResponse(response, currentTimeMs); } + private void maybeSendGroupMetadataUpdateEvent() { + if (previousGroupMetadataUpdateEvent == null || + !previousGroupMetadataUpdateEvent.memberId().equals(membershipManager.memberId()) || + previousGroupMetadataUpdateEvent.memberEpoch() != membershipManager.memberEpoch()) { + + final GroupMetadataUpdateEvent currentGroupMetadataUpdateEvent = new GroupMetadataUpdateEvent( + membershipManager.memberEpoch(), + previousGroupMetadataUpdateEvent != null && membershipManager.memberId() == null ? + previousGroupMetadataUpdateEvent.memberId() : membershipManager.memberId() + ); + this.backgroundEventHandler.add(currentGroupMetadataUpdateEvent); + previousGroupMetadataUpdateEvent = currentGroupMetadataUpdateEvent; + } + } + private void onErrorResponse(final ConsumerGroupHeartbeatResponse response, final long currentTimeMs) { Errors error = Errors.forCode(response.data().errorCode()); String errorMessage = response.data().errorMessage(); String message; + + this.heartbeatState.reset(); + // TODO: upon encountering a fatal/fenced error, trigger onPartitionLost logic to give up the current // assignments. switch (error) { @@ -307,7 +349,7 @@ private void logInfo(final String message, private void handleFatalFailure(Throwable error) { backgroundEventHandler.add(new ErrorBackgroundEvent(error)); - membershipManager.transitionToFailed(); + membershipManager.transitionToFatal(); } /** @@ -326,17 +368,6 @@ static class HeartbeatRequestState extends RequestState { */ private long heartbeatIntervalMs; - public HeartbeatRequestState( - final LogContext logContext, - final Time time, - final long heartbeatIntervalMs, - final long retryBackoffMs, - final long retryBackoffMaxMs) { - super(logContext, HeartbeatRequestState.class.getName(), retryBackoffMs, retryBackoffMaxMs); - this.heartbeatIntervalMs = heartbeatIntervalMs; - this.heartbeatTimer = time.timer(heartbeatIntervalMs); - } - public HeartbeatRequestState( final LogContext logContext, final Time time, @@ -379,4 +410,133 @@ private void updateHeartbeatIntervalMs(final long heartbeatIntervalMs) { this.heartbeatTimer.updateAndReset(heartbeatIntervalMs); } } + + /** + * Builds the heartbeat requests correctly, ensuring that all information is sent according to + * the protocol, but subsequent requests do not send information which has not changed. This + * is important to ensure that reconciliation completes successfully. + */ + static class HeartbeatState { + private final SubscriptionState subscriptions; + private final MembershipManager membershipManager; + private final int rebalanceTimeoutMs; + private final SentFields sentFields; + + public HeartbeatState( + final SubscriptionState subscriptions, + final MembershipManager membershipManager, + final int rebalanceTimeoutMs) { + this.subscriptions = subscriptions; + this.membershipManager = membershipManager; + this.rebalanceTimeoutMs = rebalanceTimeoutMs; + this.sentFields = new SentFields(); + } + + + public void reset() { + sentFields.reset(); + } + + public ConsumerGroupHeartbeatRequestData buildRequestData() { + ConsumerGroupHeartbeatRequestData data = new ConsumerGroupHeartbeatRequestData(); + + // GroupId - always sent + data.setGroupId(membershipManager.groupId()); + + // MemberId - always sent, empty until it has been received from the coordinator + data.setMemberId(membershipManager.memberId()); + + // MemberEpoch - always sent + data.setMemberEpoch(membershipManager.memberEpoch()); + + // InstanceId - only sent if has changed since the last heartbeat + membershipManager.groupInstanceId().ifPresent(groupInstanceId -> { + if (!groupInstanceId.equals(sentFields.instanceId)) { + data.setInstanceId(groupInstanceId); + sentFields.instanceId = groupInstanceId; + } + }); + + // RebalanceTimeoutMs - only sent if has changed since the last heartbeat + if (sentFields.rebalanceTimeoutMs != rebalanceTimeoutMs) { + data.setRebalanceTimeoutMs(rebalanceTimeoutMs); + sentFields.rebalanceTimeoutMs = rebalanceTimeoutMs; + } + + if (!this.subscriptions.hasPatternSubscription()) { + // SubscribedTopicNames - only sent if has changed since the last heartbeat + TreeSet subscribedTopicNames = new TreeSet<>(this.subscriptions.subscription()); + if (!subscribedTopicNames.equals(sentFields.subscribedTopicNames)) { + data.setSubscribedTopicNames(new ArrayList<>(this.subscriptions.subscription())); + sentFields.subscribedTopicNames = subscribedTopicNames; + } + } else { + // SubscribedTopicRegex - only sent if has changed since the last heartbeat + // - not supported yet + } + + // ServerAssignor - only sent if has changed since the last heartbeat + this.membershipManager.serverAssignor().ifPresent(serverAssignor -> { + if (!serverAssignor.equals(sentFields.serverAssignor)) { + data.setServerAssignor(serverAssignor); + sentFields.serverAssignor = serverAssignor; + } + }); + + // ClientAssignors - not supported yet + + // TopicPartitions - only sent if has changed since the last heartbeat + // Note that TopicIdPartition.toString is being avoided here so that + // the string consists of just the topic ID and the partition. + // When an assignment is received, we might not yet know the topic name + // and then it is learnt subsequently by a metadata update. + TreeSet assignedPartitions = membershipManager.currentAssignment().stream() + .map(tp -> tp.topicId() + "-" + tp.partition()) + .collect(Collectors.toCollection(TreeSet::new)); + if (!assignedPartitions.equals(sentFields.topicPartitions)) { + List topicPartitions = + buildTopicPartitionsList(membershipManager.currentAssignment()); + data.setTopicPartitions(topicPartitions); + sentFields.topicPartitions = assignedPartitions; + } + + return data; + } + + private List buildTopicPartitionsList(Set topicIdPartitions) { + List result = new ArrayList<>(); + Map> partitionsPerTopicId = new HashMap<>(); + for (TopicIdPartition topicIdPartition : topicIdPartitions) { + Uuid topicId = topicIdPartition.topicId(); + partitionsPerTopicId.computeIfAbsent(topicId, __ -> new ArrayList<>()).add(topicIdPartition.partition()); + } + for (Map.Entry> entry : partitionsPerTopicId.entrySet()) { + Uuid topicId = entry.getKey(); + List partitions = entry.getValue(); + result.add(new ConsumerGroupHeartbeatRequestData.TopicPartitions() + .setTopicId(topicId) + .setPartitions(partitions)); + } + return result; + } + + // Fields of ConsumerHeartbeatRequest sent in the most recent request + static class SentFields { + private String instanceId = null; + private int rebalanceTimeoutMs = -1; + private TreeSet subscribedTopicNames = null; + private String serverAssignor = null; + private TreeSet topicPartitions = null; + + SentFields() {} + + void reset() { + instanceId = null; + rebalanceTimeoutMs = -1; + subscribedTopicNames = null; + serverAssignor = null; + topicPartitions = null; + } + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java new file mode 100644 index 0000000000000..563b437cba6cd --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/LegacyKafkaConsumer.java @@ -0,0 +1,1276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.ClientUtils; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; +import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.GroupProtocol; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.NoOffsetForPartitionException; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.InvalidGroupIdException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryUtils; +import org.apache.kafka.common.utils.AppInfoParser; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; +import org.slf4j.event.Level; + +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_RACK_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createConsumerNetworkClient; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; +import static org.apache.kafka.common.utils.Utils.closeQuietly; +import static org.apache.kafka.common.utils.Utils.isBlank; +import static org.apache.kafka.common.utils.Utils.join; +import static org.apache.kafka.common.utils.Utils.swallow; + +/** + * A client that consumes records from a Kafka cluster using the {@link GroupProtocol#GENERIC generic group protocol}. + * In this implementation, all network I/O happens in the thread of the application making the call. + * + *

+ * + * Note: per its name, this implementation is left for backward compatibility purposes. The updated consumer + * group protocol (from KIP-848) introduces allows users continue using the legacy "generic" group protocol. + * This class should not be invoked directly; users should instead create a {@link KafkaConsumer} as before. + */ +public class LegacyKafkaConsumer implements ConsumerDelegate { + + private static final long NO_CURRENT_THREAD = -1L; + public static final String DEFAULT_REASON = "rebalance enforced by user"; + + private final Metrics metrics; + private final KafkaConsumerMetrics kafkaConsumerMetrics; + private Logger log; + private final String clientId; + private final Optional groupId; + private final ConsumerCoordinator coordinator; + private final Deserializers deserializers; + private final Fetcher fetcher; + private final OffsetFetcher offsetFetcher; + private final TopicMetadataFetcher topicMetadataFetcher; + private final ConsumerInterceptors interceptors; + private final IsolationLevel isolationLevel; + + private final Time time; + private final ConsumerNetworkClient client; + private final SubscriptionState subscriptions; + private final ConsumerMetadata metadata; + private final long retryBackoffMs; + private final long retryBackoffMaxMs; + private final int requestTimeoutMs; + private final int defaultApiTimeoutMs; + private volatile boolean closed = false; + private final List assignors; + private final Optional clientTelemetryReporter; + + // currentThread holds the threadId of the current thread accessing LegacyKafkaConsumer + // and is used to prevent multi-threaded access + private final AtomicLong currentThread = new AtomicLong(NO_CURRENT_THREAD); + // refcount is used to allow reentrant access by the thread who has acquired currentThread + private final AtomicInteger refcount = new AtomicInteger(0); + + // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates + private boolean cachedSubscriptionHasAllFetchPositions; + + LegacyKafkaConsumer(ConsumerConfig config, Deserializer keyDeserializer, Deserializer valueDeserializer) { + try { + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, + GroupRebalanceConfig.ProtocolType.CONSUMER); + + this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); + this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); + LogContext logContext = createLogContext(config, groupRebalanceConfig); + this.log = logContext.logger(getClass()); + boolean enableAutoCommit = config.getBoolean(ENABLE_AUTO_COMMIT_CONFIG); + groupId.ifPresent(groupIdStr -> { + if (groupIdStr.isEmpty()) { + log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release."); + } + }); + + log.debug("Initializing the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); + this.time = Time.SYSTEM; + List reporters = CommonClientConfigs.metricsReporters(clientId, config); + this.clientTelemetryReporter = CommonClientConfigs.telemetryReporter(clientId, config); + this.clientTelemetryReporter.ifPresent(reporters::add); + this.metrics = createMetrics(config, time, reporters); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + this.retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + + List> interceptorList = configuredConsumerInterceptors(config); + this.interceptors = new ConsumerInterceptors<>(interceptorList); + this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); + this.subscriptions = createSubscriptionState(config, logContext); + ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners( + metrics.reporters(), + interceptorList, + Arrays.asList(this.deserializers.keyDeserializer, this.deserializers.valueDeserializer)); + this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); + List addresses = ClientUtils.parseAndValidateAddresses(config); + this.metadata.bootstrap(addresses); + + FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); + FetchConfig fetchConfig = new FetchConfig(config); + this.isolationLevel = fetchConfig.isolationLevel; + + ApiVersions apiVersions = new ApiVersions(); + this.client = createConsumerNetworkClient(config, + metrics, + logContext, + apiVersions, + time, + metadata, + fetchMetricsManager.throttleTimeSensor(), + retryBackoffMs, + clientTelemetryReporter.map(ClientTelemetryReporter::telemetrySender).orElse(null)); + + this.assignors = ConsumerPartitionAssignor.getAssignorInstances( + config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), + config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) + ); + + // no coordinator will be constructed for the default (null) group id + if (!groupId.isPresent()) { + config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); + config.ignore(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); + this.coordinator = null; + } else { + this.coordinator = new ConsumerCoordinator(groupRebalanceConfig, + logContext, + this.client, + assignors, + this.metadata, + this.subscriptions, + metrics, + CONSUMER_METRIC_GROUP_PREFIX, + this.time, + enableAutoCommit, + config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), + this.interceptors, + config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED), + config.getString(ConsumerConfig.CLIENT_RACK_CONFIG), + clientTelemetryReporter); + } + this.fetcher = new Fetcher<>( + logContext, + this.client, + this.metadata, + this.subscriptions, + fetchConfig, + this.deserializers, + fetchMetricsManager, + this.time, + apiVersions); + this.offsetFetcher = new OffsetFetcher(logContext, + client, + metadata, + subscriptions, + time, + retryBackoffMs, + requestTimeoutMs, + isolationLevel, + apiVersions); + this.topicMetadataFetcher = new TopicMetadataFetcher(logContext, + client, + retryBackoffMs, + retryBackoffMaxMs); + + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); + + config.logUnused(); + AppInfoParser.registerAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics, time.milliseconds()); + log.debug("Kafka consumer initialized"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 + // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. + if (this.log != null) { + close(Duration.ZERO, true); + } + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + } + } + + // visible for testing + LegacyKafkaConsumer(LogContext logContext, + Time time, + ConsumerConfig config, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + KafkaClient client, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + List assignors) { + this.log = logContext.logger(getClass()); + this.time = time; + this.subscriptions = subscriptions; + this.metadata = metadata; + this.metrics = new Metrics(time); + this.clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); + this.groupId = Optional.ofNullable(config.getString(ConsumerConfig.GROUP_ID_CONFIG)); + this.deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + this.isolationLevel = ConsumerUtils.configuredIsolationLevel(config); + this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); + this.assignors = assignors; + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); + this.interceptors = new ConsumerInterceptors<>(Collections.emptyList()); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + this.retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.clientTelemetryReporter = Optional.empty(); + + int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); + int rebalanceTimeoutMs = config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG); + int heartbeatIntervalMs = config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG); + boolean enableAutoCommit = config.getBoolean(ENABLE_AUTO_COMMIT_CONFIG); + boolean throwOnStableOffsetNotSupported = config.getBoolean(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); + int autoCommitIntervalMs = config.getInt(AUTO_COMMIT_INTERVAL_MS_CONFIG); + String rackId = config.getString(CLIENT_RACK_CONFIG); + Optional groupInstanceId = Optional.ofNullable(config.getString(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG)); + + this.client = new ConsumerNetworkClient( + logContext, + client, + metadata, + time, + retryBackoffMs, + requestTimeoutMs, + heartbeatIntervalMs + ); + + if (groupId.isPresent()) { + GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig( + sessionTimeoutMs, + rebalanceTimeoutMs, + heartbeatIntervalMs, + groupId.get(), + groupInstanceId, + retryBackoffMs, + retryBackoffMaxMs, + true + ); + this.coordinator = new ConsumerCoordinator( + rebalanceConfig, + logContext, + this.client, + assignors, + metadata, + subscriptions, + metrics, + CONSUMER_METRIC_GROUP_PREFIX, + time, + enableAutoCommit, + autoCommitIntervalMs, + interceptors, + throwOnStableOffsetNotSupported, + rackId, + clientTelemetryReporter + ); + } else { + this.coordinator = null; + } + + int maxBytes = config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG); + int maxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); + int minBytes = config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG); + int fetchSize = config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG); + int maxPollRecords = config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); + boolean checkCrcs = config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG); + + ConsumerMetrics metricsRegistry = new ConsumerMetrics(CONSUMER_METRIC_GROUP_PREFIX); + FetchMetricsManager metricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics); + ApiVersions apiVersions = new ApiVersions(); + FetchConfig fetchConfig = new FetchConfig( + minBytes, + maxBytes, + maxWaitMs, + fetchSize, + maxPollRecords, + checkCrcs, + rackId, + isolationLevel + ); + this.fetcher = new Fetcher<>( + logContext, + this.client, + metadata, + subscriptions, + fetchConfig, + deserializers, + metricsManager, + time, + apiVersions + ); + this.offsetFetcher = new OffsetFetcher( + logContext, + this.client, + metadata, + subscriptions, + time, + retryBackoffMs, + requestTimeoutMs, + isolationLevel, + apiVersions + ); + this.topicMetadataFetcher = new TopicMetadataFetcher( + logContext, + this.client, + retryBackoffMs, + retryBackoffMaxMs + ); + } + + public Set assignment() { + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(this.subscriptions.assignedPartitions()); + } finally { + release(); + } + } + + public Set subscription() { + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(new HashSet<>(this.subscriptions.subscription())); + } finally { + release(); + } + } + + @Override + public void subscribe(Collection topics, ConsumerRebalanceListener listener) { + if (listener == null) + throw new IllegalArgumentException("RebalanceListener cannot be null"); + + subscribeInternal(topics, Optional.of(listener)); + } + + @Override + public void subscribe(Collection topics) { + subscribeInternal(topics, Optional.empty()); + } + + /** + * Internal helper method for {@link #subscribe(Collection)} and + * {@link #subscribe(Collection, ConsumerRebalanceListener)} + *

+ * Subscribe to the given list of topics to get dynamically assigned partitions. + * Topic subscriptions are not incremental. This list will replace the current + * assignment (if there is one). It is not possible to combine topic subscription with group management + * with manual partition assignment through {@link #assign(Collection)}. + * + * If the given list of topics is empty, it is treated the same as {@link #unsubscribe()}. + * + *

+ * @param topics The list of topics to subscribe to + * @param listener {@link Optional} listener instance to get notifications on partition assignment/revocation + * for the subscribed topics + * @throws IllegalArgumentException If topics is null or contains null or empty elements + * @throws IllegalStateException If {@code subscribe()} is called previously with pattern, or assign is called + * previously (without a subsequent call to {@link #unsubscribe()}), or if not + * configured at-least one partition assignment strategy + */ + private void subscribeInternal(Collection topics, Optional listener) { + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + if (topics == null) + throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); + if (topics.isEmpty()) { + // treat subscribing to empty topic list as the same as unsubscribing + this.unsubscribe(); + } else { + for (String topic : topics) { + if (isBlank(topic)) + throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); + } + + throwIfNoAssignorsConfigured(); + + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (topics.contains(tp.topic())) + currentTopicPartitions.add(tp); + } + + fetcher.clearBufferedDataForUnassignedPartitions(currentTopicPartitions); + + log.info("Subscribed to topic(s): {}", join(topics, ", ")); + if (this.subscriptions.subscribe(new HashSet<>(topics), listener)) + metadata.requestUpdateForNewTopics(); + } + } finally { + release(); + } + } + + @Override + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + if (listener == null) + throw new IllegalArgumentException("RebalanceListener cannot be null"); + + subscribeInternal(pattern, Optional.of(listener)); + } + + @Override + public void subscribe(Pattern pattern) { + subscribeInternal(pattern, Optional.empty()); + } + + /** + * Internal helper method for {@link #subscribe(Pattern)} and + * {@link #subscribe(Pattern, ConsumerRebalanceListener)} + *

+ * Subscribe to all topics matching specified pattern to get dynamically assigned partitions. + * The pattern matching will be done periodically against all topics existing at the time of check. + * This can be controlled through the {@code metadata.max.age.ms} configuration: by lowering + * the max metadata age, the consumer will refresh metadata more often and check for matching topics. + *

+ * See {@link #subscribe(Collection, ConsumerRebalanceListener)} for details on the + * use of the {@link ConsumerRebalanceListener}. Generally rebalances are triggered when there + * is a change to the topics matching the provided pattern and when consumer group membership changes. + * Group rebalances only take place during an active call to {@link #poll(Duration)}. + * + * @param pattern Pattern to subscribe to + * @param listener {@link Optional} listener instance to get notifications on partition assignment/revocation + * for the subscribed topics + * @throws IllegalArgumentException If pattern or listener is null + * @throws IllegalStateException If {@code subscribe()} is called previously with topics, or assign is called + * previously (without a subsequent call to {@link #unsubscribe()}), or if not + * configured at-least one partition assignment strategy + */ + private void subscribeInternal(Pattern pattern, Optional listener) { + maybeThrowInvalidGroupIdException(); + if (pattern == null || pattern.toString().equals("")) + throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? + "null" : "empty")); + + acquireAndEnsureOpen(); + try { + throwIfNoAssignorsConfigured(); + log.info("Subscribed to pattern: '{}'", pattern); + this.subscriptions.subscribe(pattern, listener); + this.coordinator.updatePatternSubscription(metadata.fetch()); + this.metadata.requestUpdateForNewTopics(); + } finally { + release(); + } + } + + public void unsubscribe() { + acquireAndEnsureOpen(); + try { + fetcher.clearBufferedDataForUnassignedPartitions(Collections.emptySet()); + if (this.coordinator != null) { + this.coordinator.onLeavePrepare(); + this.coordinator.maybeLeaveGroup("the consumer unsubscribed from all topics"); + } + this.subscriptions.unsubscribe(); + log.info("Unsubscribed all topics or patterns and assigned partitions"); + } finally { + release(); + } + } + + @Override + public void assign(Collection partitions) { + acquireAndEnsureOpen(); + try { + if (partitions == null) { + throw new IllegalArgumentException("Topic partition collection to assign to cannot be null"); + } else if (partitions.isEmpty()) { + this.unsubscribe(); + } else { + for (TopicPartition tp : partitions) { + String topic = (tp != null) ? tp.topic() : null; + if (isBlank(topic)) + throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); + } + fetcher.clearBufferedDataForUnassignedPartitions(partitions); + + // make sure the offsets of topic partitions the consumer is unsubscribing from + // are committed since there will be no following rebalance + if (coordinator != null) + this.coordinator.maybeAutoCommitOffsetsAsync(time.milliseconds()); + + log.info("Assigned to partition(s): {}", join(partitions, ", ")); + if (this.subscriptions.assignFromUser(new HashSet<>(partitions))) + metadata.requestUpdateForNewTopics(); + } + } finally { + release(); + } + } + + @Deprecated + @Override + public ConsumerRecords poll(final long timeoutMs) { + return poll(time.timer(timeoutMs), false); + } + + @Override + public ConsumerRecords poll(final Duration timeout) { + return poll(time.timer(timeout), true); + } + + /** + * @throws KafkaException if the rebalance callback throws exception + */ + private ConsumerRecords poll(final Timer timer, final boolean includeMetadataInTimeout) { + acquireAndEnsureOpen(); + try { + this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); + + if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { + throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); + } + + do { + client.maybeTriggerWakeup(); + + if (includeMetadataInTimeout) { + // try to update assignment metadata BUT do not need to block on the timer for join group + updateAssignmentMetadataIfNeeded(timer, false); + } else { + while (!updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE), true)) { + log.warn("Still waiting for metadata"); + } + } + + final Fetch fetch = pollForFetches(timer); + if (!fetch.isEmpty()) { + // before returning the fetched records, we can send off the next round of fetches + // and avoid block waiting for their responses to enable pipelining while the user + // is handling the fetched records. + // + // NOTE: since the consumed position has already been updated, we must not allow + // wakeups or any other errors to be triggered prior to returning the fetched records. + if (sendFetches() > 0 || client.hasPendingRequests()) { + client.transmitSends(); + } + + if (fetch.records().isEmpty()) { + log.trace("Returning empty records from `poll()` " + + "since the consumer's position has advanced for at least one topic partition"); + } + + return this.interceptors.onConsume(new ConsumerRecords<>(fetch.records())); + } + } while (timer.notExpired()); + + return ConsumerRecords.empty(); + } finally { + release(); + this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); + } + } + + private int sendFetches() { + offsetFetcher.validatePositionsOnMetadataChange(); + return fetcher.sendFetches(); + } + + boolean updateAssignmentMetadataIfNeeded(final Timer timer, final boolean waitForJoinGroup) { + if (coordinator != null && !coordinator.poll(timer, waitForJoinGroup)) { + return false; + } + + return updateFetchPositions(timer); + } + + /** + * @throws KafkaException if the rebalance callback throws exception + */ + private Fetch pollForFetches(Timer timer) { + long pollTimeout = coordinator == null ? timer.remainingMs() : + Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); + + // if data is available already, return it immediately + final Fetch fetch = fetcher.collectFetch(); + if (!fetch.isEmpty()) { + return fetch; + } + + // send any new fetches (won't resend pending fetches) + sendFetches(); + + // We do not want to be stuck blocking in poll if we are missing some positions + // since the offset lookup may be backing off after a failure + + // NOTE: the use of cachedSubscriptionHasAllFetchPositions means we MUST call + // updateAssignmentMetadataIfNeeded before this method. + if (!cachedSubscriptionHasAllFetchPositions && pollTimeout > retryBackoffMs) { + pollTimeout = retryBackoffMs; + } + + log.trace("Polling for fetches with timeout {}", pollTimeout); + + Timer pollTimer = time.timer(pollTimeout); + client.poll(pollTimer, () -> { + // since a fetch might be completed by the background thread, we need this poll condition + // to ensure that we do not block unnecessarily in poll() + return !fetcher.hasAvailableFetches(); + }); + timer.update(pollTimer.currentTimeMs()); + + return fetcher.collectFetch(); + } + + @Override + public void commitSync() { + commitSync(Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public void commitSync(Duration timeout) { + commitSync(subscriptions.allConsumed(), timeout); + } + + @Override + public void commitSync(final Map offsets) { + commitSync(offsets, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public void commitSync(final Map offsets, final Duration timeout) { + acquireAndEnsureOpen(); + long commitStart = time.nanoseconds(); + try { + maybeThrowInvalidGroupIdException(); + offsets.forEach(this::updateLastSeenEpochIfNewer); + if (!coordinator.commitOffsetsSync(new HashMap<>(offsets), time.timer(timeout))) { + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before successfully " + + "committing offsets " + offsets); + } + } finally { + kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart); + release(); + } + } + + @Override + public void commitAsync() { + commitAsync(null); + } + + @Override + public void commitAsync(OffsetCommitCallback callback) { + commitAsync(subscriptions.allConsumed(), callback); + } + + @Override + public void commitAsync(final Map offsets, OffsetCommitCallback callback) { + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + log.debug("Committing offsets: {}", offsets); + offsets.forEach(this::updateLastSeenEpochIfNewer); + coordinator.commitOffsetsAsync(new HashMap<>(offsets), callback); + } finally { + release(); + } + } + + @Override + public void seek(TopicPartition partition, long offset) { + if (offset < 0) + throw new IllegalArgumentException("seek offset must not be a negative number"); + + acquireAndEnsureOpen(); + try { + log.info("Seeking to offset {} for partition {}", offset, partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offset, + Optional.empty(), // This will ensure we skip validation + this.metadata.currentLeader(partition)); + this.subscriptions.seekUnvalidated(partition, newPosition); + } finally { + release(); + } + } + + @Override + public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { + long offset = offsetAndMetadata.offset(); + if (offset < 0) { + throw new IllegalArgumentException("seek offset must not be a negative number"); + } + + acquireAndEnsureOpen(); + try { + if (offsetAndMetadata.leaderEpoch().isPresent()) { + log.info("Seeking to offset {} for partition {} with epoch {}", + offset, partition, offsetAndMetadata.leaderEpoch().get()); + } else { + log.info("Seeking to offset {} for partition {}", offset, partition); + } + Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.currentLeader(partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offsetAndMetadata.offset(), + offsetAndMetadata.leaderEpoch(), + currentLeaderAndEpoch); + this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); + this.subscriptions.seekUnvalidated(partition, newPosition); + } finally { + release(); + } + } + + @Override + public void seekToBeginning(Collection partitions) { + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + acquireAndEnsureOpen(); + try { + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); + } finally { + release(); + } + } + + @Override + public void seekToEnd(Collection partitions) { + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + acquireAndEnsureOpen(); + try { + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); + } finally { + release(); + } + } + + @Override + public long position(TopicPartition partition) { + return position(partition, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public long position(TopicPartition partition, final Duration timeout) { + acquireAndEnsureOpen(); + try { + if (!this.subscriptions.isAssigned(partition)) + throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); + + Timer timer = time.timer(timeout); + do { + SubscriptionState.FetchPosition position = this.subscriptions.validPosition(partition); + if (position != null) + return position.offset; + + updateFetchPositions(timer); + client.poll(timer); + } while (timer.notExpired()); + + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + + "for partition " + partition + " could be determined"); + } finally { + release(); + } + } + + @Deprecated + @Override + public OffsetAndMetadata committed(TopicPartition partition) { + return committed(partition, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Deprecated + @Override + public OffsetAndMetadata committed(TopicPartition partition, final Duration timeout) { + return committed(Collections.singleton(partition), timeout).get(partition); + } + + @Override + public Map committed(final Set partitions) { + return committed(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public Map committed(final Set partitions, final Duration timeout) { + acquireAndEnsureOpen(); + long start = time.nanoseconds(); + try { + maybeThrowInvalidGroupIdException(); + final Map offsets; + offsets = coordinator.fetchCommittedOffsets(partitions, time.timer(timeout)); + if (offsets == null) { + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the last " + + "committed offset for partitions " + partitions + " could be determined. Try tuning " + + ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG + " larger to relax the threshold."); + } else { + offsets.forEach(this::updateLastSeenEpochIfNewer); + return offsets; + } + } finally { + kafkaConsumerMetrics.recordCommitted(time.nanoseconds() - start); + release(); + } + } + + @Override + public Uuid clientInstanceId(Duration timeout) { + if (!clientTelemetryReporter.isPresent()) { + throw new IllegalStateException("Telemetry is not enabled. Set config `" + ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG + "` to `true`."); + + } + + return ClientTelemetryUtils.fetchClientInstanceId(clientTelemetryReporter.get(), timeout); + } + + @Override + public Map metrics() { + return Collections.unmodifiableMap(this.metrics.metrics()); + } + + @Override + public List partitionsFor(String topic) { + return partitionsFor(topic, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public List partitionsFor(String topic, Duration timeout) { + acquireAndEnsureOpen(); + try { + Cluster cluster = this.metadata.fetch(); + List parts = cluster.partitionsForTopic(topic); + if (!parts.isEmpty()) + return parts; + + Timer timer = time.timer(timeout); + List topicMetadata = topicMetadataFetcher.getTopicMetadata(topic, metadata.allowAutoTopicCreation(), timer); + return topicMetadata != null ? topicMetadata : Collections.emptyList(); + } finally { + release(); + } + } + + @Override + public Map> listTopics() { + return listTopics(Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public Map> listTopics(Duration timeout) { + acquireAndEnsureOpen(); + try { + return topicMetadataFetcher.getAllTopicMetadata(time.timer(timeout)); + } finally { + release(); + } + } + + @Override + public void pause(Collection partitions) { + acquireAndEnsureOpen(); + try { + log.debug("Pausing partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.pause(partition); + } + } finally { + release(); + } + } + + @Override + public void resume(Collection partitions) { + acquireAndEnsureOpen(); + try { + log.debug("Resuming partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.resume(partition); + } + } finally { + release(); + } + } + + @Override + public Set paused() { + acquireAndEnsureOpen(); + try { + return Collections.unmodifiableSet(subscriptions.pausedPartitions()); + } finally { + release(); + } + } + + @Override + public Map offsetsForTimes(Map timestampsToSearch) { + return offsetsForTimes(timestampsToSearch, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public Map offsetsForTimes(Map timestampsToSearch, Duration timeout) { + acquireAndEnsureOpen(); + try { + for (Map.Entry entry : timestampsToSearch.entrySet()) { + // we explicitly exclude the earliest and latest offset here so the timestamp in the returned + // OffsetAndTimestamp is always positive. + if (entry.getValue() < 0) + throw new IllegalArgumentException("The target time for partition " + entry.getKey() + " is " + + entry.getValue() + ". The target time cannot be negative."); + } + return offsetFetcher.offsetsForTimes(timestampsToSearch, time.timer(timeout)); + } finally { + release(); + } + } + + @Override + public Map beginningOffsets(Collection partitions) { + return beginningOffsets(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public Map beginningOffsets(Collection partitions, Duration timeout) { + acquireAndEnsureOpen(); + try { + return offsetFetcher.beginningOffsets(partitions, time.timer(timeout)); + } finally { + release(); + } + } + + @Override + public Map endOffsets(Collection partitions) { + return endOffsets(partitions, Duration.ofMillis(defaultApiTimeoutMs)); + } + + @Override + public Map endOffsets(Collection partitions, Duration timeout) { + acquireAndEnsureOpen(); + try { + return offsetFetcher.endOffsets(partitions, time.timer(timeout)); + } finally { + release(); + } + } + + @Override + public OptionalLong currentLag(TopicPartition topicPartition) { + acquireAndEnsureOpen(); + try { + final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel); + + // if the log end offset is not known and hence cannot return lag and there is + // no in-flight list offset requested yet, + // issue a list offset request for that partition so that next time + // we may get the answer; we do not need to wait for the return value + // since we would not try to poll the network client synchronously + if (lag == null) { + if (subscriptions.partitionEndOffset(topicPartition, isolationLevel) == null && + !subscriptions.partitionEndOffsetRequested(topicPartition)) { + log.info("Requesting the log end offset for {} in order to compute lag", topicPartition); + subscriptions.requestPartitionEndOffset(topicPartition); + offsetFetcher.endOffsets(Collections.singleton(topicPartition), time.timer(0L)); + } + + return OptionalLong.empty(); + } + + return OptionalLong.of(lag); + } finally { + release(); + } + } + + @Override + public ConsumerGroupMetadata groupMetadata() { + acquireAndEnsureOpen(); + try { + maybeThrowInvalidGroupIdException(); + return coordinator.groupMetadata(); + } finally { + release(); + } + } + + @Override + public void enforceRebalance(final String reason) { + acquireAndEnsureOpen(); + try { + if (coordinator == null) { + throw new IllegalStateException("Tried to force a rebalance but consumer does not have a group."); + } + coordinator.requestRejoin(reason == null || reason.isEmpty() ? DEFAULT_REASON : reason); + } finally { + release(); + } + } + + @Override + public void enforceRebalance() { + enforceRebalance(null); + } + + @Override + public void close() { + close(Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS)); + } + + @Override + public void close(Duration timeout) { + if (timeout.toMillis() < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + acquire(); + try { + if (!closed) { + // need to close before setting the flag since the close function + // itself may trigger rebalance callback that needs the consumer to be open still + close(timeout, false); + } + } finally { + closed = true; + release(); + } + } + + @Override + public void wakeup() { + this.client.wakeup(); + } + + private Timer createTimerForRequest(final Duration timeout) { + // this.time could be null if an exception occurs in constructor prior to setting the this.time field + final Time localTime = (time == null) ? Time.SYSTEM : time; + return localTime.timer(Math.min(timeout.toMillis(), requestTimeoutMs)); + } + + private void close(Duration timeout, boolean swallowException) { + log.trace("Closing the Kafka consumer"); + AtomicReference firstException = new AtomicReference<>(); + + final Timer closeTimer = createTimerForRequest(timeout); + clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(timeout.toMillis())); + closeTimer.update(); + // Close objects with a timeout. The timeout is required because the coordinator & the fetcher send requests to + // the server in the process of closing which may not respect the overall timeout defined for closing the + // consumer. + if (coordinator != null) { + // This is a blocking call bound by the time remaining in closeTimer + swallow(log, Level.ERROR, "Failed to close coordinator with a timeout(ms)=" + closeTimer.timeoutMs(), () -> coordinator.close(closeTimer), firstException); + } + + if (fetcher != null) { + // the timeout for the session close is at-most the requestTimeoutMs + long remainingDurationInTimeout = Math.max(0, timeout.toMillis() - closeTimer.elapsedMs()); + if (remainingDurationInTimeout > 0) { + remainingDurationInTimeout = Math.min(requestTimeoutMs, remainingDurationInTimeout); + } + + closeTimer.reset(remainingDurationInTimeout); + + // This is a blocking call bound by the time remaining in closeTimer + swallow(log, Level.ERROR, "Failed to close fetcher with a timeout(ms)=" + closeTimer.timeoutMs(), () -> fetcher.close(closeTimer), firstException); + } + + closeQuietly(interceptors, "consumer interceptors", firstException); + closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); + closeQuietly(metrics, "consumer metrics", firstException); + closeQuietly(client, "consumer network client", firstException); + closeQuietly(deserializers, "consumer deserializers", firstException); + clientTelemetryReporter.ifPresent(reporter -> closeQuietly(reporter, "consumer telemetry reporter", firstException)); + AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); + log.debug("Kafka consumer has been closed"); + Throwable exception = firstException.get(); + if (exception != null && !swallowException) { + if (exception instanceof InterruptException) { + throw (InterruptException) exception; + } + throw new KafkaException("Failed to close kafka consumer", exception); + } + } + + /** + * Set the fetch position to the committed position (if there is one) + * or reset it using the offset reset policy the user has configured. + * + * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details + * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is + * defined + * @return true iff the operation completed without timing out + */ + private boolean updateFetchPositions(final Timer timer) { + // If any partitions have been truncated due to a leader change, we need to validate the offsets + offsetFetcher.validatePositionsIfNeeded(); + + cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions(); + if (cachedSubscriptionHasAllFetchPositions) return true; + + // If there are any partitions which do not have a valid position and are not + // awaiting reset, then we need to fetch committed offsets. We will only do a + // coordinator lookup if there are partitions which have missing positions, so + // a consumer with manually assigned partitions can avoid a coordinator dependence + // by always ensuring that assigned partitions have an initial position. + if (coordinator != null && !coordinator.initWithCommittedOffsetsIfNeeded(timer)) return false; + + // If there are partitions still needing a position and a reset policy is defined, + // request reset using the default policy. If no reset strategy is defined and there + // are partitions with a missing position, then we will raise an exception. + subscriptions.resetInitializingPositions(); + + // Finally send an asynchronous request to look up and update the positions of any + // partitions which are awaiting reset. + offsetFetcher.resetPositionsIfNeeded(); + + return true; + } + + /** + * Acquire the light lock and ensure that the consumer hasn't been closed. + * @throws IllegalStateException If the consumer has been closed + */ + private void acquireAndEnsureOpen() { + acquire(); + if (this.closed) { + release(); + throw new IllegalStateException("This consumer has already been closed."); + } + } + + /** + * Acquire the light lock protecting this consumer from multi-threaded access. Instead of blocking + * when the lock is not available, however, we just throw an exception (since multi-threaded usage is not + * supported). + * @throws ConcurrentModificationException if another thread already has the lock + */ + private void acquire() { + final Thread thread = Thread.currentThread(); + final long threadId = thread.getId(); + if (threadId != currentThread.get() && !currentThread.compareAndSet(NO_CURRENT_THREAD, threadId)) + throw new ConcurrentModificationException("KafkaConsumer is not safe for multi-threaded access. " + + "currentThread(name: " + thread.getName() + ", id: " + threadId + ")" + + " otherThread(id: " + currentThread.get() + ")" + ); + refcount.incrementAndGet(); + } + + /** + * Release the light lock protecting the consumer from multi-threaded access. + */ + private void release() { + if (refcount.decrementAndGet() == 0) + currentThread.set(NO_CURRENT_THREAD); + } + + private void throwIfNoAssignorsConfigured() { + if (assignors.isEmpty()) + throw new IllegalStateException("Must configure at least one partition assigner class name to " + + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG + " configuration property"); + } + + private void maybeThrowInvalidGroupIdException() { + if (!groupId.isPresent()) + throw new InvalidGroupIdException("To use the group management or offset commit APIs, you must " + + "provide a valid " + ConsumerConfig.GROUP_ID_CONFIG + " in the consumer configuration."); + } + + private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) { + if (offsetAndMetadata != null) + offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch)); + } + + // Functions below are for testing only + @Override + public String clientId() { + return clientId; + } + + @Override + public Metrics metricsRegistry() { + return metrics; + } + + @Override + public KafkaConsumerMetrics kafkaConsumerMetrics() { + return kafkaConsumerMetrics; + } + + @Override + public boolean updateAssignmentMetadataIfNeeded(final Timer timer) { + return updateAssignmentMetadataIfNeeded(timer, true); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java index ebcb279cf37a8..35d70b0fc990d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.common.protocol.Errors; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -24,50 +25,103 @@ public enum MemberState { /** - * Member has not joined a consumer group yet, or has been fenced and needs to re-join. + * Member has a group id, but it is not subscribed to any topic to receive automatic + * assignments. This will be the state when the member has never subscribed, or when it has + * unsubscribed from all topics. While in this state the member can commit offsets but won't + * be an active member of the consumer group (no heartbeats sent). */ - UNJOINED, + UNSUBSCRIBED, + + /** + * Member is attempting to join a consumer group. While in this state, the member will send + * heartbeat requests on the interval, with epoch 0, until it gets a response with an epoch > 0 + * or a fatal failure. A member transitions to this state when it tries to join the group for + * the first time with a call to subscribe, or when it has been fenced and tries to re-join. + */ + JOINING, /** * Member has received a new target assignment (partitions could have been assigned or - * revoked), and it is processing it. While in this state, the member will - * invoke the user callbacks for onPartitionsAssigned or onPartitionsRevoked, and then make - * the new assignment effective. + * revoked), and it is processing it. While in this state, the member will continue to send + * heartbeat on the interval, and reconcile the assignment (it will commit offsets if + * needed, invoke the user callbacks for onPartitionsAssigned or onPartitionsRevoked, and make + * the new assignment effective). Note that while in this state the member may be trying to + * resolve metadata for the target assignment, or triggering commits/callbacks if topic names + * already resolved. */ - // TODO: determine if separate state will be needed for assign/revoke (not for now) RECONCILING, /** - * Member is active in a group (heartbeating) and has processed all assignments received. + * Member has completed reconciling an assignment received, and stays in this state only until + * the next heartbeat request is sent out to acknowledge the assignment to the server. This + * state indicates that the next heartbeat request must be sent without waiting for the + * heartbeat interval to expire. Note that once the ack is sent, the member could go back to + * {@link #RECONCILING} if it still has assignment waiting to be reconciled (assignments + * waiting for metadata, assignments for which metadata was resolved, or new assignments + * received from the broker) + */ + ACKNOWLEDGING, + + /** + * Member is active in a group and has processed all assignments received. While in this + * state, the member will send heartbeats on the interval. */ STABLE, /** - * Member transitions to this state when it receives a - * {@link org.apache.kafka.common.protocol.Errors#UNKNOWN_MEMBER_ID} or - * {@link org.apache.kafka.common.protocol.Errors#FENCED_MEMBER_EPOCH} error from the - * broker. This is a recoverable state, where the member - * gives up its partitions by invoking the user callbacks for onPartitionsLost, and then - * transitions to {@link #UNJOINED} to rejoin the group as a new member. + * Member transitions to this state when it receives a {@link Errors#UNKNOWN_MEMBER_ID} or + * {@link Errors#FENCED_MEMBER_EPOCH} error from the broker, indicating that it has been + * left out of the group. While in this state, the member will stop sending heartbeats, it + * will give up its partitions by invoking the user callbacks for onPartitionsLost, and then + * transition to {@link #JOINING} to re-join the group as a new member. */ FENCED, /** - * The member failed with an unrecoverable error + * The member transitions to this state after a call to unsubscribe. While in this state, the + * member will stop sending heartbeats, will commit offsets if needed and release its + * assignment (calling user's callback for partitions revoked or lost). When all these + * actions complete, the member will transition out of this state into {@link #LEAVING} to + * effectively leave the group. + */ + PREPARE_LEAVING, + + /** + * Member has committed offsets and releases its assignment, so it stays in this state until + * the next heartbeat request is sent out with epoch -1 or -2 to effectively leave the group. + * This state indicates that the next heartbeat request must be sent without waiting for the + * heartbeat interval to expire. */ - FAILED; + LEAVING, + /** + * The member failed with an unrecoverable error received in a heartbeat response. This in an + * unrecoverable state where the member won't send any requests to the broker and cannot + * perform any other transition. + */ + FATAL; + + // Valid state transitions static { - // Valid state transitions - STABLE.previousValidStates = Arrays.asList(UNJOINED, RECONCILING); - RECONCILING.previousValidStates = Arrays.asList(STABLE, UNJOINED); + STABLE.previousValidStates = Arrays.asList(JOINING, ACKNOWLEDGING); + + RECONCILING.previousValidStates = Arrays.asList(STABLE, JOINING, ACKNOWLEDGING); + + ACKNOWLEDGING.previousValidStates = Arrays.asList(RECONCILING); + + FATAL.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING); + + FENCED.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING); + + JOINING.previousValidStates = Arrays.asList(FENCED, UNSUBSCRIBED); - FAILED.previousValidStates = Arrays.asList(UNJOINED, STABLE, RECONCILING); + PREPARE_LEAVING.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, + ACKNOWLEDGING, UNSUBSCRIBED, FENCED); - FENCED.previousValidStates = Arrays.asList(STABLE, RECONCILING); + LEAVING.previousValidStates = Arrays.asList(PREPARE_LEAVING); - UNJOINED.previousValidStates = Arrays.asList(FENCED); + UNSUBSCRIBED.previousValidStates = Arrays.asList(LEAVING); } private List previousValidStates; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java index 8a95a80c65991..4727daa0f64b8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java @@ -16,9 +16,12 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.common.TopicIdPartition; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; /** * A stateful object tracking the state of a single member in relationship to a consumer group: @@ -58,11 +61,25 @@ public interface MembershipManager { MemberState state(); /** - * Update member info and transition member state based on a heartbeat response. + * Update member info and transition member state based on a successful heartbeat response. * * @param response Heartbeat response to extract member info and errors from. */ - void updateState(ConsumerGroupHeartbeatResponseData response); + void onHeartbeatResponseReceived(ConsumerGroupHeartbeatResponseData response); + + /** + * Update state when a heartbeat is sent out. This will transition out of the states that end + * when a heartbeat request is sent, without waiting for a response (ex. + * {@link MemberState#ACKNOWLEDGING} and {@link MemberState#LEAVING}). + */ + void onHeartbeatRequestSent(); + + /** + * Transition out of the {@link MemberState#LEAVING} state even if the heartbeat was not sent + * . This will ensure that the member is not blocked on {@link MemberState#LEAVING} (best + * effort to send the request, without any response handling or retry logic) + */ + void onHeartbeatRequestSkipped(); /** * @return Server-side assignor implementation configured for the member, that will be sent @@ -73,30 +90,51 @@ public interface MembershipManager { /** * @return Current assignment for the member. */ - ConsumerGroupHeartbeatResponseData.Assignment currentAssignment(); + Set currentAssignment(); /** - * Update the assignment for the member, indicating that the provided assignment is the new - * current assignment. - */ - void onTargetAssignmentProcessComplete(ConsumerGroupHeartbeatResponseData.Assignment assignment); - - /** - * Transition the member to the FENCED state and update the member info as required. This is - * only invoked when the heartbeat returns a FENCED_MEMBER_EPOCH or UNKNOWN_MEMBER_ID error. - * code. + * Transition the member to the FENCED state, where the member will release the assignment by + * calling the onPartitionsLost callback, and when the callback completes, it will transition + * to {@link MemberState#JOINING} to rejoin the group. This is expected to be invoked when + * the heartbeat returns a FENCED_MEMBER_EPOCH or UNKNOWN_MEMBER_ID error. */ void transitionToFenced(); /** * Transition the member to the FAILED state and update the member info as required. This is * invoked when un-recoverable errors occur (ex. when the heartbeat returns a non-retriable - * error or when errors occur while executing the user-provided callbacks) + * error) + */ + void transitionToFatal(); + + /** + * Release assignment and transition to {@link MemberState#PREPARE_LEAVING} so that a heartbeat + * request is sent indicating the broker that the member wants to leave the group. This is + * expected to be invoked when the user calls the unsubscribe API. + * + * @return Future that will complete when the callback execution completes and the heartbeat + * to leave the group has been sent out. + */ + CompletableFuture leaveGroup(); + + /** + * @return True if the member should send heartbeat to the coordinator without waiting for + * the interval. + */ + boolean shouldHeartbeatNow(); + + /** + * @return True if the member should skip sending the heartbeat to the coordinator. This + * could be the case then the member is not in a group, or when it failed with a fatal error. */ - void transitionToFailed(); + boolean shouldSkipHeartbeat(); /** - * @return True if the member should send heartbeat to the coordinator. + * Join the group with the updated subscription, if the member is not part of it yet. If the + * member is already part of the group, this will only ensure that the updated subscription + * is included in the next heartbeat request. + *

+ * Note that list of topics of the subscription is taken from the shared subscription state. */ - boolean shouldSendHeartbeat(); + void onSubscriptionUpdated(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java index 2a9a5d2992daa..8e19f10f2447f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java @@ -17,25 +17,102 @@ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.Utils.TopicIdPartitionComparator; +import org.apache.kafka.clients.consumer.internals.Utils.TopicPartitionComparator; +import org.apache.kafka.common.ClusterResource; +import org.apache.kafka.common.ClusterResourceListener; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; /** - * Membership manager that maintains group membership for a single member, following the new - * consumer group protocol. + * Group manager for a single consumer that has a group id defined in the config + * {@link ConsumerConfig#GROUP_ID_CONFIG}, to use the Kafka-based offset management capability, + * and the consumer group protocol to get automatically assigned partitions when calling the + * subscribe API. + * + *

+ * + * While the subscribe API hasn't been called (or if the consumer called unsubscribe), this manager + * will only be responsible for keeping the member in the {@link MemberState#UNSUBSCRIBED} state, + * where it can commit offsets to the group identified by the {@link #groupId()}, without joining + * the group. + * *

- * This is responsible for: - *

  • Keeping member info (ex. member id, member epoch, assignment, etc.)
    • - *
  • Keeping member state as defined in {@link MemberState}.
    • + * + * If the consumer subscribe API is called, this manager will use the {@link #groupId()} to join the + * consumer group, and based on the consumer group protocol heartbeats, will handle the full + * lifecycle of the member as it joins the group, reconciles assignments, handles fencing and + * fatal errors, and leaves the group. + * + *

    + * + * Reconciliation process:

    + * The member accepts all assignments received from the broker, resolves topic names from + * metadata, reconciles the resolved assignments, and keeps the unresolved to be reconciled when + * discovered with a metadata update. Reconciliations of resolved assignments are executed + * sequentially and acknowledged to the server as they complete. The reconciliation process + * involves multiple async operations, so the member will continue to heartbeat while these + * operations complete, to make sure that the member stays in the group while reconciling. + * *

    - * Member info and state are updated based on the heartbeat responses the member receives. + * + * Reconciliation steps: + *

      + *
    1. Resolve topic names for all topic IDs received in the target assignment. Topic names + * found in metadata are then ready to be reconciled. Topic IDs not found are kept as + * unresolved, and the member request metadata updates until it resolves them (or the broker + * removes it from the target assignment.
      2. + *
    2. Commit offsets if auto-commit is enabled.
      3. + *
    3. Invoke the user-defined onPartitionsRevoked listener.
      4. + *
    4. Invoke the user-defined onPartitionsAssigned listener.
      5. + *
    5. When the above steps complete, the member acknowledges the reconciled assignment, + * which is the subset of the target that was resolved from metadata and actually reconciled. + * The ack is performed by sending a heartbeat request back to the broker, including the + * reconciled assignment.
      6. + *
    + * + * Note that user-defined callbacks are triggered from this manager that runs in the + * BackgroundThread, but executed in the Application Thread, where a failure will be returned to + * the user if the callbacks fail. This manager is only concerned about the callbacks completion to + * know that it can proceed with the reconciliation. */ -public class MembershipManagerImpl implements MembershipManager { +public class MembershipManagerImpl implements MembershipManager, ClusterResourceListener { + + /** + * TopicPartition comparator based on topic name and partition id. + */ + private final static TopicPartitionComparator TOPIC_PARTITION_COMPARATOR = + new TopicPartitionComparator(); + + /** + * TopicIdPartition comparator based on topic name and partition id (ignoring ID while sorting, + * as this is sorted mainly for logging purposes). + */ + private final static TopicIdPartitionComparator TOPIC_ID_PARTITION_COMPARATOR = + new TopicIdPartitionComparator(); /** * Group ID of the consumer group the member will be part of, provided when creating the current @@ -52,7 +129,7 @@ public class MembershipManagerImpl implements MembershipManager { * Member ID assigned by the server to the member, received in a heartbeat response when * joining the group specified in {@link #groupId} */ - private String memberId; + private String memberId = ""; /** * Current epoch of the member. It will be set to 0 by the member, and provided to the server @@ -60,7 +137,7 @@ public class MembershipManagerImpl implements MembershipManager { * incremented as the member reconciles and acknowledges the assignments it receives. It will * be reset to 0 if the member gets fenced. */ - private int memberEpoch; + private int memberEpoch = 0; /** * Current state of this member as part of the consumer group, as defined in {@link MemberState} @@ -77,32 +154,116 @@ public class MembershipManagerImpl implements MembershipManager { /** * Assignment that the member received from the server and successfully processed. */ - private ConsumerGroupHeartbeatResponseData.Assignment currentAssignment; + private Set currentAssignment; + + /** + * Subscription state object holding the current assignment the member has for the topics it + * subscribed to. + */ + private final SubscriptionState subscriptions; /** - * Assignment that the member received from the server but hasn't completely processed - * yet. + * Metadata that allows us to create the partitions needed for {@link ConsumerRebalanceListener}. */ - private Optional targetAssignment; + private final ConsumerMetadata metadata; /** * Logger. */ private final Logger log; - public MembershipManagerImpl(String groupId, LogContext logContext) { - this(groupId, null, null, logContext); + /** + * Manager to perform commit requests needed before revoking partitions (if auto-commit is + * enabled) + */ + private final CommitRequestManager commitRequestManager; + + /** + * Local cache of assigned topic IDs and names. Topics are added here when received in a + * target assignment, as we discover their names in the Metadata cache, and removed when the + * topic is not in the subscription anymore. The purpose of this cache is to avoid metadata + * requests in cases where a currently assigned topic is in the target assignment (new + * partition assigned, or revoked), but it is not present the Metadata cache at that moment. + * The cache is cleared when the subscription changes ({@link #transitionToJoining()}, the + * member fails ({@link #transitionToFatal()} or leaves the group ({@link #leaveGroup()}). + */ + private final Map assignedTopicNamesCache; + + /** + * Topic IDs received in a target assignment for which we haven't found topic names yet. + * Items are added to this set every time a target assignment is received. Items are removed + * when metadata is found for the topic. This is where the member collects all assignments + * received from the broker, even though they may not be ready to reconcile due to missing + * metadata. + */ + private final Map> assignmentUnresolved; + + /** + * Assignment received for which topic names have been resolved, so it's ready to be + * reconciled. Items are added to this set when received in a target assignment (if metadata + * available), or when a metadata update is received. This is where the member keeps all the + * assignment ready to reconcile, even though the reconciliation might need to wait if there + * is already another on in process. + */ + private final SortedSet assignmentReadyToReconcile; + + /** + * If there is a reconciliation running (triggering commit, callbacks) for the + * assignmentReadyToReconcile. This will be true if {@link #reconcile()} has been triggered + * after receiving a heartbeat response, or a metadata update. + */ + private boolean reconciliationInProgress; + + /** + * Epoch the member had when the reconciliation in progress started. This is used to identify if + * the member has rejoined while it was reconciling an assignment (in which case the result + * of the reconciliation is not applied.) + */ + private int memberEpochOnReconciliationStart; + + /** + * If the member is currently leaving the group after a call to {@link #leaveGroup()}}, this + * will have a future that will complete when the ongoing leave operation completes + * (callbacks executed and heartbeat request to leave is sent out). This will be empty is the + * member is not leaving. + */ + private Optional> leaveGroupInProgress; + + /** + * True if the member has registered to be notified when the cluster metadata is updated. + * This is initially false, as the member that is not part of a consumer group does not + * require metadata updated. This becomes true the first time the member joins on the + * {@link #transitionToJoining()} + */ + private boolean isRegisteredForMetadataUpdates; + + public MembershipManagerImpl(String groupId, + SubscriptionState subscriptions, + CommitRequestManager commitRequestManager, + ConsumerMetadata metadata, + LogContext logContext) { + this(groupId, Optional.empty(), Optional.empty(), subscriptions, commitRequestManager, metadata, + logContext); } public MembershipManagerImpl(String groupId, - String groupInstanceId, - String serverAssignor, + Optional groupInstanceId, + Optional serverAssignor, + SubscriptionState subscriptions, + CommitRequestManager commitRequestManager, + ConsumerMetadata metadata, LogContext logContext) { this.groupId = groupId; - this.state = MemberState.UNJOINED; - this.serverAssignor = Optional.ofNullable(serverAssignor); - this.groupInstanceId = Optional.ofNullable(groupInstanceId); - this.targetAssignment = Optional.empty(); + this.state = MemberState.UNSUBSCRIBED; + this.serverAssignor = serverAssignor; + this.groupInstanceId = groupInstanceId; + this.subscriptions = subscriptions; + this.commitRequestManager = commitRequestManager; + this.metadata = metadata; + this.assignedTopicNamesCache = new HashMap<>(); + this.assignmentUnresolved = new HashMap<>(); + this.assignmentReadyToReconcile = new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR); + this.currentAssignment = new HashSet<>(); this.log = logContext.logger(MembershipManagerImpl.class); } @@ -113,7 +274,7 @@ public MembershipManagerImpl(String groupId, * nextState is not allowed as defined in {@link MemberState}. */ private void transitionTo(MemberState nextState) { - if (!this.state.equals(nextState) && !nextState.getPreviousValidStates().contains(state)) { + if (!state.equals(nextState) && !nextState.getPreviousValidStates().contains(state)) { throw new IllegalStateException(String.format("Invalid state transition from %s to %s", state, nextState)); } @@ -157,7 +318,7 @@ public int memberEpoch() { * {@inheritDoc} */ @Override - public void updateState(ConsumerGroupHeartbeatResponseData response) { + public void onHeartbeatResponseReceived(ConsumerGroupHeartbeatResponseData response) { if (response.errorCode() != Errors.NONE.code()) { String errorMessage = String.format( "Unexpected error in Heartbeat response. Expected no error, but received: %s", @@ -168,10 +329,35 @@ public void updateState(ConsumerGroupHeartbeatResponseData response) { this.memberId = response.memberId(); this.memberEpoch = response.memberEpoch(); ConsumerGroupHeartbeatResponseData.Assignment assignment = response.assignment(); + if (assignment != null) { - setTargetAssignment(assignment); + transitionTo(MemberState.RECONCILING); + replaceUnresolvedAssignmentWithNewAssignment(assignment); + resolveMetadataForUnresolvedAssignment(); + reconcile(); + } else if (allPendingAssignmentsReconciled()) { + transitionTo(MemberState.STABLE); } - maybeTransitionToStable(); + } + + /** + * Overwrite collection of unresolved topic Ids with the new target assignment. This will + * effectively achieve the following: + * + * - all topics received in assignment will try to be resolved to find their topic names + * + * - any topic received in a previous assignment that was still unresolved, and that is + * not included in the assignment anymore, will be removed from the unresolved collection. + * This should be the case when a topic is sent in an assignment, deleted right after, and + * removed from the assignment the next time a broker sends one to the member. + * + * @param assignment Target assignment received from the broker. + */ + private void replaceUnresolvedAssignmentWithNewAssignment( + ConsumerGroupHeartbeatResponseData.Assignment assignment) { + assignmentUnresolved.clear(); + assignment.topicPartitions().forEach(topicPartitions -> + assignmentUnresolved.put(topicPartitions.topicId(), topicPartitions.partitions())); } /** @@ -179,65 +365,650 @@ public void updateState(ConsumerGroupHeartbeatResponseData response) { */ @Override public void transitionToFenced() { - resetEpoch(); transitionTo(MemberState.FENCED); + resetEpoch(); + log.debug("Member {} with epoch {} transitioned to {} state. It will release its " + + "assignment and rejoin the group.", memberId, memberEpoch, MemberState.FENCED); + + // Release assignment + CompletableFuture callbackResult = invokeOnPartitionsLostCallback(subscriptions.assignedPartitions()); + callbackResult.whenComplete((result, error) -> { + if (error != null) { + log.error("onPartitionsLost callback invocation failed while releasing assignment" + + " after member got fenced. Member will rejoin the group anyways.", error); + } + updateSubscription(Collections.emptySet(), true); + transitionToJoining(); + }); } /** * {@inheritDoc} */ @Override - public void transitionToFailed() { - log.error("Member {} transitioned to {} state", memberId, MemberState.FAILED); - transitionTo(MemberState.FAILED); + public void transitionToFatal() { + transitionTo(MemberState.FATAL); + log.error("Member {} with epoch {} transitioned to {} state", memberId, memberEpoch, MemberState.FATAL); + + // Release assignment + CompletableFuture callbackResult = invokeOnPartitionsLostCallback(subscriptions.assignedPartitions()); + callbackResult.whenComplete((result, error) -> { + if (error != null) { + log.error("onPartitionsLost callback invocation failed while releasing assignment" + + "after member failed with fatal error.", error); + } + updateSubscription(Collections.emptySet(), true); + }); } + /** + * {@inheritDoc} + */ + public void onSubscriptionUpdated() { + if (state == MemberState.UNSUBSCRIBED) { + transitionToJoining(); + } + } + + /** + * Update a new assignment by setting the assigned partitions in the member subscription. + * + * @param assignedPartitions Topic partitions to take as the new subscription assignment + * @param clearAssignments True if the pending assignments and metadata cache should be cleared + */ + private void updateSubscription(Collection assignedPartitions, + boolean clearAssignments) { + subscriptions.assignFromSubscribed(assignedPartitions); + if (clearAssignments) { + clearPendingAssignmentsAndLocalNamesCache(); + } + } + + /** + * Transition to the {@link MemberState#JOINING} state, indicating that the member will + * try to join the group on the next heartbeat request. This is expected to be invoked when + * the user calls the subscribe API, or when the member wants to rejoin after getting fenced. + * Visible for testing. + */ + void transitionToJoining() { + if (state == MemberState.FATAL) { + log.warn("No action taken to join the group with the updated subscription because " + + "the member is in FATAL state"); + return; + } + resetEpoch(); + transitionTo(MemberState.JOINING); + clearPendingAssignmentsAndLocalNamesCache(); + registerForMetadataUpdates(); + } + + /** + * Register to get notified when the cluster metadata is updated, via the + * {@link #onUpdate(ClusterResource)}. Register only if the manager is not register already. + */ + private void registerForMetadataUpdates() { + if (!isRegisteredForMetadataUpdates) { + this.metadata.addClusterUpdateListener(this); + isRegisteredForMetadataUpdates = true; + } + } + + /** + * {@inheritDoc} + */ @Override - public boolean shouldSendHeartbeat() { - return state() != MemberState.FAILED; + public CompletableFuture leaveGroup() { + if (state == MemberState.UNSUBSCRIBED || state == MemberState.FATAL) { + // Member is not part of the group. No-op and return completed future to avoid + // unnecessary transitions. + return CompletableFuture.completedFuture(null); + } + + if (state == MemberState.PREPARE_LEAVING || state == MemberState.LEAVING) { + // Member already leaving. No-op and return existing leave group future that will + // complete when the ongoing leave operation completes. + return leaveGroupInProgress.get(); + } + + transitionTo(MemberState.PREPARE_LEAVING); + CompletableFuture leaveResult = new CompletableFuture<>(); + leaveGroupInProgress = Optional.of(leaveResult); + + CompletableFuture callbackResult = invokeOnPartitionsRevokedOrLostToReleaseAssignment(); + callbackResult.whenComplete((result, error) -> { + // Clear the subscription, no matter if the callback execution failed or succeeded. + updateSubscription(Collections.emptySet(), true); + + // Transition to ensure that a heartbeat request is sent out to effectively leave the + // group (even in the case where the member had no assignment to release or when the + // callback execution failed.) + transitionToSendingLeaveGroup(); + }); + + // Return future to indicate that the leave group is done when the callbacks + // complete, and the transition to send the heartbeat has been made. + return leaveResult; } /** - * Transition to {@link MemberState#STABLE} only if there are no target assignments left to - * reconcile. Transition to {@link MemberState#RECONCILING} otherwise. + * Release member assignment by calling the user defined callbacks for onPartitionsRevoked or + * onPartitionsLost. + *
      + *
    • If the member is part of the group (epoch > 0), this will invoke onPartitionsRevoked. + * This will be the case when releasing assignment because the member is intentionally + * leaving the group (after a call to unsubscribe)
      • + * + *
    • If the member is not part of the group (epoch <=0), this will invoke onPartitionsLost. + * This will be the case when releasing assignment after being fenced .
      • + *
    + * + * @return Future that will complete when the callback execution completes. */ - private boolean maybeTransitionToStable() { - if (!hasPendingTargetAssignment()) { - transitionTo(MemberState.STABLE); + private CompletableFuture invokeOnPartitionsRevokedOrLostToReleaseAssignment() { + SortedSet droppedPartitions = new TreeSet<>(TOPIC_PARTITION_COMPARATOR); + droppedPartitions.addAll(subscriptions.assignedPartitions()); + + CompletableFuture callbackResult; + if (droppedPartitions.isEmpty()) { + // No assignment to release + callbackResult = CompletableFuture.completedFuture(null); } else { - transitionTo(MemberState.RECONCILING); + // Release assignment + if (memberEpoch > 0) { + // Member is part of the group. Invoke onPartitionsRevoked. + callbackResult = revokePartitions(droppedPartitions); + } else { + // Member is not part of the group anymore. Invoke onPartitionsLost. + callbackResult = invokeOnPartitionsLostCallback(droppedPartitions); + } + } + return callbackResult; + } + + /** + * Reset member epoch to the value required for the leave the group heartbeat request, and + * transition to the {@link MemberState#LEAVING} state so that a heartbeat + * request is sent out with it. + */ + private void transitionToSendingLeaveGroup() { + memberEpoch = ConsumerGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH; + currentAssignment = new HashSet<>(); + transitionTo(MemberState.LEAVING); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean shouldHeartbeatNow() { + MemberState state = state(); + return state == MemberState.ACKNOWLEDGING || state == MemberState.LEAVING; + } + + /** + * {@inheritDoc} + */ + @Override + public void onHeartbeatRequestSent() { + MemberState state = state(); + if (state == MemberState.ACKNOWLEDGING) { + if (allPendingAssignmentsReconciled()) { + transitionTo(MemberState.STABLE); + } else { + log.debug("Member {} with epoch {} transitioned to {} after a heartbeat was sent " + + "to ack a previous reconciliation. New assignments are ready to " + + "be reconciled.", memberId, memberEpoch, MemberState.RECONCILING); + transitionTo(MemberState.RECONCILING); + } + } else if (state == MemberState.LEAVING) { + transitionToUnsubscribed(); } - return state.equals(MemberState.STABLE); } /** - * Take new target assignment received from the server and set it as targetAssignment to be - * processed. Following the consumer group protocol, the server won't send a new target - * member while a previous one hasn't been acknowledged by the member, so this will fail - * if a target assignment already exists. + * {@inheritDoc} + */ + @Override + public void onHeartbeatRequestSkipped() { + if (state == MemberState.LEAVING) { + log.debug("Heartbeat for leaving group could not be sent. Member {} with epoch {} will transition to {}.", + memberId, memberEpoch, MemberState.UNSUBSCRIBED); + transitionToUnsubscribed(); + } + } + + private void transitionToUnsubscribed() { + transitionTo(MemberState.UNSUBSCRIBED); + leaveGroupInProgress.get().complete(null); + leaveGroupInProgress = Optional.empty(); + } + + /** + * @return True if there are no assignments waiting to be resolved from metadata or reconciled. + */ + private boolean allPendingAssignmentsReconciled() { + return assignmentUnresolved.isEmpty() && assignmentReadyToReconcile.isEmpty(); + } + + @Override + public boolean shouldSkipHeartbeat() { + MemberState state = state(); + return state == MemberState.UNSUBSCRIBED || state == MemberState.FATAL; + } + + /** + * Reconcile the assignment that has been received from the server and for which topic names + * are resolved, kept in the {@link #assignmentReadyToReconcile}. This will commit if needed, + * trigger the callbacks and update the subscription state. Note that only one reconciliation + * can be in progress at a time. If there is already another one in progress when this is + * triggered, it will be no-op, and the assignment will be reconciled on the next + * reconciliation loop. + */ + boolean reconcile() { + if (reconciliationInProgress) { + log.debug("Ignoring reconciliation attempt. Another reconciliation is already in progress. Assignment " + + assignmentReadyToReconcile + " will be handled in the next reconciliation loop."); + return false; + } + + // Make copy of the assignment to reconcile as it could change as new assignments or metadata updates are received + SortedSet assignedTopicIdPartitions = new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR); + assignedTopicIdPartitions.addAll(assignmentReadyToReconcile); + + SortedSet ownedPartitions = new TreeSet<>(TOPIC_PARTITION_COMPARATOR); + ownedPartitions.addAll(subscriptions.assignedPartitions()); + + // Keep copy of assigned TopicPartitions created from the TopicIdPartitions that are + // being reconciled. Needed for interactions with the centralized subscription state that + // does not support topic IDs yet, and for the callbacks. + SortedSet assignedTopicPartitions = toTopicPartitionSet(assignedTopicIdPartitions); + + // Check same assignment. Based on topic names for now, until topic IDs are properly + // supported in the centralized subscription state object. + boolean sameAssignmentReceived = assignedTopicPartitions.equals(ownedPartitions); + + if (sameAssignmentReceived) { + log.debug("Ignoring reconciliation attempt. Target assignment ready to reconcile {} " + + "is equal to the member current assignment {}.", assignedTopicPartitions, ownedPartitions); + return false; + } + + markReconciliationInProgress(); + + // Partitions to assign (not previously owned) + SortedSet addedPartitions = new TreeSet<>(TOPIC_PARTITION_COMPARATOR); + addedPartitions.addAll(assignedTopicPartitions); + addedPartitions.removeAll(ownedPartitions); + + // Partitions to revoke + SortedSet revokedPartitions = new TreeSet<>(TOPIC_PARTITION_COMPARATOR); + revokedPartitions.addAll(ownedPartitions); + revokedPartitions.removeAll(assignedTopicPartitions); + + log.info("Updating assignment with\n" + + "\tAssigned partitions: {}\n" + + "\tCurrent owned partitions: {}\n" + + "\tAdded partitions (assigned - owned): {}\n" + + "\tRevoked partitions (owned - assigned): {}\n", + assignedTopicIdPartitions, + ownedPartitions, + addedPartitions, + revokedPartitions + ); + + CompletableFuture revocationResult; + if (!revokedPartitions.isEmpty()) { + revocationResult = revokePartitions(revokedPartitions); + } else { + revocationResult = CompletableFuture.completedFuture(null); + // Reschedule the auto commit starting from now (new assignment received without any + // revocation). + commitRequestManager.resetAutoCommitTimer(); + } + + // Future that will complete when the full reconciliation process completes (revocation + // and assignment, executed sequentially) + CompletableFuture reconciliationResult = + revocationResult.thenCompose(__ -> { + boolean memberHasRejoined = memberEpochOnReconciliationStart != memberEpoch; + if (state == MemberState.RECONCILING && !memberHasRejoined) { + // Apply assignment + CompletableFuture assignResult = assignPartitions(assignedTopicPartitions, + addedPartitions); + + // Clear topic names cache only for topics that are not in the subscription anymore + for (TopicPartition tp : revokedPartitions) { + if (!subscriptions.subscription().contains(tp.topic())) { + assignedTopicNamesCache.values().remove(tp.topic()); + } + } + return assignResult; + } else { + log.debug("Revocation callback completed but the member already " + + "transitioned out of the reconciling state for epoch {} into " + + "{} state with epoch {}. Interrupting reconciliation as it's " + + "not relevant anymore,", memberEpochOnReconciliationStart, state, memberEpoch); + String reason = interruptedReconciliationErrorMessage(); + CompletableFuture res = new CompletableFuture<>(); + res.completeExceptionally(new KafkaException("Interrupting reconciliation" + + " after revocation. " + reason)); + return res; + } + }); + + reconciliationResult.whenComplete((result, error) -> { + markReconciliationCompleted(); + if (error != null) { + // Leaving member in RECONCILING state after callbacks fail. The member + // won't send the ack, and the expectation is that the broker will kick the + // member out of the group after the rebalance timeout expires, leading to a + // RECONCILING -> FENCED transition. + log.error("Reconciliation failed.", error); + } else { + boolean memberHasRejoined = memberEpochOnReconciliationStart != memberEpoch; + if (state == MemberState.RECONCILING && !memberHasRejoined) { + // Make assignment effective on the broker by transitioning to send acknowledge. + transitionTo(MemberState.ACKNOWLEDGING); + + // Make assignment effective on the member group manager + currentAssignment = assignedTopicIdPartitions; + + // Indicate that we completed reconciling a subset of the assignment ready to + // reconcile (new assignments might have been received or discovered in + // metadata) + assignmentReadyToReconcile.removeAll(assignedTopicIdPartitions); + } else { + String reason = interruptedReconciliationErrorMessage(); + log.error("Interrupting reconciliation after partitions assigned callback " + + "completed. " + reason); + } + } + }); + + return true; + } + + /** + * Build set of {@link TopicPartition} from the given set of {@link TopicIdPartition}. + */ + private SortedSet toTopicPartitionSet(SortedSet topicIdPartitions) { + SortedSet result = new TreeSet<>(TOPIC_PARTITION_COMPARATOR); + topicIdPartitions.forEach(topicIdPartition -> result.add(topicIdPartition.topicPartition())); + return result; + } + + /** + * @return Reason for interrupting a reconciliation progress when callbacks complete. + */ + private String interruptedReconciliationErrorMessage() { + String reason; + if (state != MemberState.RECONCILING) { + reason = "The member already transitioned out of the reconciling state into " + state; + } else { + reason = "The member has re-joined the group."; + } + return reason; + } + + /** + * Visible for testing. + */ + void markReconciliationInProgress() { + reconciliationInProgress = true; + memberEpochOnReconciliationStart = memberEpoch; + } + + /** + * Visible for testing. + */ + void markReconciliationCompleted() { + reconciliationInProgress = false; + } + + /** + * Build set of TopicPartition (topic name and partition id) from the target assignment + * received from the broker (topic IDs and list of partitions). + * + *

    + * This will: + * + *

      + *
    1. Try to find topic names in the metadata cache
      2. + *
    2. For topics not found in metadata, try to find names in the local topic names cache + * (contains topic id and names currently assigned and resolved)
      3. + *
    3. If there are topics that are not in metadata cache or in the local cached + * of topic names assigned to this member, request a metadata update, and continue + * resolving names as the cache is updated. + *
      4. + *
    + */ + private void resolveMetadataForUnresolvedAssignment() { + // Try to resolve topic names from metadata cache or subscription cache, and move + // assignments from the "unresolved" collection, to the "readyToReconcile" one. + Iterator>> it = assignmentUnresolved.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> e = it.next(); + Uuid topicId = e.getKey(); + List topicPartitions = e.getValue(); + + Optional nameFromMetadata = findTopicNameInGlobalOrLocalCache(topicId); + nameFromMetadata.ifPresent(resolvedTopicName -> { + // Name resolved, so assignment is ready for reconciliation. + SortedSet topicIdPartitions = + buildAssignedPartitionsWithTopicName(topicId, resolvedTopicName, topicPartitions); + assignmentReadyToReconcile.addAll(topicIdPartitions); + it.remove(); + }); + } + + if (!assignmentUnresolved.isEmpty()) { + log.debug("Topic Ids {} received in target assignment were not found in metadata and " + + "are not currently assigned. Requesting a metadata update now to resolve " + + "topic names.", assignmentUnresolved.keySet()); + metadata.requestUpdate(true); + } + } + + /** + * Look for topic in the global metadata cache. If found, add it to the local cache and + * return it. If not found, look for it in the local metadata cache. Return empty if not + * found in any of the two. + */ + private Optional findTopicNameInGlobalOrLocalCache(Uuid topicId) { + String nameFromMetadataCache = metadata.topicNames().getOrDefault(topicId, null); + if (nameFromMetadataCache != null) { + // Add topic name to local cache, so it can be reused if included in a next target + // assignment if metadata cache not available. + assignedTopicNamesCache.put(topicId, nameFromMetadataCache); + return Optional.of(nameFromMetadataCache); + } else { + // Topic ID was not found in metadata. Check if the topic name is in the local + // cache of topics currently assigned. This will avoid a metadata request in the + // case where the metadata cache may have been flushed right before the + // revocation of a previously assigned topic. + String nameFromSubscriptionCache = assignedTopicNamesCache.getOrDefault(topicId, null); + return Optional.ofNullable(nameFromSubscriptionCache); + } + } + + /** + * Build set of TopicPartition for the partitions included in the heartbeat topicPartitions, + * and using the given topic name. + */ + private SortedSet buildAssignedPartitionsWithTopicName( + Uuid topicId, + String topicName, + List topicPartitions) { + SortedSet assignedPartitions = + new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR); + topicPartitions.forEach(tp -> { + TopicIdPartition topicIdPartition = new TopicIdPartition( + topicId, + new TopicPartition(topicName, tp)); + assignedPartitions.add(topicIdPartition); + }); + return assignedPartitions; + } + + /** + * Revoke partitions. This will: + *
      + *
    • Trigger an async commit offsets request if auto-commit enabled.
      • + *
    • Invoke the onPartitionsRevoked callback if the user has registered it.
      • + *
    + * + * This will wait on the commit request to finish before invoking the callback. If the commit + * request fails, this will proceed to invoke the user callbacks anyway, + * returning a future that will complete or fail depending on the callback execution only. + * + * @param revokedPartitions Partitions to revoke. + * @return Future that will complete when the commit request and user callback completes. + */ + private CompletableFuture revokePartitions(Set revokedPartitions) { + log.info("Revoking previously assigned partitions {}", Utils.join(revokedPartitions, ", ")); + + logPausedPartitionsBeingRevoked(revokedPartitions); + + // Mark partitions as pending revocation to stop fetching from the partitions (no new + // fetches sent out, and no in-flight fetches responses processed). + markPendingRevocationToPauseFetching(revokedPartitions); + + // Future that will complete when the revocation completes (including offset commit + // request and user callback execution) + CompletableFuture revocationResult = new CompletableFuture<>(); + + // Commit offsets if auto-commit enabled. + CompletableFuture commitResult; + if (commitRequestManager.autoCommitEnabled()) { + commitResult = commitRequestManager.maybeAutoCommitAllConsumed(); + } else { + commitResult = CompletableFuture.completedFuture(null); + } + + commitResult.whenComplete((result, error) -> { + if (error != null) { + // Commit request failed (commit request manager internally retries on + // retriable errors, so at this point we assume this is non-retriable, but + // proceed with the revocation anyway). + log.error("Commit request before revocation failed with non-retriable error. Will" + + " proceed with the revocation anyway.", error); + } + + CompletableFuture userCallbackResult = invokeOnPartitionsRevokedCallback(revokedPartitions); + userCallbackResult.whenComplete((callbackResult, callbackError) -> { + if (callbackError != null) { + log.error("onPartitionsRevoked callback invocation failed for partitions {}", + revokedPartitions, callbackError); + revocationResult.completeExceptionally(callbackError); + } else { + revocationResult.complete(null); + } + + }); + }); + return revocationResult; + } + + + /** + * Make new assignment effective and trigger onPartitionsAssigned callback for the partitions + * added. * - * @throws IllegalStateException If a target assignment already exists. + * @param assignedPartitions New assignment that will be updated in the member subscription + * state. + * @param addedPartitions Partitions contained in the new assignment that were not owned by + * the member before. These will be provided to the + * onPartitionsAssigned callback. + * @return Future that will complete when the callback execution completes. + */ + private CompletableFuture assignPartitions( + SortedSet assignedPartitions, + SortedSet addedPartitions) { + + // Make assignment effective on the client by updating the subscription state. + updateSubscription(assignedPartitions, false); + + // Invoke user call back + return invokeOnPartitionsAssignedCallback(addedPartitions); + } + + /** + * Mark partitions as 'pending revocation', to effectively stop fetching while waiting for + * the commit offsets request to complete, and ensure the application's position don't get + * ahead of the committed positions. This mark will ensure that: + *
      + *
    • No new fetches will be sent out for the partitions being revoked
      • + *
    • Previous in-flight fetch requests that may complete while the partitions are being revoked won't be processed.
      • + *
    */ - private void setTargetAssignment(ConsumerGroupHeartbeatResponseData.Assignment newTargetAssignment) { - if (!targetAssignment.isPresent()) { - log.info("Member {} accepted new target assignment {} to reconcile", memberId, newTargetAssignment); - targetAssignment = Optional.of(newTargetAssignment); + private void markPendingRevocationToPauseFetching(Set partitionsToRevoke) { + // When asynchronously committing offsets prior to the revocation of a set of partitions, there will be a + // window of time between when the offset commit is sent and when it returns and revocation completes. It is + // possible for pending fetches for these partitions to return during this time, which means the application's + // position may get ahead of the committed position prior to revocation. This can cause duplicate consumption. + // To prevent this, we mark the partitions as "pending revocation," which stops the Fetcher from sending new + // fetches or returning data from previous fetches to the user. + log.debug("Marking partitions pending for revocation: {}", partitionsToRevoke); + subscriptions.markPendingRevocation(partitionsToRevoke); + } + + private CompletableFuture invokeOnPartitionsRevokedCallback(Set partitionsRevoked) { + // This should not trigger the callback if partitionsRevoked is empty, to keep the + // current behaviour. + Optional listener = subscriptions.rebalanceListener(); + if (!partitionsRevoked.isEmpty() && listener.isPresent()) { + throw new UnsupportedOperationException("User-defined callbacks not supported yet"); } else { - transitionToFailed(); - throw new IllegalStateException("Cannot set new target assignment because a " + - "previous one pending to be reconciled already exists."); + return CompletableFuture.completedFuture(null); + } + } + + private CompletableFuture invokeOnPartitionsAssignedCallback(Set partitionsAssigned) { + // This should always trigger the callback, even if partitionsAssigned is empty, to keep + // the current behaviour. + Optional listener = subscriptions.rebalanceListener(); + if (listener.isPresent()) { + throw new UnsupportedOperationException("User-defined callbacks not supported yet"); + } else { + return CompletableFuture.completedFuture(null); + } + } + + private CompletableFuture invokeOnPartitionsLostCallback(Set partitionsLost) { + // This should not trigger the callback if partitionsLost is empty, to keep the current + // behaviour. + Optional listener = subscriptions.rebalanceListener(); + if (!partitionsLost.isEmpty() && listener.isPresent()) { + throw new UnsupportedOperationException("User-defined callbacks not supported yet"); + } else { + return CompletableFuture.completedFuture(null); + } + } + + /** + * Log partitions being revoked that were already paused, since the pause flag will be + * effectively lost. + */ + private void logPausedPartitionsBeingRevoked(Set partitionsToRevoke) { + Set revokePausedPartitions = subscriptions.pausedPartitions(); + revokePausedPartitions.retainAll(partitionsToRevoke); + if (!revokePausedPartitions.isEmpty()) { + log.info("The pause flag in partitions [{}] will be removed due to revocation.", Utils.join(revokePausedPartitions, ", ")); } } /** - * Returns true if the member has a target assignment being processed. + * Discard assignments received that have not been reconciled yet (waiting for metadata + * or the next reconciliation loop). Remove all elements from the topic names cache. */ - private boolean hasPendingTargetAssignment() { - return targetAssignment.isPresent(); + private void clearPendingAssignmentsAndLocalNamesCache() { + assignmentUnresolved.clear(); + assignmentReadyToReconcile.clear(); + assignedTopicNamesCache.clear(); } private void resetEpoch() { - this.memberEpoch = 0; + this.memberEpoch = ConsumerGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH; } /** @@ -260,42 +1031,51 @@ public Optional serverAssignor() { * {@inheritDoc} */ @Override - public ConsumerGroupHeartbeatResponseData.Assignment currentAssignment() { + public Set currentAssignment() { return this.currentAssignment; } /** - * @return Assignment that the member received from the server but hasn't completely processed - * yet. Visible for testing. + * @return Set of topic IDs received in a target assignment that have not been reconciled yet + * because topic names are not in metadata. Visible for testing. */ - Optional targetAssignment() { - return targetAssignment; + Set topicsWaitingForMetadata() { + return Collections.unmodifiableSet(assignmentUnresolved.keySet()); } /** - * This indicates that the reconciliation of the target assignment has been successfully - * completed, so it will make it effective by assigning it to the current assignment. - * - * @params Assignment that has been successfully reconciled. This is expected to - * match the target assignment defined in {@link #targetAssignment()} + * @return Topic partitions received in a target assignment that have been resolved in + * metadata and are ready to be reconciled. Visible for testing. + */ + Set assignmentReadyToReconcile() { + return Collections.unmodifiableSet(assignmentReadyToReconcile); + } + + /** + * @return If there is a reconciliation in process now. Note that reconciliation is triggered + * by a call to {@link #reconcile()}. Visible for testing. + */ + boolean reconciliationInProgress() { + return reconciliationInProgress; + } + + /** + * When cluster metadata is updated, try to resolve topic names for topic IDs received in + * assignment that hasn't been resolved yet. + *
      + *
    • Try to find topic names for all unresolved assignments
      • + *
    • Add discovered topic names to the local topic names cache
      • + *
    • If any topics are resolved, trigger a reconciliation process
      • + *
    • If some topics still remain unresolved, request another metadata update
      • + *
    */ @Override - public void onTargetAssignmentProcessComplete(ConsumerGroupHeartbeatResponseData.Assignment assignment) { - if (assignment == null) { - throw new IllegalArgumentException("Assignment cannot be null"); - } - if (!assignment.equals(targetAssignment.orElse(null))) { - // This could be simplified to remove the assignment param and just assume that what - // was reconciled was the targetAssignment, but keeping it explicit and failing fast - // here to uncover any issues in the interaction of the assignment processing logic - // and this. - throw new IllegalStateException(String.format("Reconciled assignment %s does not " + - "match the expected target assignment %s", assignment, - targetAssignment.orElse(null))); + public void onUpdate(ClusterResource clusterResource) { + resolveMetadataForUnresolvedAssignment(); + if (!assignmentReadyToReconcile.isEmpty()) { + transitionTo(MemberState.RECONCILING); + reconcile(); } - this.currentAssignment = assignment; - targetAssignment = Optional.empty(); - transitionTo(MemberState.STABLE); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index 83f81ed4e8631..141f5f955c8b5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -30,6 +30,7 @@ import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; @@ -203,11 +204,6 @@ public Node leastLoadedNode() { return this.client.leastLoadedNode(time.milliseconds()); } - public void send(final UnsentRequest r) { - r.setTimer(this.time, this.requestTimeoutMs); - unsentRequests.add(r); - } - public void wakeup() { client.wakeup(); } @@ -225,19 +221,25 @@ public void close() throws IOException { } public long addAll(PollResult pollResult) { + Objects.requireNonNull(pollResult); addAll(pollResult.unsentRequests); return pollResult.timeUntilNextPollMs; } public void addAll(final List requests) { + Objects.requireNonNull(requests); if (!requests.isEmpty()) { - requests.forEach(ur -> ur.setTimer(time, requestTimeoutMs)); - unsentRequests.addAll(requests); + requests.forEach(this::add); } } - public static class PollResult { + public void add(final UnsentRequest r) { + Objects.requireNonNull(r); + r.setTimer(this.time, this.requestTimeoutMs); + unsentRequests.add(r); + } + public static class PollResult { public static final long WAIT_FOREVER = Long.MAX_VALUE; public static final PollResult EMPTY = new PollResult(WAIT_FOREVER); public final long timeUntilNextPollMs; @@ -265,6 +267,7 @@ public static class UnsentRequest { private final AbstractRequest.Builder requestBuilder; private final FutureCompletionHandler handler; private final Optional node; // empty if random node can be chosen + private Timer timer; public UnsentRequest(final AbstractRequest.Builder requestBuilder, @@ -279,6 +282,10 @@ void setTimer(final Time time, final long requestTimeoutMs) { this.timer = time.timer(requestTimeoutMs); } + Timer timer() { + return timer; + } + CompletableFuture future() { return handler.future; } @@ -322,7 +329,11 @@ public static class FutureCompletionHandler implements RequestCompletionHandler public void onFailure(final long currentTimeMs, final RuntimeException e) { this.responseCompletionTimeMs = currentTimeMs; - this.future.completeExceptionally(e); + if (e != null) { + this.future.completeExceptionally(e); + } else { + this.future.completeExceptionally(DisconnectException.INSTANCE); + } } public long completionTimeMs() { @@ -359,7 +370,8 @@ public static Supplier supplier(final Time time, final ConsumerConfig config, final ApiVersions apiVersions, final Metrics metrics, - final FetchMetricsManager fetchMetricsManager) { + final FetchMetricsManager fetchMetricsManager, + final ClientTelemetrySender clientTelemetrySender) { return new CachedSupplier() { @Override protected NetworkClientDelegate create() { @@ -371,7 +383,8 @@ protected NetworkClientDelegate create() { time, CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, metadata, - fetchMetricsManager.throttleTimeSensor()); + fetchMetricsManager.throttleTimeSensor(), + clientTelemetrySender); return new NetworkClientDelegate(time, config, logContext, client); } }; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 8592035dda27d..0cda89aedfb76 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -62,4 +62,19 @@ public interface RequestManager { default PollResult pollOnClose() { return EMPTY; } + + /** + * Returns the delay for which the application thread can safely wait before it should be responsive + * to results from the request managers. For example, the subscription state can change when heartbeats + * are sent, so blocking for longer than the heartbeat interval might mean the application thread is not + * responsive to changes. + * + * @param currentTimeMs The current system time at which the method was called; useful for determining if + * time-sensitive operations should be performed + * + * @return The maximum delay in milliseconds + */ + default long maximumTimeToWait(long currentTimeMs) { + return Long.MAX_VALUE; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 8f9efe35e3919..f170cc3b08f03 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -49,6 +49,7 @@ public class RequestManagers implements Closeable { public final Optional coordinatorRequestManager; public final Optional commitRequestManager; public final Optional heartbeatRequestManager; + public final Optional membershipManager; public final OffsetsRequestManager offsetsRequestManager; public final TopicMetadataRequestManager topicMetadataRequestManager; public final FetchRequestManager fetchRequestManager; @@ -61,7 +62,8 @@ public RequestManagers(LogContext logContext, FetchRequestManager fetchRequestManager, Optional coordinatorRequestManager, Optional commitRequestManager, - Optional heartbeatRequestManager) { + Optional heartbeatRequestManager, + Optional membershipManager) { this.log = logContext.logger(RequestManagers.class); this.offsetsRequestManager = requireNonNull(offsetsRequestManager, "OffsetsRequestManager cannot be null"); this.coordinatorRequestManager = coordinatorRequestManager; @@ -69,6 +71,7 @@ public RequestManagers(LogContext logContext, this.topicMetadataRequestManager = topicMetadataRequestManager; this.fetchRequestManager = fetchRequestManager; this.heartbeatRequestManager = heartbeatRequestManager; + this.membershipManager = membershipManager; List> list = new ArrayList<>(); list.add(coordinatorRequestManager); @@ -149,6 +152,7 @@ protected RequestManagers create() { logContext, config); HeartbeatRequestManager heartbeatRequestManager = null; + MembershipManager membershipManager = null; CoordinatorRequestManager coordinator = null; CommitRequestManager commit = null; @@ -160,8 +164,19 @@ protected RequestManagers create() { retryBackoffMaxMs, backgroundEventHandler, groupState.groupId); - commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState); - MembershipManager membershipManager = new MembershipManagerImpl(groupState.groupId, logContext); + commit = new CommitRequestManager(time, + logContext, + subscriptions, + config, + coordinator, + backgroundEventHandler, + groupState); + membershipManager = new MembershipManagerImpl( + groupState.groupId, + subscriptions, + commit, + metadata, + logContext); heartbeatRequestManager = new HeartbeatRequestManager( logContext, time, @@ -179,7 +194,8 @@ protected RequestManagers create() { fetch, Optional.ofNullable(coordinator), Optional.ofNullable(commit), - Optional.ofNullable(heartbeatRequestManager) + Optional.ofNullable(heartbeatRequestManager), + Optional.ofNullable(membershipManager) ); } }; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestState.java index 449d5a471cc9a..321be5e8fe808 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestState.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestState.java @@ -78,7 +78,7 @@ public boolean canSendRequest(final long currentTimeMs) { return true; } - if (this.lastReceivedMs == -1 || this.lastReceivedMs < this.lastSentMs) { + if (requestInFlight()) { log.trace("An inflight request already exists for {}", this); return false; } @@ -93,6 +93,14 @@ public boolean canSendRequest(final long currentTimeMs) { } } + /** + * @return True if no response has been received after the last send, indicating that there + * is a request in-flight. + */ + public boolean requestInFlight() { + return this.lastSentMs > -1 && this.lastReceivedMs < this.lastSentMs; + } + public void onSendAttempt(final long currentTimeMs) { // Here we update the timer everytime we try to send a request. Also increment number of attempts. this.lastSentMs = currentTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Utils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Utils.java index 2954a5df902f5..7d7d788a8dd4b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Utils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Utils.java @@ -16,11 +16,13 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; + import java.io.Serializable; import java.util.Comparator; import java.util.List; import java.util.Map; -import org.apache.kafka.common.TopicPartition; public final class Utils { @@ -59,4 +61,23 @@ public int compare(TopicPartition topicPartition1, TopicPartition topicPartition } } } + + public final static class TopicIdPartitionComparator implements Comparator, Serializable { + private static final long serialVersionUID = 1L; + + /** + * Comparison based on topic name and partition number. + */ + @Override + public int compare(TopicIdPartition topicPartition1, TopicIdPartition topicPartition2) { + String topic1 = topicPartition1.topic(); + String topic2 = topicPartition2.topic(); + + if (topic1.equals(topic2)) { + return topicPartition1.partition() - topicPartition2.partition(); + } else { + return topic1.compareTo(topic2); + } + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/WakeupTrigger.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/WakeupTrigger.java index aaaf0b92ac1ae..5a030f6307db9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/WakeupTrigger.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/WakeupTrigger.java @@ -21,6 +21,7 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** @@ -44,6 +45,10 @@ public void wakeup() { ActiveFuture active = (ActiveFuture) task; active.future().completeExceptionally(new WakeupException()); return null; + } else if (task instanceof FetchAction) { + FetchAction fetchAction = (FetchAction) task; + fetchAction.fetchBuffer().wakeup(); + return new WakeupFuture(); } else { return task; } @@ -75,17 +80,51 @@ public CompletableFuture setActiveTask(final CompletableFuture current return currentTask; } - public void clearActiveTask() { + public void setFetchAction(final FetchBuffer fetchBuffer) { + final AtomicBoolean throwWakeupException = new AtomicBoolean(false); pendingTask.getAndUpdate(task -> { if (task == null) { + return new FetchAction(fetchBuffer); + } else if (task instanceof WakeupFuture) { + throwWakeupException.set(true); return null; - } else if (task instanceof ActiveFuture) { + } + // last active state is still active + throw new IllegalStateException("Last active task is still active"); + }); + if (throwWakeupException.get()) { + throw new WakeupException(); + } + } + + public void clearTask() { + pendingTask.getAndUpdate(task -> { + if (task == null) { + return null; + } else if (task instanceof ActiveFuture || task instanceof FetchAction) { return null; } return task; }); } + public void maybeTriggerWakeup() { + final AtomicBoolean throwWakeupException = new AtomicBoolean(false); + pendingTask.getAndUpdate(task -> { + if (task == null) { + return null; + } else if (task instanceof WakeupFuture) { + throwWakeupException.set(true); + return null; + } else { + return task; + } + }); + if (throwWakeupException.get()) { + throw new WakeupException(); + } + } + Wakeupable getPendingTask() { return pendingTask.get(); } @@ -105,4 +144,17 @@ public CompletableFuture future() { } static class WakeupFuture implements Wakeupable { } + + static class FetchAction implements Wakeupable { + + private final FetchBuffer fetchBuffer; + + public FetchAction(FetchBuffer fetchBuffer) { + this.fetchBuffer = fetchBuffer; + } + + public FetchBuffer fetchBuffer() { + return fetchBuffer; + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java index 133836da3b753..552c283b43fbc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java @@ -25,7 +25,8 @@ public abstract class ApplicationEvent { public enum Type { COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, - LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA + LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA, SUBSCRIPTION_CHANGE, + UNSUBSCRIBE } private final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index 2917d507d7f02..f564a80d47f7f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -81,6 +81,18 @@ public void wakeupNetworkThread() { networkThread.wakeup(); } + /** + * Returns the delay for which the application thread can safely wait before it should be responsive + * to results from the request managers. For example, the subscription state can change when heartbeats + * are sent, so blocking for longer than the heartbeat interval might mean the application thread is not + * responsive to changes. + * + * @return The maximum delay in milliseconds + */ + public long maximumTimeToWait() { + return networkThread.maximumTimeToWait(); + } + /** * Add a {@link CompletableApplicationEvent} to the handler. The method blocks waiting for the result, and will * return the result value upon successful completion; otherwise throws an error. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index ccbdd21b9dc24..e076c46cd36c7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.internals.CommitRequestManager; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; +import org.apache.kafka.clients.consumer.internals.MembershipManager; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; @@ -104,6 +105,14 @@ public void process(ApplicationEvent event) { processValidatePositionsEvent(); return; + case SUBSCRIPTION_CHANGE: + processSubscriptionChangeEvent(); + return; + + case UNSUBSCRIBE: + processUnsubscribeEvent((UnsubscribeApplicationEvent) event); + return; + default: log.warn("Application event type " + event.type() + " was not expected"); } @@ -166,6 +175,38 @@ private void process(final ListOffsetsApplicationEvent event) { event.chain(future); } + /** + * Process event that indicates that the subscription changed. This will make the + * consumer join the group if it is not part of it yet, or send the updated subscription if + * it is already a member. + */ + private void processSubscriptionChangeEvent() { + if (!requestManagers.membershipManager.isPresent()) { + throw new RuntimeException("Group membership manager not present when processing a " + + "subscribe event"); + } + MembershipManager membershipManager = requestManagers.membershipManager.get(); + membershipManager.onSubscriptionUpdated(); + } + + /** + * Process event indicating that the consumer unsubscribed from all topics. This will make + * the consumer release its assignment and send a request to leave the group. + * + * @param event Unsubscribe event containing a future that will complete when the callback + * execution for releasing the assignment completes, and the request to leave + * the group is sent out. + */ + private void processUnsubscribeEvent(UnsubscribeApplicationEvent event) { + if (!requestManagers.membershipManager.isPresent()) { + throw new RuntimeException("Group membership manager not present when processing an " + + "unsubscribe event"); + } + MembershipManager membershipManager = requestManagers.membershipManager.get(); + CompletableFuture result = membershipManager.leaveGroup(); + event.chain(result); + } + private void processResetPositionsEvent() { requestManagers.offsetsRequestManager.resetPositionsIfNeeded(); } @@ -176,7 +217,7 @@ private void processValidatePositionsEvent() { private void process(final TopicMetadataApplicationEvent event) { final CompletableFuture>> future = - this.requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic())); + requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic())); event.chain(future); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java index a7dc3e454a776..0e44fe032fe60 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java @@ -27,6 +27,7 @@ public abstract class BackgroundEvent { public enum Type { ERROR, + GROUP_METADATA_UPDATE } protected final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java deleted file mode 100644 index cafd4fba492ec..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.utils.LogContext; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicReference; - -/** - * An {@link EventProcessor} that is created and executes in the application thread for the purpose of processing - * {@link BackgroundEvent background events} generated by the {@link ConsumerNetworkThread network thread}. - * Those events are generally of two types: - * - *
      - *
    • Errors that occur in the network thread that need to be propagated to the application thread
      • - *
    • {@link ConsumerRebalanceListener} callbacks that are to be executed on the application thread
      • - *
    - */ -public class BackgroundEventProcessor extends EventProcessor { - - public BackgroundEventProcessor(final LogContext logContext, - final BlockingQueue backgroundEventQueue) { - super(logContext, backgroundEventQueue); - } - - /** - * Process the events—if any—that were produced by the {@link ConsumerNetworkThread network thread}. - * It is possible that {@link ErrorBackgroundEvent an error} could occur when processing the events. - * In such cases, the processor will take a reference to the first error, continue to process the - * remaining events, and then throw the first error that occurred. - */ - @Override - public void process() { - AtomicReference firstError = new AtomicReference<>(); - process((event, error) -> firstError.compareAndSet(null, error)); - - if (firstError.get() != null) - throw firstError.get(); - } - - @Override - public void process(final BackgroundEvent event) { - if (event.type() == BackgroundEvent.Type.ERROR) - process((ErrorBackgroundEvent) event); - else - throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); - } - - @Override - protected Class getEventClass() { - return BackgroundEvent.class; - } - - private void process(final ErrorBackgroundEvent event) { - throw event.error(); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/GroupMetadataUpdateEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/GroupMetadataUpdateEvent.java new file mode 100644 index 0000000000000..120e671724209 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/GroupMetadataUpdateEvent.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; + +import java.util.Objects; + +/** + * This event is sent by the {@link ConsumerNetworkThread consumer's network thread} to the application thread + * so that when the user calls the {@link Consumer#groupMetadata()} API, the information is up-to-date. The + * information for the current state of the group member is managed on the consumer network thread and thus + * requires this interplay between threads. + */ +public class GroupMetadataUpdateEvent extends BackgroundEvent { + + final private int memberEpoch; + final private String memberId; + + public GroupMetadataUpdateEvent(final int memberEpoch, + final String memberId) { + super(Type.GROUP_METADATA_UPDATE); + this.memberEpoch = memberEpoch; + this.memberId = memberId; + } + + public int memberEpoch() { + return memberEpoch; + } + + public String memberId() { + return memberId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + GroupMetadataUpdateEvent that = (GroupMetadataUpdateEvent) o; + return memberEpoch == that.memberEpoch && + Objects.equals(memberId, that.memberId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), memberEpoch, memberId); + } + + @Override + public String toStringBase() { + return super.toStringBase() + + ", memberEpoch=" + memberEpoch + + ", memberId='" + memberId + '\''; + } + + @Override + public String toString() { + return "GroupMetadataUpdateEvent{" + + toStringBase() + + '}'; + } + +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/SubscriptionChangeApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/SubscriptionChangeApplicationEvent.java new file mode 100644 index 0000000000000..73fd15fb14408 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/SubscriptionChangeApplicationEvent.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.consumer.internals.events; + +/** + * Application event indicating that the subscription state has changed, triggered when a user + * calls the subscribe API. This will make the consumer join a consumer group if not part of it + * yet, or just send the updated subscription to the broker if it's already a member of the group. + */ +public class SubscriptionChangeApplicationEvent extends ApplicationEvent { + + public SubscriptionChangeApplicationEvent() { + super(Type.SUBSCRIPTION_CHANGE); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/UnsubscribeApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/UnsubscribeApplicationEvent.java new file mode 100644 index 0000000000000..a1ccb896fdf57 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/UnsubscribeApplicationEvent.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.consumer.internals.events; + +/** + * Application event triggered when a user calls the unsubscribe API. This will make the consumer + * release all its assignments and send a heartbeat request to leave the consumer group. + * This event holds a future that will complete when the invocation of callbacks to release + * complete and the heartbeat to leave the group is sent out (minimal effort to send the + * leave group heartbeat, without waiting for any response or considering timeouts). + */ +public class UnsubscribeApplicationEvent extends CompletableApplicationEvent { + public UnsubscribeApplicationEvent() { + super(Type.UNSUBSCRIBE); + } +} + diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 2b9b4c2b05a47..94273358e7d92 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -66,10 +66,13 @@ import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryUtils; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; @@ -80,6 +83,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -257,6 +261,7 @@ public class KafkaProducer implements Producer { private final ProducerInterceptors interceptors; private final ApiVersions apiVersions; private final TransactionManager transactionManager; + private final Optional clientTelemetryReporter; /** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings @@ -364,6 +369,8 @@ private void warnIfPartitionerDeprecated() { .recordLevel(Sensor.RecordingLevel.forName(config.getString(ProducerConfig.METRICS_RECORDING_LEVEL_CONFIG))) .tags(metricTags); List reporters = CommonClientConfigs.metricsReporters(clientId, config); + this.clientTelemetryReporter = CommonClientConfigs.telemetryReporter(clientId, config); + this.clientTelemetryReporter.ifPresent(reporters::add); MetricsContext metricsContext = new KafkaMetricsContext(JMX_PREFIX, config.originalsWithPrefix(CommonClientConfigs.METRICS_CONTEXT_PREFIX)); this.metrics = new Metrics(metricConfig, reporters, time, metricsContext); @@ -480,7 +487,8 @@ private void warnIfPartitionerDeprecated() { ProducerInterceptors interceptors, Partitioner partitioner, Time time, - KafkaThread ioThread) { + KafkaThread ioThread, + Optional clientTelemetryReporter) { this.producerConfig = config; this.time = time; this.clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); @@ -503,6 +511,7 @@ private void warnIfPartitionerDeprecated() { this.metadata = metadata; this.sender = sender; this.ioThread = ioThread; + this.clientTelemetryReporter = clientTelemetryReporter; } // visible for testing @@ -519,7 +528,8 @@ Sender newSender(LogContext logContext, KafkaClient kafkaClient, ProducerMetadat time, maxInflightRequests, metadata, - throttleTimeSensor); + throttleTimeSensor, + clientTelemetryReporter.map(ClientTelemetryReporter::telemetrySender).orElse(null)); short acks = Short.parseShort(producerConfig.getString(ProducerConfig.ACKS_CONFIG)); return new Sender(logContext, @@ -1280,7 +1290,11 @@ public List partitionsFor(String topic) { */ @Override public Uuid clientInstanceId(Duration timeout) { - throw new UnsupportedOperationException(); + if (!clientTelemetryReporter.isPresent()) { + throw new IllegalStateException("Telemetry is not enabled. Set config `" + ProducerConfig.ENABLE_METRICS_PUSH_CONFIG + "` to `true`."); + } + + return ClientTelemetryUtils.fetchClientInstanceId(clientTelemetryReporter.get(), timeout); } /** @@ -1341,16 +1355,22 @@ private void close(Duration timeout, boolean swallowException) { timeoutMs); } else { // Try to close gracefully. - if (this.sender != null) + final Timer closeTimer = time.timer(timeout); + if (this.sender != null) { this.sender.initiateClose(); + closeTimer.update(); + } if (this.ioThread != null) { try { - this.ioThread.join(timeoutMs); + this.ioThread.join(closeTimer.remainingMs()); } catch (InterruptedException t) { firstException.compareAndSet(null, new InterruptException(t)); log.error("Interrupted while joining ioThread", t); + } finally { + closeTimer.update(); } } + clientTelemetryReporter.ifPresent(reporter -> reporter.initiateClose(closeTimer.remainingMs())); } } @@ -1374,6 +1394,7 @@ private void close(Duration timeout, boolean swallowException) { Utils.closeQuietly(keySerializer, "producer keySerializer", firstException); Utils.closeQuietly(valueSerializer, "producer valueSerializer", firstException); Utils.closeQuietly(partitioner, "producer partitioner", firstException); + clientTelemetryReporter.ifPresent(reporter -> Utils.closeQuietly(reporter, "producer telemetry reporter", firstException)); AppInfoParser.unregisterAppInfo(JMX_PREFIX, clientId, metrics); Throwable exception = firstException.get(); if (exception != null && !swallowException) { diff --git a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java index e4e83387544c8..b680321ef00a1 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java +++ b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java @@ -38,6 +38,10 @@ public class BrokerSecurityConfigs { public static final String CONNECTIONS_MAX_REAUTH_MS = "connections.max.reauth.ms"; public static final int DEFAULT_SASL_SERVER_MAX_RECEIVE_SIZE = 524288; public static final String SASL_SERVER_MAX_RECEIVE_SIZE_CONFIG = "sasl.server.max.receive.size"; + public static final String SSL_ALLOW_DN_CHANGES_CONFIG = "ssl.allow.dn.changes"; + public static final boolean DEFAULT_SSL_ALLOW_DN_CHANGES_VALUE = false; + public static final String SSL_ALLOW_SAN_CHANGES_CONFIG = "ssl.allow.san.changes"; + public static final boolean DEFAULT_SSL_ALLOW_SAN_CHANGES_VALUE = false; public static final String PRINCIPAL_BUILDER_CLASS_DOC = "The fully qualified name of a class that implements the " + "KafkaPrincipalBuilder interface, which is used to build the KafkaPrincipal object used during " + @@ -95,4 +99,10 @@ public class BrokerSecurityConfigs { public static final String SASL_SERVER_MAX_RECEIVE_SIZE_DOC = "The maximum receive size allowed before and during initial SASL authentication." + " Default receive size is 512KB. GSSAPI limits requests to 64K, but we allow upto 512KB by default for custom SASL mechanisms. In practice," + " PLAIN, SCRAM and OAUTH mechanisms can use much smaller limits."; + + public static final String SSL_ALLOW_DN_CHANGES_DOC = "Indicates whether changes to the certificate distinguished name should be allowed during" + + " a dynamic reconfiguration of certificates or not."; + + public static final String SSL_ALLOW_SAN_CHANGES_DOC = "Indicates whether changes to the certificate subject alternative names should be allowed during " + + "a dynamic reconfiguration of certificates or not."; } diff --git a/clients/src/main/java/org/apache/kafka/common/errors/TelemetryTooLargeException.java b/clients/src/main/java/org/apache/kafka/common/errors/TelemetryTooLargeException.java new file mode 100644 index 0000000000000..4be6ef50a4b83 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/TelemetryTooLargeException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.errors; + +/** + * This exception indicates that the size of the telemetry metrics data is too large. + */ +public class TelemetryTooLargeException extends ApiException { + + public TelemetryTooLargeException(String message) { + super(message); + } +} + diff --git a/clients/src/main/java/org/apache/kafka/common/errors/UnknownSubscriptionIdException.java b/clients/src/main/java/org/apache/kafka/common/errors/UnknownSubscriptionIdException.java new file mode 100644 index 0000000000000..e2041db0ed9d1 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/errors/UnknownSubscriptionIdException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.errors; + +/** + * This exception indicates that the client sent an invalid or outdated SubscriptionId + */ +public class UnknownSubscriptionIdException extends ApiException { + + public UnknownSubscriptionIdException(String message) { + super(message); + } +} + diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 1635da2e200da..a5c6ef5ea832a 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -116,7 +116,8 @@ public enum ApiKeys { CONTROLLER_REGISTRATION(ApiMessageType.CONTROLLER_REGISTRATION), GET_TELEMETRY_SUBSCRIPTIONS(ApiMessageType.GET_TELEMETRY_SUBSCRIPTIONS), PUSH_TELEMETRY(ApiMessageType.PUSH_TELEMETRY), - ASSIGN_REPLICAS_TO_DIRS(ApiMessageType.ASSIGN_REPLICAS_TO_DIRS); + ASSIGN_REPLICAS_TO_DIRS(ApiMessageType.ASSIGN_REPLICAS_TO_DIRS), + LIST_CLIENT_METRICS_RESOURCES(ApiMessageType.LIST_CLIENT_METRICS_RESOURCES); private static final Map> APIS_BY_LISTENER = new EnumMap<>(ApiMessageType.ListenerType.class); diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java index e2d57278ef881..8b4c78f890cff 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Errors.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; import org.apache.kafka.common.errors.BrokerNotAvailableException; +import org.apache.kafka.common.errors.TelemetryTooLargeException; import org.apache.kafka.common.errors.ClusterAuthorizationException; import org.apache.kafka.common.errors.ConcurrentTransactionsException; import org.apache.kafka.common.errors.ControllerMovedException; @@ -118,6 +119,7 @@ import org.apache.kafka.common.errors.TransactionCoordinatorFencedException; import org.apache.kafka.common.errors.TransactionalIdAuthorizationException; import org.apache.kafka.common.errors.TransactionalIdNotFoundException; +import org.apache.kafka.common.errors.UnknownSubscriptionIdException; import org.apache.kafka.common.errors.UnacceptableCredentialException; import org.apache.kafka.common.errors.UnknownControllerIdException; import org.apache.kafka.common.errors.UnknownLeaderEpochException; @@ -386,7 +388,9 @@ public enum Errors { STALE_MEMBER_EPOCH(113, "The member epoch is stale. The member must retry after receiving its updated member epoch via the ConsumerGroupHeartbeat API.", StaleMemberEpochException::new), MISMATCHED_ENDPOINT_TYPE(114, "The request was sent to an endpoint of the wrong type.", MismatchedEndpointTypeException::new), UNSUPPORTED_ENDPOINT_TYPE(115, "This endpoint type is not supported yet.", UnsupportedEndpointTypeException::new), - UNKNOWN_CONTROLLER_ID(116, "This controller ID is not known.", UnknownControllerIdException::new); + UNKNOWN_CONTROLLER_ID(116, "This controller ID is not known.", UnknownControllerIdException::new), + UNKNOWN_SUBSCRIPTION_ID(117, "Client sent a push telemetry request with an invalid or outdated subscription ID.", UnknownSubscriptionIdException::new), + TELEMETRY_TOO_LARGE(118, "Client sent a push telemetry request larger than the maximum size the broker will accept.", TelemetryTooLargeException::new); private static final Logger log = LoggerFactory.getLogger(Errors.class); diff --git a/clients/src/main/java/org/apache/kafka/common/record/ConvertedRecords.java b/clients/src/main/java/org/apache/kafka/common/record/ConvertedRecords.java index d9150e5044db6..79ce2c83f2894 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/ConvertedRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/ConvertedRecords.java @@ -19,18 +19,18 @@ public class ConvertedRecords { private final T records; - private final RecordConversionStats recordConversionStats; + private final RecordValidationStats recordValidationStats; - public ConvertedRecords(T records, RecordConversionStats recordConversionStats) { + public ConvertedRecords(T records, RecordValidationStats recordValidationStats) { this.records = records; - this.recordConversionStats = recordConversionStats; + this.recordValidationStats = recordValidationStats; } public T records() { return records; } - public RecordConversionStats recordConversionStats() { - return recordConversionStats; + public RecordValidationStats recordConversionStats() { + return recordValidationStats; } } diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java index 6ff9b39096505..23a88c277c969 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java +++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java @@ -284,7 +284,7 @@ public ConvertedRecords downConvert(byte toMagic, long firstO // are not enough available bytes in the response to read it fully. Note that this is // only possible prior to KIP-74, after which the broker was changed to always return at least // one full record batch, even if it requires exceeding the max fetch size requested by the client. - return new ConvertedRecords<>(this, RecordConversionStats.EMPTY); + return new ConvertedRecords<>(this, RecordValidationStats.EMPTY); } else { return convertedRecords; } diff --git a/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecordsSend.java b/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecordsSend.java index f5f8dcecb67d3..001158b5a9ae3 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecordsSend.java +++ b/clients/src/main/java/org/apache/kafka/common/record/LazyDownConversionRecordsSend.java @@ -36,14 +36,14 @@ public final class LazyDownConversionRecordsSend extends RecordsSend> convertedRecordsIterator; public LazyDownConversionRecordsSend(LazyDownConversionRecords records) { super(records, records.sizeInBytes()); convertedRecordsWriter = null; - recordConversionStats = new RecordConversionStats(); + recordValidationStats = new RecordValidationStats(); convertedRecordsIterator = records().iterator(MAX_READ_SIZE); } @@ -77,7 +77,7 @@ public int writeTo(TransferableChannel channel, int previouslyWritten, int remai // Get next chunk of down-converted messages ConvertedRecords recordsAndStats = convertedRecordsIterator.next(); convertedRecords = (MemoryRecords) recordsAndStats.records(); - recordConversionStats.add(recordsAndStats.recordConversionStats()); + recordValidationStats.add(recordsAndStats.recordConversionStats()); log.debug("Down-converted records for partition {} with length={}", topicPartition(), convertedRecords.sizeInBytes()); } else { convertedRecords = buildOverflowBatch(remaining); @@ -97,8 +97,8 @@ public int writeTo(TransferableChannel channel, int previouslyWritten, int remai return (int) convertedRecordsWriter.writeTo(channel); } - public RecordConversionStats recordConversionStats() { - return recordConversionStats; + public RecordValidationStats recordConversionStats() { + return recordValidationStats; } public TopicPartition topicPartition() { diff --git a/clients/src/main/java/org/apache/kafka/common/record/MultiRecordsSend.java b/clients/src/main/java/org/apache/kafka/common/record/MultiRecordsSend.java index e12cc58e00e0e..929b16467c12d 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/MultiRecordsSend.java +++ b/clients/src/main/java/org/apache/kafka/common/record/MultiRecordsSend.java @@ -37,7 +37,7 @@ public class MultiRecordsSend implements Send { private final Queue sendQueue; private final long size; - private Map recordConversionStats; + private Map recordConversionStats; private long totalWritten = 0; private Send current; @@ -114,7 +114,7 @@ public long writeTo(TransferableChannel channel) throws IOException { * Get any statistics that were recorded as part of executing this {@link MultiRecordsSend}. * @return Records processing statistics (could be null if no statistics were collected) */ - public Map recordConversionStats() { + public Map recordConversionStats() { return recordConversionStats; } diff --git a/clients/src/main/java/org/apache/kafka/common/record/RecordConversionStats.java b/clients/src/main/java/org/apache/kafka/common/record/RecordValidationStats.java similarity index 78% rename from clients/src/main/java/org/apache/kafka/common/record/RecordConversionStats.java rename to clients/src/main/java/org/apache/kafka/common/record/RecordValidationStats.java index 4f0bca527fb97..4dfcf66c0aeea 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/RecordConversionStats.java +++ b/clients/src/main/java/org/apache/kafka/common/record/RecordValidationStats.java @@ -16,25 +16,30 @@ */ package org.apache.kafka.common.record; -public class RecordConversionStats { +/** + * This class tracks resource usage during broker record validation for eventual reporting in metrics. + * Record validation covers integrity checks on inbound data (e.g. checksum verification), structural + * validation to make sure that records are well-formed, and conversion between record formats if needed. + */ +public class RecordValidationStats { - public static final RecordConversionStats EMPTY = new RecordConversionStats(); + public static final RecordValidationStats EMPTY = new RecordValidationStats(); private long temporaryMemoryBytes; private int numRecordsConverted; private long conversionTimeNanos; - public RecordConversionStats(long temporaryMemoryBytes, int numRecordsConverted, long conversionTimeNanos) { + public RecordValidationStats(long temporaryMemoryBytes, int numRecordsConverted, long conversionTimeNanos) { this.temporaryMemoryBytes = temporaryMemoryBytes; this.numRecordsConverted = numRecordsConverted; this.conversionTimeNanos = conversionTimeNanos; } - public RecordConversionStats() { + public RecordValidationStats() { this(0, 0, 0); } - public void add(RecordConversionStats stats) { + public void add(RecordValidationStats stats) { temporaryMemoryBytes += stats.temporaryMemoryBytes; numRecordsConverted += stats.numRecordsConverted; conversionTimeNanos += stats.conversionTimeNanos; @@ -64,7 +69,7 @@ public long conversionTimeNanos() { @Override public String toString() { - return String.format("RecordConversionStats(temporaryMemoryBytes=%d, numRecordsConverted=%d, conversionTimeNanos=%d)", + return String.format("RecordValidationStats(temporaryMemoryBytes=%d, numRecordsConverted=%d, conversionTimeNanos=%d)", temporaryMemoryBytes, numRecordsConverted, conversionTimeNanos); } } diff --git a/clients/src/main/java/org/apache/kafka/common/record/RecordsUtil.java b/clients/src/main/java/org/apache/kafka/common/record/RecordsUtil.java index 423d1e1656f86..8328e206146e9 100644 --- a/clients/src/main/java/org/apache/kafka/common/record/RecordsUtil.java +++ b/clients/src/main/java/org/apache/kafka/common/record/RecordsUtil.java @@ -96,7 +96,7 @@ protected static ConvertedRecords downConvert(Iterable(MemoryRecords.readableRecords(buffer), stats); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java index 76342dd124aa9..6516de3f9ac3d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java @@ -21,6 +21,34 @@ // Abstract class for all control requests including UpdateMetadataRequest, LeaderAndIsrRequest and StopReplicaRequest public abstract class AbstractControlRequest extends AbstractRequest { + /** + * Indicates if a controller request is incremental, full, or unknown. + * Used by LeaderAndIsrRequest.Type and UpdateMetadataRequest.Type fields. + */ + public enum Type { + UNKNOWN(0), + INCREMENTAL(1), + FULL(2); + + private final byte type; + private Type(int type) { + this.type = (byte) type; + } + + public byte toByte() { + return type; + } + + public static Type fromByte(byte type) { + for (Type t : Type.values()) { + if (t.type == type) { + return t; + } + } + return UNKNOWN; + } + } + public static final long UNKNOWN_BROKER_EPOCH = -1L; public static abstract class Builder extends AbstractRequest.Builder { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index e71c5debf62c7..23f67cb5273e5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -322,6 +322,8 @@ private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, return PushTelemetryRequest.parse(buffer, apiVersion); case ASSIGN_REPLICAS_TO_DIRS: return AssignReplicasToDirsRequest.parse(buffer, apiVersion); + case LIST_CLIENT_METRICS_RESOURCES: + return ListClientMetricsResourcesRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index d849a0740b3a3..f99da4e2119df 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -259,6 +259,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer response return PushTelemetryResponse.parse(responseBuffer, version); case ASSIGN_REPLICAS_TO_DIRS: return AssignReplicasToDirsResponse.parse(responseBuffer, version); + case LIST_CLIENT_METRICS_RESOURCES: + return ListClientMetricsResourcesResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 87f0277d19628..f8f364f06f794 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -114,7 +114,8 @@ public static ApiVersionsResponse createApiVersionsResponse( NodeApiVersions controllerApiVersions, ListenerType listenerType, boolean enableUnstableLastVersion, - boolean zkMigrationEnabled + boolean zkMigrationEnabled, + boolean clientTelemetryEnabled ) { ApiVersionCollection apiKeys; if (controllerApiVersions != null) { @@ -122,13 +123,15 @@ public static ApiVersionsResponse createApiVersionsResponse( listenerType, minRecordVersion, controllerApiVersions.allSupportedApiVersions(), - enableUnstableLastVersion + enableUnstableLastVersion, + clientTelemetryEnabled ); } else { apiKeys = filterApis( minRecordVersion, listenerType, - enableUnstableLastVersion + enableUnstableLastVersion, + clientTelemetryEnabled ); } @@ -167,16 +170,21 @@ public static ApiVersionCollection filterApis( RecordVersion minRecordVersion, ApiMessageType.ListenerType listenerType ) { - return filterApis(minRecordVersion, listenerType, false); + return filterApis(minRecordVersion, listenerType, false, false); } public static ApiVersionCollection filterApis( RecordVersion minRecordVersion, ApiMessageType.ListenerType listenerType, - boolean enableUnstableLastVersion + boolean enableUnstableLastVersion, + boolean clientTelemetryEnabled ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { + // Skip telemetry APIs if client telemetry is disabled. + if ((apiKey == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS || apiKey == ApiKeys.PUSH_TELEMETRY) && !clientTelemetryEnabled) + continue; + if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { apiKey.toApiVersion(enableUnstableLastVersion).ifPresent(apiKeys::add); } @@ -203,13 +211,15 @@ public static ApiVersionCollection collectApis( * @param minRecordVersion min inter broker magic * @param activeControllerApiVersions controller ApiVersions * @param enableUnstableLastVersion whether unstable versions should be advertised or not + * @param clientTelemetryEnabled whether client telemetry is enabled or not * @return commonly agreed ApiVersion collection */ public static ApiVersionCollection intersectForwardableApis( final ApiMessageType.ListenerType listenerType, final RecordVersion minRecordVersion, final Map activeControllerApiVersions, - boolean enableUnstableLastVersion + boolean enableUnstableLastVersion, + boolean clientTelemetryEnabled ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { @@ -220,6 +230,10 @@ public static ApiVersionCollection intersectForwardableApis( continue; } + // Skip telemetry APIs if client telemetry is disabled. + if ((apiKey == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS || apiKey == ApiKeys.PUSH_TELEMETRY) && !clientTelemetryEnabled) + continue; + final ApiVersion finalApiVersion; if (!apiKey.forwardable) { finalApiVersion = brokerApiVersion.get(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AssignReplicasToDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AssignReplicasToDirsRequest.java index 4edd726850a50..5941181ed81dc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AssignReplicasToDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AssignReplicasToDirsRequest.java @@ -27,6 +27,13 @@ public class AssignReplicasToDirsRequest extends AbstractRequest { + /** + * The maximum number of assignments to be included in a single request. + * This limit was chosen based on the maximum size of AssignReplicasToDirsRequest for + * 10 different directory IDs, so that it still fits in a single TCP packet. i.e. 64KB. + */ + public static final int MAX_ASSIGNMENTS_PER_REQUEST = 2250; + public static class Builder extends AbstractRequest.Builder { private final AssignReplicasToDirsRequestData data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequest.java index 862d9c9d4e497..48a03bd8bb680 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequest.java @@ -23,6 +23,8 @@ import org.apache.kafka.common.protocol.Errors; import java.nio.ByteBuffer; +import java.util.List; +import java.util.stream.Collectors; public class ConsumerGroupDescribeRequest extends AbstractRequest { @@ -83,4 +85,15 @@ public static ConsumerGroupDescribeRequest parse(ByteBuffer buffer, short versio version ); } + + public static List getErrorDescribedGroupList( + List groupIds, + Errors error + ) { + return groupIds.stream() + .map(groupId -> new ConsumerGroupDescribeResponseData.DescribedGroup() + .setGroupId(groupId) + .setErrorCode(error.code()) + ).collect(Collectors.toList()); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupHeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupHeartbeatRequest.java index 1b56c5b91c164..ced4880adf6fb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupHeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ConsumerGroupHeartbeatRequest.java @@ -30,6 +30,12 @@ public class ConsumerGroupHeartbeatRequest extends AbstractRequest { * A member epoch of -1 means that the member wants to leave the group. */ public static final int LEAVE_GROUP_MEMBER_EPOCH = -1; + public static final int LEAVE_GROUP_STATIC_MEMBER_EPOCH = -2; + + /** + * A member epoch of 0 means that the member wants to join the group. + */ + public static final int JOIN_GROUP_MEMBER_EPOCH = 0; public static class Builder extends AbstractRequest.Builder { private final ConsumerGroupHeartbeatRequestData data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchSnapshotRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchSnapshotRequest.java index 1769e94f47f35..34db2d1c12419 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchSnapshotRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchSnapshotRequest.java @@ -61,6 +61,7 @@ public FetchSnapshotRequestData data() { */ public static FetchSnapshotRequestData singleton( String clusterId, + int replicaId, TopicPartition topicPartition, UnaryOperator operator ) { @@ -70,6 +71,7 @@ public static FetchSnapshotRequestData singleton( return new FetchSnapshotRequestData() .setClusterId(clusterId) + .setReplicaId(replicaId) .setTopics( Collections.singletonList( new FetchSnapshotRequestData.TopicSnapshot() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index 5661705d1a293..bf8083d910712 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -37,6 +37,12 @@ public JoinGroupResponse(JoinGroupResponseData data, short version) { if (version < 7 && data.protocolName() == null) { data.setProtocolName(""); } + + // If nullable string for the protocol name is supported, + // we set empty string to be null to ensure compliance. + if (version >= 7 && data.protocolName() != null && data.protocolName().isEmpty()) { + data.setProtocolName(null); + } } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index dbd59b5b69252..9a07a88a35d20 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -42,30 +42,6 @@ public class LeaderAndIsrRequest extends AbstractControlRequest { - public enum Type { - UNKNOWN(0), - INCREMENTAL(1), - FULL(2); - - private final byte type; - private Type(int type) { - this.type = (byte) type; - } - - public byte toByte() { - return type; - } - - public static Type fromByte(byte type) { - for (Type t : Type.values()) { - if (t.type == type) { - return t; - } - } - return UNKNOWN; - } - } - public static class Builder extends AbstractControlRequest.Builder { private final List partitionStates; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index e1fd291316304..c32d61442632c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -61,15 +61,15 @@ public LeaveGroupResponse(LeaveGroupResponseData data, short version) { if (version >= 3) { this.data = data; + } else if (data.errorCode() != Errors.NONE.code()) { + this.data = new LeaveGroupResponseData().setErrorCode(data.errorCode()); } else { if (data.members().size() != 1) { throw new UnsupportedVersionException("LeaveGroup response version " + version + " can only contain one member, got " + data.members().size() + " members."); } - Errors topLevelError = Errors.forCode(data.errorCode()); - short errorCode = getError(topLevelError, data.members()).code(); - this.data = new LeaveGroupResponseData().setErrorCode(errorCode); + this.data = new LeaveGroupResponseData().setErrorCode(data.members().get(0).errorCode()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesRequest.java new file mode 100644 index 0000000000000..ab396ff10c8b2 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesRequest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.ListClientMetricsResourcesRequestData; +import org.apache.kafka.common.message.ListClientMetricsResourcesResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; + +public class ListClientMetricsResourcesRequest extends AbstractRequest { + public static class Builder extends AbstractRequest.Builder { + public final ListClientMetricsResourcesRequestData data; + + public Builder(ListClientMetricsResourcesRequestData data) { + super(ApiKeys.LIST_CLIENT_METRICS_RESOURCES); + this.data = data; + } + + @Override + public ListClientMetricsResourcesRequest build(short version) { + return new ListClientMetricsResourcesRequest(data, version); + } + + @Override + public String toString() { + return data.toString(); + } + } + + private final ListClientMetricsResourcesRequestData data; + + private ListClientMetricsResourcesRequest(ListClientMetricsResourcesRequestData data, short version) { + super(ApiKeys.LIST_CLIENT_METRICS_RESOURCES, version); + this.data = data; + } + + public ListClientMetricsResourcesRequestData data() { + return data; + } + + @Override + public ListClientMetricsResourcesResponse getErrorResponse(int throttleTimeMs, Throwable e) { + Errors error = Errors.forException(e); + ListClientMetricsResourcesResponseData response = new ListClientMetricsResourcesResponseData() + .setErrorCode(error.code()) + .setThrottleTimeMs(throttleTimeMs); + return new ListClientMetricsResourcesResponse(response); + } + + public static ListClientMetricsResourcesRequest parse(ByteBuffer buffer, short version) { + return new ListClientMetricsResourcesRequest(new ListClientMetricsResourcesRequestData( + new ByteBufferAccessor(buffer), version), version); + } + + @Override + public String toString(boolean verbose) { + return data.toString(); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesResponse.java new file mode 100644 index 0000000000000..189aad1eb308d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListClientMetricsResourcesResponse.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.clients.admin.ClientMetricsResourceListing; +import org.apache.kafka.common.message.ListClientMetricsResourcesResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.Map; + +public class ListClientMetricsResourcesResponse extends AbstractResponse { + private final ListClientMetricsResourcesResponseData data; + + public ListClientMetricsResourcesResponse(ListClientMetricsResourcesResponseData data) { + super(ApiKeys.LIST_CLIENT_METRICS_RESOURCES); + this.data = data; + } + + public ListClientMetricsResourcesResponseData data() { + return data; + } + + public ApiError error() { + return new ApiError(Errors.forCode(data.errorCode())); + } + + @Override + public Map errorCounts() { + return errorCounts(Errors.forCode(data.errorCode())); + } + + public static ListClientMetricsResourcesResponse parse(ByteBuffer buffer, short version) { + return new ListClientMetricsResourcesResponse(new ListClientMetricsResourcesResponseData( + new ByteBufferAccessor(buffer), version)); + } + + @Override + public String toString() { + return data.toString(); + } + + @Override + public int throttleTimeMs() { + return data.throttleTimeMs(); + } + + @Override + public void maybeSetThrottleTimeMs(int throttleTimeMs) { + data.setThrottleTimeMs(throttleTimeMs); + } + + public Collection clientMetricsResources() { + return data.clientMetricsResources() + .stream() + .map(entry -> new ClientMetricsResourceListing(entry.name())) + .collect(Collectors.toList()); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index a8c2a801e4ce7..186ad9b80a19f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.common.requests; +import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ProduceResponseData; import org.apache.kafka.common.message.ProduceResponseData.LeaderIdAndEpoch; @@ -72,7 +73,7 @@ public ProduceResponse(ProduceResponseData produceResponseData) { */ @Deprecated public ProduceResponse(Map responses) { - this(responses, DEFAULT_THROTTLE_TIME); + this(responses, DEFAULT_THROTTLE_TIME, Collections.emptyList()); } /** @@ -83,10 +84,23 @@ public ProduceResponse(Map responses) { */ @Deprecated public ProduceResponse(Map responses, int throttleTimeMs) { - this(toData(responses, throttleTimeMs)); + this(toData(responses, throttleTimeMs, Collections.emptyList())); } - private static ProduceResponseData toData(Map responses, int throttleTimeMs) { + /** + * Constructor for the latest version + * This is deprecated in favor of using the ProduceResponseData constructor, KafkaApis should switch to that + * in KAFKA-10730 + * @param responses Produced data grouped by topic-partition + * @param throttleTimeMs Time in milliseconds the response was throttled + * @param nodeEndpoints List of node endpoints + */ + @Deprecated + public ProduceResponse(Map responses, int throttleTimeMs, List nodeEndpoints) { + this(toData(responses, throttleTimeMs, nodeEndpoints)); + } + + private static ProduceResponseData toData(Map responses, int throttleTimeMs, List nodeEndpoints) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); responses.forEach((tp, response) -> { ProduceResponseData.TopicProduceResponse tpr = data.responses().find(tp.topic()); @@ -110,6 +124,12 @@ private static ProduceResponseData toData(Map .setBatchIndexErrorMessage(e.message)) .collect(Collectors.toList()))); }); + nodeEndpoints.forEach(endpoint -> data.nodeEndpoints() + .add(new ProduceResponseData.NodeEndpoint() + .setNodeId(endpoint.id()) + .setHost(endpoint.host()) + .setPort(endpoint.port()) + .setRack(endpoint.rack()))); return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/PushTelemetryRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/PushTelemetryRequest.java index 5df03ed3461b0..07827ebff7b49 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/PushTelemetryRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/PushTelemetryRequest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.common.requests; import org.apache.kafka.common.message.PushTelemetryRequestData; @@ -22,11 +21,14 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; import java.nio.ByteBuffer; public class PushTelemetryRequest extends AbstractRequest { + private static final String OTLP_CONTENT_TYPE = "OTLP"; + public static class Builder extends AbstractRequest.Builder { private final PushTelemetryRequestData data; @@ -60,10 +62,7 @@ public PushTelemetryRequest(PushTelemetryRequestData data, short version) { @Override public PushTelemetryResponse getErrorResponse(int throttleTimeMs, Throwable e) { - PushTelemetryResponseData responseData = new PushTelemetryResponseData() - .setErrorCode(Errors.forException(e).code()) - .setThrottleTimeMs(throttleTimeMs); - return new PushTelemetryResponse(responseData); + return errorResponse(throttleTimeMs, Errors.forException(e)); } @Override @@ -71,6 +70,31 @@ public PushTelemetryRequestData data() { return data; } + public PushTelemetryResponse errorResponse(int throttleTimeMs, Errors errors) { + PushTelemetryResponseData responseData = new PushTelemetryResponseData(); + responseData.setErrorCode(errors.code()); + responseData.setThrottleTimeMs(throttleTimeMs); + return new PushTelemetryResponse(responseData); + } + + public String metricsContentType() { + // Future versions of PushTelemetryRequest and GetTelemetrySubscriptionsRequest may include a content-type + // field to allow for updated OTLP format versions (or additional formats), but this field is currently not + // included since only one format is specified in the current proposal of the kip-714 + return OTLP_CONTENT_TYPE; + } + + public ByteBuffer metricsData() { + CompressionType cType = CompressionType.forId(this.data.compressionType()); + return (cType == CompressionType.NONE) ? + ByteBuffer.wrap(this.data.metrics()) : decompressMetricsData(cType, this.data.metrics()); + } + + private static ByteBuffer decompressMetricsData(CompressionType compressionType, byte[] metrics) { + // TODO: Add support for decompression of metrics data + return ByteBuffer.wrap(metrics); + } + public static PushTelemetryRequest parse(ByteBuffer buffer, short version) { return new PushTelemetryRequest(new PushTelemetryRequestData( new ByteBufferAccessor(buffer), version), version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java index c0fd3000cc502..ea9ae8141984a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java @@ -47,21 +47,28 @@ public static class Builder extends AbstractControlRequest.Builder partitionStates; private final List liveBrokers; private final Map topicIds; + private final Type updateType; public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, List partitionStates, List liveBrokers, Map topicIds) { this(version, controllerId, controllerEpoch, brokerEpoch, partitionStates, - liveBrokers, topicIds, false); + liveBrokers, topicIds, false, Type.UNKNOWN); } public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch, List partitionStates, List liveBrokers, - Map topicIds, boolean kraftController) { + Map topicIds, boolean kraftController, Type updateType) { super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch, kraftController); this.partitionStates = partitionStates; this.liveBrokers = liveBrokers; this.topicIds = topicIds; + + if (version >= 8) { + this.updateType = updateType; + } else { + this.updateType = Type.UNKNOWN; + } } @Override @@ -95,6 +102,7 @@ public UpdateMetadataRequest build(short version) { if (version >= 8) { data.setIsKRaftController(kraftController); + data.setType(updateType.toByte()); } if (version >= 5) { @@ -129,6 +137,8 @@ public String toString() { bld.append("(type: UpdateMetadataRequest="). append(", controllerId=").append(controllerId). append(", controllerEpoch=").append(controllerEpoch). + append(", kraftController=").append(kraftController). + append(", type=").append(updateType). append(", brokerEpoch=").append(brokerEpoch). append(", partitionStates=").append(partitionStates). append(", liveBrokers=").append(Utils.join(liveBrokers, ", ")). @@ -196,6 +206,10 @@ public boolean isKRaftController() { return data.isKRaftController(); } + public Type updateType() { + return Type.fromByte(data.type()); + } + @Override public int controllerEpoch() { return data.controllerEpoch(); diff --git a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java index a0a7f7239ee9c..0d4091b8eab8e 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java +++ b/clients/src/main/java/org/apache/kafka/common/security/ssl/SslFactory.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.network.Mode; import org.apache.kafka.common.security.auth.SslEngineFactory; +import org.apache.kafka.common.utils.ConfigUtils; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -167,7 +168,10 @@ private SslEngineFactory createNewSslEngineFactory(Map newConfigs) { "which a keystore was configured."); } - CertificateEntries.ensureCompatible(newSslEngineFactory.keystore(), sslEngineFactory.keystore()); + boolean allowDnChanges = ConfigUtils.getBoolean(nextConfigs, BrokerSecurityConfigs.SSL_ALLOW_DN_CHANGES_CONFIG, BrokerSecurityConfigs.DEFAULT_SSL_ALLOW_DN_CHANGES_VALUE); + boolean allowSanChanges = ConfigUtils.getBoolean(nextConfigs, BrokerSecurityConfigs.SSL_ALLOW_SAN_CHANGES_CONFIG, BrokerSecurityConfigs.DEFAULT_SSL_ALLOW_SAN_CHANGES_VALUE); + + CertificateEntries.ensureCompatible(newSslEngineFactory.keystore(), sslEngineFactory.keystore(), allowDnChanges, allowSanChanges); } if (sslEngineFactory.truststore() == null && newSslEngineFactory.truststore() != null) { throw new ConfigException("Cannot add SSL truststore to an existing listener for which no " + @@ -302,18 +306,31 @@ static List create(KeyStore keystore) throws GeneralSecurity return entries; } - static void ensureCompatible(KeyStore newKeystore, KeyStore oldKeystore) throws GeneralSecurityException { + static void ensureCompatible(KeyStore newKeystore, KeyStore oldKeystore, boolean allowDnChanges, boolean allowSanChanges) throws GeneralSecurityException { List newEntries = CertificateEntries.create(newKeystore); List oldEntries = CertificateEntries.create(oldKeystore); + + if (!allowDnChanges) { + ensureCompatibleDNs(newEntries, oldEntries); + } + + if (!allowSanChanges) { + ensureCompatibleSANs(newEntries, oldEntries); + } + } + + private static void ensureCompatibleDNs(List newEntries, List oldEntries) { if (newEntries.size() != oldEntries.size()) { throw new ConfigException(String.format("Keystore entries do not match, existing store contains %d entries, new store contains %d entries", oldEntries.size(), newEntries.size())); } + for (int i = 0; i < newEntries.size(); i++) { CertificateEntries newEntry = newEntries.get(i); CertificateEntries oldEntry = oldEntries.get(i); Principal newPrincipal = newEntry.subjectPrincipal; Principal oldPrincipal = oldEntry.subjectPrincipal; + // Compare principal objects to compare canonical names (e.g. to ignore leading/trailing whitespaces). // Canonical names may differ if the tags of a field changes from one with a printable string representation // to one without or vice-versa due to optional conversion to hex representation based on the tag. So we @@ -323,6 +340,19 @@ static void ensureCompatible(KeyStore newKeystore, KeyStore oldKeystore) throws " existing={alias=%s, DN=%s}, new={alias=%s, DN=%s}", oldEntry.alias, oldEntry.subjectPrincipal, newEntry.alias, newEntry.subjectPrincipal)); } + } + } + + private static void ensureCompatibleSANs(List newEntries, List oldEntries) { + if (newEntries.size() != oldEntries.size()) { + throw new ConfigException(String.format("Keystore entries do not match, existing store contains %d entries, new store contains %d entries", + oldEntries.size(), newEntries.size())); + } + + for (int i = 0; i < newEntries.size(); i++) { + CertificateEntries newEntry = newEntries.get(i); + CertificateEntries oldEntry = oldEntries.get(i); + if (!newEntry.subjectAltNames.containsAll(oldEntry.subjectAltNames)) { throw new ConfigException(String.format("Keystore SubjectAltNames do not match: " + " existing={alias=%s, SAN=%s}, new={alias=%s, SAN=%s}", diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitter.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitter.java new file mode 100644 index 0000000000000..2504bedec5e2f --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; + +public class ClientTelemetryEmitter implements MetricsEmitter { + + private final Predicate selector; + private final List emitted; + private final boolean deltaMetrics; + + ClientTelemetryEmitter(Predicate selector, boolean deltaMetrics) { + this.selector = selector; + this.emitted = new ArrayList<>(); + this.deltaMetrics = deltaMetrics; + } + + @Override + public boolean shouldEmitMetric(MetricKeyable metricKeyable) { + return selector.test(metricKeyable); + } + + @Override + public boolean shouldEmitDeltaMetrics() { + return deltaMetrics; + } + + @Override + public boolean emitMetric(SinglePointMetric metric) { + if (!shouldEmitMetric(metric)) { + return false; + } + + emitted.add(metric); + return true; + } + + @Override + public List emittedMetrics() { + return Collections.unmodifiableList(emitted); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryProvider.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryProvider.java new file mode 100644 index 0000000000000..d5eb3fb0c075d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryProvider.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.resource.v1.Resource; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.metrics.MetricsContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ClientTelemetryProvider implements Configurable { + + public static final String DOMAIN = "org.apache.kafka"; + // Client metrics tags + public static final String CLIENT_RACK = "client_rack"; + public static final String GROUP_ID = "group_id"; + public static final String GROUP_INSTANCE_ID = "group_instance_id"; + public static final String GROUP_MEMBER_ID = "group_member_id"; + public static final String TRANSACTIONAL_ID = "transactional_id"; + + private static final String PRODUCER_NAMESPACE = "kafka.producer"; + private static final String CONSUMER_NAMESPACE = "kafka.consumer"; + + private static final Map PRODUCER_CONFIG_MAPPING = new HashMap<>(); + private static final Map CONSUMER_CONFIG_MAPPING = new HashMap<>(); + + private volatile Resource resource = null; + private Map config = null; + + // Mapping of config keys to telemetry keys. Contains only keys which can be fetched from config. + // Config like group_member_id is not present here as it is not fetched from config. + static { + PRODUCER_CONFIG_MAPPING.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, ClientTelemetryProvider.TRANSACTIONAL_ID); + + CONSUMER_CONFIG_MAPPING.put(ConsumerConfig.GROUP_ID_CONFIG, ClientTelemetryProvider.GROUP_ID); + CONSUMER_CONFIG_MAPPING.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, ClientTelemetryProvider.GROUP_INSTANCE_ID); + } + + @Override + public synchronized void configure(Map configs) { + this.config = configs; + } + + /** + * Validate that all the data required for generating correct metrics is present. + * + * @param metricsContext {@link MetricsContext} + * @return false if all the data required for generating correct metrics is missing, true + * otherwise. + */ + boolean validate(MetricsContext metricsContext) { + return ClientTelemetryUtils.validateRequiredResourceLabels(metricsContext.contextLabels()); + } + + /** + * Sets the metrics tags for the service or library exposing metrics. This will be called before + * {@link org.apache.kafka.common.metrics.MetricsReporter#init(List)} and may be called anytime + * after that. + * + * @param metricsContext {@link MetricsContext} + */ + synchronized void contextChange(MetricsContext metricsContext) { + final Resource.Builder resourceBuilder = Resource.newBuilder(); + + final String namespace = metricsContext.contextLabels().get(MetricsContext.NAMESPACE); + if (PRODUCER_NAMESPACE.equals(namespace)) { + // Add producer resource labels. + PRODUCER_CONFIG_MAPPING.forEach((configKey, telemetryKey) -> { + if (config.containsKey(configKey)) { + addAttribute(resourceBuilder, telemetryKey, String.valueOf(config.get(configKey))); + } + }); + } else if (CONSUMER_NAMESPACE.equals(namespace)) { + // Add consumer resource labels. + CONSUMER_CONFIG_MAPPING.forEach((configKey, telemetryKey) -> { + if (config.containsKey(configKey)) { + addAttribute(resourceBuilder, telemetryKey, String.valueOf(config.get(configKey))); + } + }); + } + + // Add client rack label. + if (config.containsKey(CommonClientConfigs.CLIENT_RACK_CONFIG)) { + addAttribute(resourceBuilder, CLIENT_RACK, String.valueOf(config.get(CommonClientConfigs.CLIENT_RACK_CONFIG))); + } + + resource = resourceBuilder.build(); + } + + /** + * Updates the resource labels/tags for the service or library exposing metrics. + * + * @param labels Map of labels to be updated. + */ + synchronized void updateLabels(Map labels) { + final Resource.Builder resourceBuilder = resource.toBuilder(); + labels.forEach((key, value) -> { + addAttribute(resourceBuilder, key, value); + }); + resource = resourceBuilder.build(); + } + + /** + * The metrics resource for this provider which will be used to generate the metrics. + * + * @return A fully formed {@link Resource} with all the tags. + */ + Resource resource() { + return resource; + } + + /** + * Domain of the active provider i.e. specifies prefix to the metrics. + * + * @return Domain in string format. + */ + String domain() { + return DOMAIN; + } + + private void addAttribute(Resource.Builder resourceBuilder, String key, String value) { + final KeyValue.Builder kv = KeyValue.newBuilder() + .setKey(key) + .setValue(AnyValue.newBuilder().setStringValue(value)); + resourceBuilder.addAttributes(kv); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporter.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporter.java new file mode 100644 index 0000000000000..617674f7d00f3 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporter.java @@ -0,0 +1,984 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.MetricsData; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsRequestData; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsResponseData; +import org.apache.kafka.common.message.PushTelemetryRequestData; +import org.apache.kafka.common.message.PushTelemetryResponseData; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.MetricsContext; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractRequest.Builder; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; +import org.apache.kafka.common.requests.PushTelemetryRequest; +import org.apache.kafka.common.requests.PushTelemetryResponse; +import org.apache.kafka.common.telemetry.ClientTelemetryState; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.StringJoiner; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Predicate; + +/** + * The implementation of the {@link MetricsReporter} for client telemetry which manages the life-cycle + * of the client telemetry collection process. The client telemetry reporter is responsible for + * collecting the client telemetry data and sending it to the broker. + *

    + * + * The client telemetry reporter is configured with a {@link ClientTelemetrySender} which is + * responsible for sending the client telemetry data to the broker. The client telemetry reporter + * will attempt to fetch the telemetry subscription information from the broker and send the + * telemetry data to the broker based on the subscription information i.e. push interval, temporality, + * compression type, etc. + *

    + * + * The full life-cycle of the metric collection process is defined by a state machine in + * {@link ClientTelemetryState}. Each state is associated with a different set of operations. + * For example, the client telemetry reporter will attempt to fetch the telemetry subscription + * from the broker when in the {@link ClientTelemetryState#SUBSCRIPTION_NEEDED} state. + * If the push operation fails, the client telemetry reporter will attempt to re-fetch the + * subscription information by setting the state back to {@link ClientTelemetryState#SUBSCRIPTION_NEEDED}. + *

    + * + * In an unlikely scenario, if a bad state transition is detected, an + * {@link IllegalStateException} will be thrown. + *

    + * + * The state transition follows the following steps in order: + *

      + *
    1. {@link ClientTelemetryState#SUBSCRIPTION_NEEDED}
      2. + *
    2. {@link ClientTelemetryState#SUBSCRIPTION_IN_PROGRESS}
      3. + *
    3. {@link ClientTelemetryState#PUSH_NEEDED}
      4. + *
    4. {@link ClientTelemetryState#PUSH_IN_PROGRESS}
      5. + *
    5. {@link ClientTelemetryState#TERMINATING_PUSH_NEEDED}
      6. + *
    6. {@link ClientTelemetryState#TERMINATING_PUSH_IN_PROGRESS}
      7. + *
    7. {@link ClientTelemetryState#TERMINATED}
      8. + *
    + *

    + * + * For more detail in state transition, see {@link ClientTelemetryState#validateTransition}. + */ +public class ClientTelemetryReporter implements MetricsReporter { + + private static final Logger log = LoggerFactory.getLogger(ClientTelemetryReporter.class); + public static final int DEFAULT_PUSH_INTERVAL_MS = 5 * 60 * 1000; + + private final ClientTelemetryProvider telemetryProvider; + private final ClientTelemetrySender clientTelemetrySender; + private final Time time; + + private Map rawOriginalConfig; + private KafkaMetricsCollector kafkaMetricsCollector; + + public ClientTelemetryReporter(Time time) { + this.time = time; + telemetryProvider = new ClientTelemetryProvider(); + clientTelemetrySender = new DefaultClientTelemetrySender(); + } + + @SuppressWarnings("unchecked") + @Override + public synchronized void configure(Map configs) { + rawOriginalConfig = (Map) Objects.requireNonNull(configs); + } + + @Override + public synchronized void contextChange(MetricsContext metricsContext) { + /* + If validation succeeds: initialize the provider, start the metric collection task, + set metrics labels for services/libraries that expose metrics. + */ + Objects.requireNonNull(rawOriginalConfig, "configure() was not called before contextChange()"); + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.stop(); + } + + if (!telemetryProvider.validate(metricsContext)) { + log.warn("Validation failed for {} context {}, skip starting collectors. Metrics collection is disabled", + telemetryProvider.getClass(), metricsContext.contextLabels()); + return; + } + + if (kafkaMetricsCollector == null) { + /* + Initialize the provider only once. contextChange(..) can be called more than once, + but once it's been initialized and all necessary labels are present then we don't + re-initialize. + */ + telemetryProvider.configure(rawOriginalConfig); + } + + telemetryProvider.contextChange(metricsContext); + + if (kafkaMetricsCollector == null) { + initCollectors(); + } + } + + @Override + public void init(List metrics) { + /* + metrics collector may not have been initialized (e.g. invalid context labels) + in which case metrics collection is disabled + */ + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.init(metrics); + } + } + + /** + * Method is invoked whenever a metric is added/registered + */ + @Override + public void metricChange(KafkaMetric metric) { + /* + metrics collector may not have been initialized (e.g. invalid context labels) + in which case metrics collection is disabled + */ + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.metricChange(metric); + } + } + + /** + * Method is invoked whenever a metric is removed + */ + @Override + public void metricRemoval(KafkaMetric metric) { + /* + metrics collector may not have been initialized (e.g. invalid context labels) + in which case metrics collection is disabled + */ + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.metricRemoval(metric); + } + } + + @Override + public void close() { + log.debug("Stopping ClientTelemetryReporter"); + try { + clientTelemetrySender.close(); + } catch (Exception exception) { + log.error("Failed to close client telemetry reporter", exception); + } + } + + public synchronized void updateMetricsLabels(Map labels) { + telemetryProvider.updateLabels(labels); + } + + public void initiateClose(long timeoutMs) { + log.debug("Initiate close of ClientTelemetryReporter"); + try { + clientTelemetrySender.initiateClose(timeoutMs); + } catch (Exception exception) { + log.error("Failed to initiate close of client telemetry reporter", exception); + } + } + + public ClientTelemetrySender telemetrySender() { + return clientTelemetrySender; + } + + private void initCollectors() { + kafkaMetricsCollector = new KafkaMetricsCollector( + TelemetryMetricNamingConvention.getClientTelemetryMetricNamingStrategy( + telemetryProvider.domain())); + } + + private ResourceMetrics buildMetric(Metric metric) { + return ResourceMetrics.newBuilder() + .setResource(telemetryProvider.resource()) + .addScopeMetrics(ScopeMetrics.newBuilder() + .addMetrics(metric) + .build()).build(); + } + + // Visible for testing, only for unit tests + void metricsCollector(KafkaMetricsCollector metricsCollector) { + kafkaMetricsCollector = metricsCollector; + } + + // Visible for testing, only for unit tests + MetricsCollector metricsCollector() { + return kafkaMetricsCollector; + } + + // Visible for testing, only for unit tests + ClientTelemetryProvider telemetryProvider() { + return telemetryProvider; + } + + class DefaultClientTelemetrySender implements ClientTelemetrySender { + + /* + These are the lower and upper bounds of the jitter that we apply to the initial push + telemetry API call. This helps to avoid a flood of requests all coming at the same time. + */ + private final static double INITIAL_PUSH_JITTER_LOWER = 0.5; + private final static double INITIAL_PUSH_JITTER_UPPER = 1.5; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Condition subscriptionLoaded = lock.writeLock().newCondition(); + private final Condition terminalPushInProgress = lock.writeLock().newCondition(); + /* + Initial state should be subscription needed which should allow issuing first telemetry + request of get telemetry subscription. + */ + private ClientTelemetryState state = ClientTelemetryState.SUBSCRIPTION_NEEDED; + + private ClientTelemetrySubscription subscription; + + /* + Last time a telemetry request was made. Initialized to 0 to indicate that no request has + been made yet. Telemetry requests, get or post, should always be made after the push interval + time has elapsed. + */ + private long lastRequestMs; + /* + Interval between telemetry requests in milliseconds. Initialized to 0 to indicate that the + interval has not yet been computed. The first get request will be immediately triggered as + soon as the client is ready. + */ + private int intervalMs; + /* + Whether the client telemetry sender is enabled or not. Initialized to true to indicate that + the client telemetry sender is enabled. This is used to disable the client telemetry sender + when the client receives unrecoverable error from broker. + */ + private boolean enabled; + + private DefaultClientTelemetrySender() { + enabled = true; + } + + @Override + public long timeToNextUpdate(long requestTimeoutMs) { + final long nowMs = time.milliseconds(); + final ClientTelemetryState localState; + final long localLastRequestMs; + final int localIntervalMs; + + lock.readLock().lock(); + try { + if (!enabled) { + return Integer.MAX_VALUE; + } + + localState = state; + localLastRequestMs = lastRequestMs; + localIntervalMs = intervalMs; + } finally { + lock.readLock().unlock(); + } + + final long timeMs; + final String apiName; + final String msg; + + switch (localState) { + case SUBSCRIPTION_IN_PROGRESS: + case PUSH_IN_PROGRESS: + /* + We have a network request in progress. We record the time of the request + submission, so wait that amount of the time PLUS the requestTimeout that + is provided. + */ + apiName = (localState == ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS) ? ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS.name : ApiKeys.PUSH_TELEMETRY.name; + timeMs = requestTimeoutMs; + msg = String.format("the remaining wait time for the %s network API request, as specified by %s", apiName, CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG); + break; + case TERMINATING_PUSH_IN_PROGRESS: + timeMs = Long.MAX_VALUE; + msg = String.format("the terminating push is in progress, disabling telemetry for further requests"); + break; + case TERMINATING_PUSH_NEEDED: + timeMs = 0; + msg = String.format("the client should try to submit the final %s network API request ASAP before closing", ApiKeys.PUSH_TELEMETRY.name); + break; + case SUBSCRIPTION_NEEDED: + case PUSH_NEEDED: + apiName = (localState == ClientTelemetryState.SUBSCRIPTION_NEEDED) ? ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS.name : ApiKeys.PUSH_TELEMETRY.name; + long timeRemainingBeforeRequest = localLastRequestMs + localIntervalMs - nowMs; + if (timeRemainingBeforeRequest <= 0) { + timeMs = 0; + msg = String.format("the wait time before submitting the next %s network API request has elapsed", apiName); + } else { + timeMs = timeRemainingBeforeRequest; + msg = String.format("the client will wait before submitting the next %s network API request", apiName); + } + break; + default: + throw new IllegalStateException("Unknown telemetry state: " + localState); + } + + log.debug("For telemetry state {}, returning the value {} ms; {}", localState, timeMs, msg); + return timeMs; + } + + @Override + public Optional> createRequest() { + final ClientTelemetryState localState; + final ClientTelemetrySubscription localSubscription; + + lock.readLock().lock(); + try { + localState = state; + localSubscription = subscription; + } finally { + lock.readLock().unlock(); + } + + if (localState == ClientTelemetryState.SUBSCRIPTION_NEEDED) { + return createSubscriptionRequest(localSubscription); + } else if (localState == ClientTelemetryState.PUSH_NEEDED || localState == ClientTelemetryState.TERMINATING_PUSH_NEEDED) { + return createPushRequest(localSubscription); + } + + log.warn("Cannot make telemetry request as telemetry is in state: {}", localState); + return Optional.empty(); + } + + @Override + public void handleResponse(GetTelemetrySubscriptionsResponse response) { + final long nowMs = time.milliseconds(); + final GetTelemetrySubscriptionsResponseData data = response.data(); + + final ClientTelemetryState oldState; + final ClientTelemetrySubscription oldSubscription; + lock.readLock().lock(); + try { + oldState = state; + oldSubscription = subscription; + } finally { + lock.readLock().unlock(); + } + + Optional errorIntervalMsOpt = ClientTelemetryUtils.maybeFetchErrorIntervalMs(data.errorCode(), + oldSubscription != null ? oldSubscription.pushIntervalMs() : -1); + /* + If the error code indicates that the interval ms needs to be updated as per the error + code then update the interval ms and state so that the subscription can be retried. + */ + if (errorIntervalMsOpt.isPresent()) { + /* + Update the state from SUBSCRIPTION_INR_PROGRESS to SUBSCRIPTION_NEEDED as the error + response indicates that the subscription is not valid. + */ + if (!maybeSetState(ClientTelemetryState.SUBSCRIPTION_NEEDED)) { + log.warn("Unable to transition state after failed get telemetry subscriptions from state {}", oldState); + } + updateErrorResult(errorIntervalMsOpt.get(), nowMs); + return; + } + + Uuid clientInstanceId = ClientTelemetryUtils.validateClientInstanceId(data.clientInstanceId()); + int intervalMs = ClientTelemetryUtils.validateIntervalMs(data.pushIntervalMs()); + Predicate selector = ClientTelemetryUtils.getSelectorFromRequestedMetrics( + data.requestedMetrics()); + List acceptedCompressionTypes = ClientTelemetryUtils.getCompressionTypesFromAcceptedList( + data.acceptedCompressionTypes()); + + /* + Check if the delta temporality has changed, if so, we need to reset the ledger tracking + the last value sent for each metric. + */ + if (oldSubscription != null && oldSubscription.deltaTemporality() != data.deltaTemporality()) { + log.info("Delta temporality has changed from {} to {}, resetting metric values", + oldSubscription.deltaTemporality(), data.deltaTemporality()); + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.metricsReset(); + } + } + + ClientTelemetrySubscription clientTelemetrySubscription = new ClientTelemetrySubscription( + clientInstanceId, + data.subscriptionId(), + intervalMs, + acceptedCompressionTypes, + data.deltaTemporality(), + selector); + + lock.writeLock().lock(); + try { + /* + This is the case if we began termination sometime after the subscription request + was issued. We're just now getting our callback, but we need to ignore it. + */ + if (isTerminatingState()) { + return; + } + + ClientTelemetryState newState; + if (selector == ClientTelemetryUtils.SELECTOR_NO_METRICS) { + /* + This is the case where no metrics are requested and/or match the filters. We need + to wait intervalMs then retry. + */ + newState = ClientTelemetryState.SUBSCRIPTION_NEEDED; + } else { + newState = ClientTelemetryState.PUSH_NEEDED; + } + + // If we're terminating, don't update state or set the subscription. + if (!maybeSetState(newState)) { + return; + } + + updateSubscriptionResult(clientTelemetrySubscription, nowMs); + log.info("Client telemetry registered with client instance id: {}", subscription.clientInstanceId()); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void handleResponse(PushTelemetryResponse response) { + final long nowMs = time.milliseconds(); + final PushTelemetryResponseData data = response.data(); + + lock.writeLock().lock(); + try { + Optional errorIntervalMsOpt = ClientTelemetryUtils.maybeFetchErrorIntervalMs(data.errorCode(), + subscription.pushIntervalMs()); + /* + If the error code indicates that the interval ms needs to be updated as per the error + code then update the interval ms and state so that the subscription can be re-fetched, + and the push retried. + */ + if (errorIntervalMsOpt.isPresent()) { + /* + This is the case when client began termination sometime after the last push request + was issued. Just getting the callback, hence need to ignore it. + */ + if (isTerminatingState()) { + return; + } + + if (!maybeSetState(ClientTelemetryState.SUBSCRIPTION_NEEDED)) { + log.warn("Unable to transition state after failed push telemetry from state {}", state); + } + updateErrorResult(errorIntervalMsOpt.get(), nowMs); + return; + } + + lastRequestMs = nowMs; + intervalMs = subscription.pushIntervalMs(); + if (!maybeSetState(ClientTelemetryState.PUSH_NEEDED)) { + log.warn("Unable to transition state after successful push telemetry from state {}", state); + } + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void handleFailedGetTelemetrySubscriptionsRequest(KafkaException maybeFatalException) { + log.debug("The broker generated an error for the get telemetry network API request", maybeFatalException); + handleFailedRequest(maybeFatalException != null); + } + + @Override + public void handleFailedPushTelemetryRequest(KafkaException maybeFatalException) { + log.debug("The broker generated an error for the push telemetry network API request", maybeFatalException); + handleFailedRequest(maybeFatalException != null); + } + + @Override + public Optional clientInstanceId(Duration timeout) { + final long timeoutMs = timeout.toMillis(); + if (timeoutMs < 0) { + throw new IllegalArgumentException("The timeout cannot be negative for fetching client instance id."); + } + + lock.writeLock().lock(); + try { + if (subscription == null) { + // If we have a non-negative timeout and no-subscription, let's wait for one to be retrieved. + log.debug("Waiting for telemetry subscription containing the client instance ID with timeoutMillis = {} ms.", timeoutMs); + try { + if (!subscriptionLoaded.await(timeoutMs, TimeUnit.MILLISECONDS)) { + log.debug("Wait for telemetry subscription elapsed; may not have actually loaded it"); + } + } catch (InterruptedException e) { + throw new InterruptException(e); + } + } + + if (subscription == null) { + log.debug("Client instance ID could not be retrieved with timeout {}", timeout); + return Optional.empty(); + } + + Uuid clientInstanceId = subscription.clientInstanceId(); + if (clientInstanceId == null) { + log.info("Client instance ID was null in telemetry subscription while in state {}", state); + return Optional.empty(); + } + + return Optional.of(clientInstanceId); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void close() { + log.debug("close telemetry sender for client telemetry reporter instance"); + + boolean shouldClose = false; + lock.writeLock().lock(); + try { + if (state != ClientTelemetryState.TERMINATED) { + if (maybeSetState(ClientTelemetryState.TERMINATED)) { + shouldClose = true; + } + } else { + log.debug("Ignoring subsequent close"); + } + } finally { + lock.writeLock().unlock(); + } + + if (shouldClose) { + if (kafkaMetricsCollector != null) { + kafkaMetricsCollector.stop(); + } + } + } + + @Override + public void initiateClose(long timeoutMs) { + log.debug("initiate close for client telemetry, check if terminal push required. Timeout {} ms.", timeoutMs); + + lock.writeLock().lock(); + try { + // If we never fetched a subscription, we can't really push anything. + if (lastRequestMs == 0) { + log.debug("Telemetry subscription not loaded, not attempting terminating push"); + return; + } + + if (state == ClientTelemetryState.SUBSCRIPTION_NEEDED) { + log.debug("Subscription not yet loaded, ignoring terminal push"); + return; + } + + if (isTerminatingState() || !maybeSetState(ClientTelemetryState.TERMINATING_PUSH_NEEDED)) { + log.debug("Ignoring subsequent initiateClose"); + return; + } + + try { + log.info("About to wait {} ms. for terminal telemetry push to be submitted", timeoutMs); + if (!terminalPushInProgress.await(timeoutMs, TimeUnit.MILLISECONDS)) { + log.info("Wait for terminal telemetry push to be submitted has elapsed; may not have actually sent request"); + } + } catch (InterruptedException e) { + log.warn("Error during client telemetry close", e); + } + } finally { + lock.writeLock().unlock(); + } + } + + private Optional> createSubscriptionRequest(ClientTelemetrySubscription localSubscription) { + /* + If we've previously retrieved a subscription, it will contain the client instance ID + that the broker assigned. Otherwise, per KIP-714, we send a special "null" UUID to + signal to the broker that we need to have a client instance ID assigned. + */ + Uuid clientInstanceId = (localSubscription != null) ? localSubscription.clientInstanceId() : Uuid.ZERO_UUID; + + lock.writeLock().lock(); + try { + if (isTerminatingState()) { + return Optional.empty(); + } + + if (!maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)) { + return Optional.empty(); + } + } finally { + lock.writeLock().unlock(); + } + + AbstractRequest.Builder requestBuilder = new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData().setClientInstanceId(clientInstanceId), true); + return Optional.of(requestBuilder); + } + + private Optional> createPushRequest(ClientTelemetrySubscription localSubscription) { + if (localSubscription == null) { + log.warn("Telemetry state is {} but subscription is null; not sending telemetry", state); + if (!maybeSetState(ClientTelemetryState.SUBSCRIPTION_NEEDED)) { + log.warn("Unable to transition state after failed create push telemetry from state {}", state); + } + return Optional.empty(); + } + + /* + Don't send a push request if we don't have the collector initialized. Re-attempt + the push on the next interval. + */ + if (kafkaMetricsCollector == null) { + log.warn("Cannot make telemetry request as collector is not initialized"); + // Update last accessed time for push request to be retried on next interval. + updateErrorResult(localSubscription.pushIntervalMs, time.milliseconds()); + return Optional.empty(); + } + + boolean terminating; + lock.writeLock().lock(); + try { + /* + We've already been terminated, or we've already issued our last push, so we + should just exit now. + */ + if (state == ClientTelemetryState.TERMINATED || state == ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS) { + return Optional.empty(); + } + + /* + Check the *actual* state (while locked) to make sure we're still in the state + we expect to be in. + */ + terminating = state == ClientTelemetryState.TERMINATING_PUSH_NEEDED; + if (!maybeSetState(terminating ? ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS : ClientTelemetryState.PUSH_IN_PROGRESS)) { + return Optional.empty(); + } + } finally { + lock.writeLock().unlock(); + } + + byte[] payload; + try (MetricsEmitter emitter = new ClientTelemetryEmitter(localSubscription.selector(), localSubscription.deltaTemporality())) { + emitter.init(); + kafkaMetricsCollector.collect(emitter); + payload = createPayload(emitter.emittedMetrics()); + } catch (Exception e) { + log.warn("Error constructing client telemetry payload: ", e); + // Update last accessed time for push request to be retried on next interval. + updateErrorResult(localSubscription.pushIntervalMs, time.milliseconds()); + return Optional.empty(); + } + + CompressionType compressionType = ClientTelemetryUtils.preferredCompressionType(localSubscription.acceptedCompressionTypes()); + ByteBuffer buffer = ClientTelemetryUtils.compress(payload, compressionType); + + AbstractRequest.Builder requestBuilder = new PushTelemetryRequest.Builder( + new PushTelemetryRequestData() + .setClientInstanceId(localSubscription.clientInstanceId()) + .setSubscriptionId(localSubscription.subscriptionId()) + .setTerminating(terminating) + .setCompressionType(compressionType.id) + .setMetrics(Utils.readBytes(buffer)), true); + + return Optional.of(requestBuilder); + } + + /** + * Updates the {@link ClientTelemetrySubscription}, {@link #intervalMs}, and + * {@link #lastRequestMs}. + *

    + * After the update, the {@link #subscriptionLoaded} condition is signaled so any threads + * waiting on the subscription can be unblocked. + * + * @param subscription Updated subscription that will replace any current subscription + * @param timeMs Time in milliseconds representing the current time + */ + // Visible for testing + void updateSubscriptionResult(ClientTelemetrySubscription subscription, long timeMs) { + lock.writeLock().lock(); + try { + this.subscription = Objects.requireNonNull(subscription); + /* + If the subscription is updated for the client, we want to attempt to spread out the push + requests between 50% and 150% of the push interval value from the broker. This helps us + to avoid the case where multiple clients are started at the same time and end up sending + all their data at the same time. + */ + if (state == ClientTelemetryState.PUSH_NEEDED) { + intervalMs = computeStaggeredIntervalMs(subscription.pushIntervalMs(), INITIAL_PUSH_JITTER_LOWER, INITIAL_PUSH_JITTER_UPPER); + } else { + intervalMs = subscription.pushIntervalMs(); + } + lastRequestMs = timeMs; + + log.debug("Updating subscription - subscription: {}; intervalMs: {}, lastRequestMs: {}", + subscription, intervalMs, lastRequestMs); + subscriptionLoaded.signalAll(); + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Updates the {@link #intervalMs}, {@link #lastRequestMs} and {@link #enabled}. + *

    + * The contents of the method are guarded by the {@link #lock}. + */ + private void updateErrorResult(int intervalMs, long timeMs) { + lock.writeLock().lock(); + try { + this.intervalMs = intervalMs; + lastRequestMs = timeMs; + /* + If the interval time is set to Integer.MAX_VALUE, then it means that the telemetry sender + should not send any more telemetry requests. This is used when the client received + unrecoverable error from broker. + */ + if (intervalMs == Integer.MAX_VALUE) { + enabled = false; + } + + log.debug("Updating intervalMs: {}, lastRequestMs: {}", intervalMs, lastRequestMs); + } finally { + lock.writeLock().unlock(); + } + } + + // Visible for testing + int computeStaggeredIntervalMs(int intervalMs, double lowerBound, double upperBound) { + double rand = ThreadLocalRandom.current().nextDouble(lowerBound, upperBound); + int firstPushIntervalMs = (int) Math.round(rand * intervalMs); + + log.debug("Telemetry subscription push interval value from broker was {}; to stagger requests the first push" + + " interval is being adjusted to {}", intervalMs, firstPushIntervalMs); + return firstPushIntervalMs; + } + + private boolean isTerminatingState() { + return state == ClientTelemetryState.TERMINATED || state == ClientTelemetryState.TERMINATING_PUSH_NEEDED + || state == ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS; + } + + // Visible for testing + boolean maybeSetState(ClientTelemetryState newState) { + lock.writeLock().lock(); + try { + ClientTelemetryState oldState = state; + state = oldState.validateTransition(newState); + log.debug("Setting telemetry state from {} to {}", oldState, newState); + + if (newState == ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS) { + terminalPushInProgress.signalAll(); + } + return true; + } catch (IllegalStateException e) { + log.warn("Error updating client telemetry state, disabled telemetry", e); + enabled = false; + return false; + } finally { + lock.writeLock().unlock(); + } + } + + private void handleFailedRequest(boolean shouldWait) { + final long nowMs = time.milliseconds(); + lock.writeLock().lock(); + try { + if (isTerminatingState()) { + return; + } + if (state != ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS && state != ClientTelemetryState.PUSH_IN_PROGRESS) { + log.warn("Could not transition state after failed telemetry from state {}, disabling telemetry", state); + updateErrorResult(Integer.MAX_VALUE, nowMs); + return; + } + + /* + The broker might not support telemetry. Let's sleep for a while before trying request + again. We may disconnect from the broker and connect to a broker that supports client + telemetry. + */ + if (shouldWait) { + updateErrorResult(DEFAULT_PUSH_INTERVAL_MS, nowMs); + } else { + log.warn("Received unrecoverable error from broker, disabling telemetry"); + updateErrorResult(Integer.MAX_VALUE, nowMs); + } + + if (!maybeSetState(ClientTelemetryState.SUBSCRIPTION_NEEDED)) { + log.warn("Could not transition state after failed telemetry from state {}", state); + } + } finally { + lock.writeLock().unlock(); + } + } + + private byte[] createPayload(List emittedMetrics) { + MetricsData.Builder builder = MetricsData.newBuilder(); + emittedMetrics.forEach(metric -> { + Metric m = metric.builder().build(); + ResourceMetrics rm = buildMetric(m); + builder.addResourceMetrics(rm); + }); + return builder.build().toByteArray(); + } + + // Visible for testing + ClientTelemetrySubscription subscription() { + lock.readLock().lock(); + try { + return subscription; + } finally { + lock.readLock().unlock(); + } + } + + // Visible for testing + ClientTelemetryState state() { + lock.readLock().lock(); + try { + return state; + } finally { + lock.readLock().unlock(); + } + } + + // Visible for testing + long intervalMs() { + lock.readLock().lock(); + try { + return intervalMs; + } finally { + lock.readLock().unlock(); + } + } + + // Visible for testing + long lastRequestMs() { + lock.readLock().lock(); + try { + return lastRequestMs; + } finally { + lock.readLock().unlock(); + } + } + + // Visible for testing + void enabled(boolean enabled) { + lock.writeLock().lock(); + try { + this.enabled = enabled; + } finally { + lock.writeLock().unlock(); + } + } + + // Visible for testing + boolean enabled() { + lock.readLock().lock(); + try { + return enabled; + } finally { + lock.readLock().unlock(); + } + } + } + + /** + * Representation of the telemetry subscription that is retrieved from the cluster at startup and + * then periodically afterward, following the telemetry push. + */ + static class ClientTelemetrySubscription { + + private final Uuid clientInstanceId; + private final int subscriptionId; + private final int pushIntervalMs; + private final List acceptedCompressionTypes; + private final boolean deltaTemporality; + private final Predicate selector; + + ClientTelemetrySubscription(Uuid clientInstanceId, int subscriptionId, int pushIntervalMs, + List acceptedCompressionTypes, boolean deltaTemporality, + Predicate selector) { + this.clientInstanceId = clientInstanceId; + this.subscriptionId = subscriptionId; + this.pushIntervalMs = pushIntervalMs; + this.acceptedCompressionTypes = Collections.unmodifiableList(acceptedCompressionTypes); + this.deltaTemporality = deltaTemporality; + this.selector = selector; + } + + public Uuid clientInstanceId() { + return clientInstanceId; + } + + public int subscriptionId() { + return subscriptionId; + } + + public int pushIntervalMs() { + return pushIntervalMs; + } + + public List acceptedCompressionTypes() { + return acceptedCompressionTypes; + } + + public boolean deltaTemporality() { + return deltaTemporality; + } + + public Predicate selector() { + return selector; + } + + @Override + public String toString() { + return new StringJoiner(", ", "ClientTelemetrySubscription{", "}") + .add("clientInstanceId=" + clientInstanceId) + .add("subscriptionId=" + subscriptionId) + .add("pushIntervalMs=" + pushIntervalMs) + .add("acceptedCompressionTypes=" + acceptedCompressionTypes) + .add("deltaTemporality=" + deltaTemporality) + .add("selector=" + selector) + .toString(); + } + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetrySender.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetrySender.java index b8499a3af337d..e6bb9d40e731c 100644 --- a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetrySender.java +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetrySender.java @@ -17,7 +17,10 @@ package org.apache.kafka.common.telemetry.internals; +import java.time.Duration; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.requests.AbstractRequest.Builder; import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; import org.apache.kafka.common.requests.PushTelemetryResponse; @@ -76,4 +79,36 @@ public interface ClientTelemetrySender extends AutoCloseable { * @param kafkaException the fatal exception. */ void handleFailedPushTelemetryRequest(KafkaException kafkaException); + + /** + * Determines the client's unique client instance ID used for telemetry. This ID is unique to + * the specific enclosing client instance and will not change after it is initially generated. + * The ID is useful for correlating client operations with telemetry sent to the broker and + * to its eventual monitoring destination(s). + *

    + * This method waits up to timeout for the subscription to become available in + * order to complete the request. + * + * @param timeout The maximum time to wait for enclosing client instance to determine its + * client instance ID. The value must be non-negative. Specifying a timeout + * of zero means do not wait for the initial request to complete if it hasn't + * already. + * @throws InterruptException If the thread is interrupted while blocked. + * @throws KafkaException If an unexpected error occurs while trying to determine the client + * instance ID, though this error does not necessarily imply the + * enclosing client instance is otherwise unusable. + * @throws IllegalArgumentException If the timeout is negative. + * + * @return If present, optional of the client's assigned instance id used for metrics collection. + */ + + Optional clientInstanceId(Duration timeout); + + /** + * Initiates shutdown of this client. This method is called when the enclosing client instance + * is being closed. This method should not throw an exception if the client is already closed. + * + * @param timeoutMs The maximum time to wait for the client to close. + */ + void initiateClose(long timeoutMs); } diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtils.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtils.java new file mode 100644 index 0000000000000..d1d62a7efe212 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtils.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.metrics.v1.MetricsData; + +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.metrics.MetricsContext; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; + +public class ClientTelemetryUtils { + + private final static Logger log = LoggerFactory.getLogger(ClientTelemetryUtils.class); + + public final static Predicate SELECTOR_NO_METRICS = k -> false; + + public final static Predicate SELECTOR_ALL_METRICS = k -> true; + + /** + * Examine the response data and handle different error code accordingly: + * + *

      + *
    • Invalid Request: Disable Telemetry
      • + *
    • Invalid Record: Disable Telemetry
      • + *
    • Unsupported Version: Disable Telemetry
      • + *
    • UnknownSubscription or Unsupported Compression: Retry immediately
      • + *
    • TelemetryTooLarge or ThrottlingQuotaExceeded: Retry as per next interval
      • + *
    + * + * @param errorCode response body error code + * @param intervalMs current push interval in milliseconds + * + * @return Optional of push interval in milliseconds + */ + public static Optional maybeFetchErrorIntervalMs(short errorCode, int intervalMs) { + if (errorCode == Errors.NONE.code()) + return Optional.empty(); + + int pushIntervalMs; + String reason; + + Errors error = Errors.forCode(errorCode); + switch (error) { + case INVALID_REQUEST: + case INVALID_RECORD: + case UNSUPPORTED_VERSION: + pushIntervalMs = Integer.MAX_VALUE; + reason = "The broker response indicates the client sent an request that cannot be resolved" + + " by re-trying, hence disable telemetry"; + break; + case UNKNOWN_SUBSCRIPTION_ID: + case UNSUPPORTED_COMPRESSION_TYPE: + pushIntervalMs = 0; + reason = error.message(); + break; + case TELEMETRY_TOO_LARGE: + case THROTTLING_QUOTA_EXCEEDED: + reason = error.message(); + pushIntervalMs = (intervalMs != -1) ? intervalMs : ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS; + break; + default: + reason = "Unwrapped error code"; + log.error("Error code: {}. Unmapped error for telemetry, disable telemetry.", errorCode); + pushIntervalMs = Integer.MAX_VALUE; + } + + log.debug("Error code: {}, reason: {}. Push interval update to {} ms.", errorCode, reason, pushIntervalMs); + return Optional.of(pushIntervalMs); + } + + public static Predicate getSelectorFromRequestedMetrics(List requestedMetrics) { + if (requestedMetrics == null || requestedMetrics.isEmpty()) { + log.debug("Telemetry subscription has specified no metric names; telemetry will record no metrics"); + return SELECTOR_NO_METRICS; + } else if (requestedMetrics.size() == 1 && requestedMetrics.get(0) != null && requestedMetrics.get(0).equals("*")) { + log.debug("Telemetry subscription has specified a single '*' metric name; using all metrics"); + return SELECTOR_ALL_METRICS; + } else { + log.debug("Telemetry subscription has specified to include only metrics that are prefixed with the following strings: {}", requestedMetrics); + return k -> requestedMetrics.stream().anyMatch(f -> k.key().name().startsWith(f)); + } + } + + public static List getCompressionTypesFromAcceptedList(List acceptedCompressionTypes) { + if (acceptedCompressionTypes == null || acceptedCompressionTypes.isEmpty()) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(); + for (Byte compressionByte : acceptedCompressionTypes) { + int compressionId = compressionByte.intValue(); + try { + CompressionType compressionType = CompressionType.forId(compressionId); + result.add(compressionType); + } catch (IllegalArgumentException e) { + log.warn("Accepted compressionByte type with ID {} is not a known compressionByte type; ignoring", compressionId, e); + } + } + return result; + } + + public static Uuid validateClientInstanceId(Uuid clientInstanceId) { + if (clientInstanceId == null || clientInstanceId == Uuid.ZERO_UUID) { + throw new IllegalArgumentException("clientInstanceId is not valid"); + } + + return clientInstanceId; + } + + public static int validateIntervalMs(int intervalMs) { + if (intervalMs <= 0) { + log.warn("Telemetry subscription push interval value from broker was invalid ({})," + + " substituting with default value of {}", intervalMs, ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS); + return ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS; + } + + log.debug("Telemetry subscription push interval value from broker: {}", intervalMs); + return intervalMs; + } + + public static boolean validateResourceLabel(Map m, String key) { + if (!m.containsKey(key)) { + log.trace("{} does not exist in map {}", key, m); + return false; + } + + if (m.get(key) == null) { + log.trace("{} is null. map {}", key, m); + return false; + } + + if (!(m.get(key) instanceof String)) { + log.trace("{} is not a string. map {}", key, m); + return false; + } + + String val = (String) m.get(key); + if (val.isEmpty()) { + log.trace("{} is empty string. value = {} map {}", key, val, m); + return false; + } + return true; + } + + public static boolean validateRequiredResourceLabels(Map metadata) { + return validateResourceLabel(metadata, MetricsContext.NAMESPACE); + } + + public static CompressionType preferredCompressionType(List acceptedCompressionTypes) { + // TODO: Support compression in client telemetry. + return CompressionType.NONE; + } + + public static ByteBuffer compress(byte[] raw, CompressionType compressionType) { + // TODO: Support compression in client telemetry. + if (compressionType == CompressionType.NONE) { + return ByteBuffer.wrap(raw); + } else { + throw new UnsupportedOperationException("Compression is not supported"); + } + } + + public static MetricsData deserializeMetricsData(ByteBuffer serializedMetricsData) { + try { + return MetricsData.parseFrom(serializedMetricsData); + } catch (IOException e) { + throw new KafkaException("Unable to parse MetricsData payload", e); + } + } + + public static Uuid fetchClientInstanceId(ClientTelemetryReporter clientTelemetryReporter, Duration timeout) { + if (timeout.isNegative()) { + throw new IllegalArgumentException("The timeout cannot be negative."); + } + + Optional optionalUuid = clientTelemetryReporter.telemetrySender().clientInstanceId(timeout); + return optionalUuid.orElse(null); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollector.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollector.java new file mode 100644 index 0000000000000..3aea4a1187952 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollector.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Gauge; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Measurable; +import org.apache.kafka.common.metrics.MetricValueProvider; +import org.apache.kafka.common.metrics.stats.Avg; +import org.apache.kafka.common.metrics.stats.CumulativeCount; +import org.apache.kafka.common.metrics.stats.CumulativeSum; +import org.apache.kafka.common.metrics.stats.Frequencies; +import org.apache.kafka.common.metrics.stats.Max; +import org.apache.kafka.common.metrics.stats.Meter; +import org.apache.kafka.common.metrics.stats.Min; +import org.apache.kafka.common.metrics.stats.Percentiles; +import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.metrics.stats.SimpleRate; +import org.apache.kafka.common.metrics.stats.WindowedCount; +import org.apache.kafka.common.telemetry.internals.LastValueTracker.InstantAndValue; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * All metrics implement the {@link MetricValueProvider} interface. They are divided into + * two base types: + * + *
      + *
    1. {@link Gauge}
      2. + *
    2. {@link Measurable}
      3. + *
    + * + * {@link Gauge Gauges} can have any value, but we only collect metrics with number values. + * {@link Measurable Measurables} are divided into simple types with single values + * ({@link Avg}, {@link CumulativeCount}, {@link Min}, {@link Max}, {@link Rate}, + * {@link SimpleRate}, and {@link CumulativeSum}) and compound types ({@link Frequencies}, + * {@link Meter}, and {@link Percentiles}). + * + *

    + * + * We can safely assume that a {@link CumulativeCount count} always increases in steady state. It + * should be a bug if a count metric decreases. + * + *

    + * + * Total and Sum are treated as a monotonically increasing counter. The javadocs for Total metric type + * say "An un-windowed cumulative total maintained over all time.". The standalone Total metrics in + * the codebase seem to be cumulative metrics that will always increase. The Total metric underlying + * Meter type is mostly a Total of a Count metric. + * We can assume that a Total metric always increases (but it is not guaranteed as the sample values might be both + * negative or positive). + * For now, Total is converted to CUMULATIVE_DOUBLE unless we find a valid counter-example. + * + *

    + * + * The Sum as it is a sample sum which is not a cumulative metric. It is converted to GAUGE_DOUBLE. + * + *

    + * + * The compound metrics are virtual metrics. They are composed of simple types or anonymous measurable types + * which are reported. A compound metric is never reported as-is. + * + *

    + * + * A Meter metric is always created with and reported as 2 metrics: a rate and a count. For eg: + * org.apache.kafka.common.network.Selector has Meter metric for "connection-close" but it has to be + * created with a "connection-close-rate" metric of type rate and a "connection-close-total" + * metric of type total. + * + *

    + * + * Frequencies is created with an array of Frequency objects. When a Frequencies metric is registered, each + * member Frequency object is converted into an anonymous Measurable and registered. So, a Frequencies metric + * is reported with a set of measurables with name = Frequency.name(). As there is no way to figure out the + * compound type, each component measurables is converted to a GAUGE_DOUBLE. + * + *

    + * + * Percentiles work the same way as Frequencies. The only difference is that it is composed of Percentile + * types instead. So, we should treat the component measurable as GAUGE_DOUBLE. + * + *

    + * + * Some metrics are defined as either anonymous inner classes or lambdas implementing the Measurable + * interface. As we do not have any information on how to treat them, we should fallback to treating + * them as GAUGE_DOUBLE. + * + *

    + * + * OpenTelemetry mapping for measurables: + * Avg / Rate / Min / Max / Total / Sum -> Gauge + * Count -> Sum + * Meter has 2 elements : + * Total -> Sum + * Rate -> Gauge + * Frequencies -> each component is Gauge + * Percentiles -> each component is Gauge + */ +public class KafkaMetricsCollector implements MetricsCollector { + + private static final Logger log = LoggerFactory.getLogger(KafkaMetricsCollector.class); + + private final StateLedger ledger; + private final Time time; + private final MetricNamingStrategy metricNamingStrategy; + + private static final Field METRIC_VALUE_PROVIDER_FIELD; + + static { + try { + METRIC_VALUE_PROVIDER_FIELD = KafkaMetric.class.getDeclaredField("metricValueProvider"); + METRIC_VALUE_PROVIDER_FIELD.setAccessible(true); + } catch (Exception e) { + throw new KafkaException(e); + } + } + + public KafkaMetricsCollector(MetricNamingStrategy metricNamingStrategy) { + this(metricNamingStrategy, Time.SYSTEM); + } + + // Visible for testing + KafkaMetricsCollector(MetricNamingStrategy metricNamingStrategy, Time time) { + this.metricNamingStrategy = metricNamingStrategy; + this.time = time; + this.ledger = new StateLedger(); + } + + public void init(List metrics) { + ledger.init(metrics); + } + + /** + * This is called whenever a metric is updated or added. + */ + public void metricChange(KafkaMetric metric) { + ledger.metricChange(metric); + } + + /** + * This is called whenever a metric is removed. + */ + public void metricRemoval(KafkaMetric metric) { + ledger.metricRemoval(metric); + } + + /** + * This is called whenever temporality changes, resets the value tracker for metrics. + */ + public void metricsReset() { + ledger.metricsStateReset(); + } + + // Visible for testing + Set getTrackedMetrics() { + return ledger.metricMap.keySet(); + } + + @Override + public void collect(MetricsEmitter metricsEmitter) { + for (Map.Entry entry : ledger.getMetrics()) { + MetricKey metricKey = entry.getKey(); + KafkaMetric metric = entry.getValue(); + + try { + collectMetric(metricsEmitter, metricKey, metric); + } catch (Exception e) { + // catch and log to continue processing remaining metrics + log.error("Error processing Kafka metric {}", metricKey, e); + } + } + } + + protected void collectMetric(MetricsEmitter metricsEmitter, MetricKey metricKey, KafkaMetric metric) { + Object metricValue; + + try { + metricValue = metric.metricValue(); + } catch (Exception e) { + // If an exception occurs when retrieving value, log warning and continue to process the rest of metrics + log.warn("Failed to retrieve metric value {}", metricKey.name(), e); + return; + } + + Instant now = Instant.ofEpochMilli(time.milliseconds()); + if (isMeasurable(metric)) { + Measurable measurable = metric.measurable(); + Double value = (Double) metricValue; + + if (measurable instanceof WindowedCount || measurable instanceof CumulativeSum) { + collectSum(metricKey, value, metricsEmitter, now); + } else { + collectGauge(metricKey, value, metricsEmitter, now); + } + } else { + // It is non-measurable Gauge metric. + // Collect the metric only if its value is a number. + if (metricValue instanceof Number) { + Number value = (Number) metricValue; + collectGauge(metricKey, value, metricsEmitter, now); + } else { + // skip non-measurable metrics + log.debug("Skipping non-measurable gauge metric {}", metricKey.name()); + } + } + } + + private void collectSum(MetricKey metricKey, double value, MetricsEmitter metricsEmitter, Instant timestamp) { + if (!metricsEmitter.shouldEmitMetric(metricKey)) { + return; + } + + if (metricsEmitter.shouldEmitDeltaMetrics()) { + InstantAndValue instantAndValue = ledger.delta(metricKey, timestamp, value); + + metricsEmitter.emitMetric( + SinglePointMetric.deltaSum(metricKey, instantAndValue.getValue(), true, timestamp, + instantAndValue.getIntervalStart()) + ); + } else { + metricsEmitter.emitMetric( + SinglePointMetric.sum(metricKey, value, true, timestamp, ledger.instantAdded(metricKey)) + ); + } + } + + private void collectGauge(MetricKey metricKey, Number value, MetricsEmitter metricsEmitter, Instant timestamp) { + if (!metricsEmitter.shouldEmitMetric(metricKey)) { + return; + } + + metricsEmitter.emitMetric( + SinglePointMetric.gauge(metricKey, value, timestamp) + ); + } + + private static boolean isMeasurable(KafkaMetric metric) { + // KafkaMetric does not expose the internal MetricValueProvider and throws an IllegalStateException + // exception, if measurable() is called for a Gauge. + // There are 2 ways to find the type of internal MetricValueProvider for a KafkaMetric - use reflection or + // get the information based on whether a IllegalStateException exception is thrown. + // We use reflection so that we can avoid the cost of generating the stack trace when it's + // not a measurable. + try { + Object provider = METRIC_VALUE_PROVIDER_FIELD.get(metric); + return provider instanceof Measurable; + } catch (Exception e) { + throw new KafkaException(e); + } + } + + /** + * Keeps track of the state of metrics, e.g. when they were added, what their getAndSet value is, + * and clearing them out when they're removed. + */ + private class StateLedger { + + private final Map metricMap = new ConcurrentHashMap<>(); + private final LastValueTracker doubleDeltas = new LastValueTracker<>(); + private final Map metricAdded = new ConcurrentHashMap<>(); + + private Instant instantAdded(MetricKey metricKey) { + // lookup when the metric was added to use it as the interval start. That should always + // exist, but if it doesn't (e.g. changed metrics temporality) then we use now. + return metricAdded.computeIfAbsent(metricKey, x -> Instant.ofEpochMilli(time.milliseconds())); + } + + private void init(List metrics) { + log.info("initializing Kafka metrics collector"); + for (KafkaMetric m : metrics) { + metricMap.put(metricNamingStrategy.metricKey(m.metricName()), m); + } + } + + private void metricChange(KafkaMetric metric) { + MetricKey metricKey = metricNamingStrategy.metricKey(metric.metricName()); + metricMap.put(metricKey, metric); + if (doubleDeltas.contains(metricKey)) { + log.warn("Registering a new metric {} which already has a last value tracked. " + + "Removing metric from delta register.", metric.metricName(), new Exception()); + + /* + This scenario shouldn't occur while registering a metric since it should + have already been cleared out on cleanup/shutdown. + We remove the metric here to clear out the delta register because we are running + into an issue where old metrics are being re-registered which causes us to + record a negative delta + */ + doubleDeltas.remove(metricKey); + } + metricAdded.put(metricKey, Instant.ofEpochMilli(time.milliseconds())); + } + + private void metricRemoval(KafkaMetric metric) { + log.debug("removing kafka metric : {}", metric.metricName()); + MetricKey metricKey = metricNamingStrategy.metricKey(metric.metricName()); + metricMap.remove(metricKey); + doubleDeltas.remove(metricKey); + metricAdded.remove(metricKey); + } + + private Iterable> getMetrics() { + return metricMap.entrySet(); + } + + private InstantAndValue delta(MetricKey metricKey, Instant now, Double value) { + Optional> lastValue = doubleDeltas.getAndSet(metricKey, now, value); + + return lastValue + .map(last -> new InstantAndValue<>(last.getIntervalStart(), value - last.getValue())) + .orElse(new InstantAndValue<>(instantAdded(metricKey), value)); + } + + private void metricsStateReset() { + metricAdded.clear(); + doubleDeltas.reset(); + } + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/LastValueTracker.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/LastValueTracker.java new file mode 100644 index 0000000000000..117b40656e148 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/LastValueTracker.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A LastValueTracker uses a ConcurrentHashMap to maintain historic values for a given key, and return + * a previous value and an Instant for that value. + * + * @param The type of the value. + */ +public class LastValueTracker { + private final Map> counters = new ConcurrentHashMap<>(); + + /** + * Return the last instant/value for the given MetricKey, or Optional.empty if there isn't one. + * + * @param metricKey the key for which to calculate a getAndSet. + * @param now the timestamp for the new value. + * @param value the current value. + * @return the timestamp of the previous entry and its value. If there + * isn't a previous entry, then this method returns {@link Optional#empty()} + */ + public Optional> getAndSet(MetricKey metricKey, Instant now, T value) { + InstantAndValue instantAndValue = new InstantAndValue<>(now, value); + InstantAndValue valueOrNull = counters.put(metricKey, instantAndValue); + + // there wasn't already an entry, so return empty. + if (valueOrNull == null) { + return Optional.empty(); + } + + // Return the previous instance and the value. + return Optional.of(valueOrNull); + } + + public InstantAndValue remove(MetricKey metricKey) { + return counters.remove(metricKey); + } + + public boolean contains(MetricKey metricKey) { + return counters.containsKey(metricKey); + } + + public void reset() { + counters.clear(); + } + + public static class InstantAndValue { + + private final Instant intervalStart; + private final T value; + + public InstantAndValue(Instant intervalStart, T value) { + this.intervalStart = Objects.requireNonNull(intervalStart); + this.value = Objects.requireNonNull(value); + } + + public Instant getIntervalStart() { + return intervalStart; + } + + public T getValue() { + return value; + } + } + +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/MetricsEmitter.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/MetricsEmitter.java index 8ba3a13bd327e..6c8655d21b022 100644 --- a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/MetricsEmitter.java +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/MetricsEmitter.java @@ -29,10 +29,6 @@ * *

    * - * An {@code MetricsEmitter} is stateless and the telemetry reporter should assume that the object is - * not thread safe and thus concurrent access to either the - * {@link #shouldEmitMetric(MetricKeyable)} or {@link #emitMetric(SinglePointMetric)} should be avoided. - * * Regarding threading, the {@link #init()} and {@link #close()} methods may be called from * different threads and so proper care should be taken by implementations of the * {@code MetricsCollector} interface to be thread-safe. However, the telemetry reporter must @@ -50,6 +46,13 @@ public interface MetricsEmitter extends Closeable { */ boolean shouldEmitMetric(MetricKeyable metricKeyable); + /** + * Determines if the delta aggregation temporality metrics are to be emitted. + * + * @return {@code true} if the delta metric should be emitted, {@code false} otherwise + */ + boolean shouldEmitDeltaMetrics(); + /** * Emits the metric in an implementation-specific fashion. Depending on the implementation, * calls made to this after {@link #close()} has been invoked will fail. diff --git a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/SinglePointMetric.java b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/SinglePointMetric.java index f81a9bfb60ec3..4194bd469bf79 100644 --- a/clients/src/main/java/org/apache/kafka/common/telemetry/internals/SinglePointMetric.java +++ b/clients/src/main/java/org/apache/kafka/common/telemetry/internals/SinglePointMetric.java @@ -16,16 +16,28 @@ */ package org.apache.kafka.common.telemetry.internals; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.AggregationTemporality; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; + +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.TimeUnit; + /** - * This class represents a metric that does not yet contain resource tags. + * This class represents a telemetry metric that does not yet contain resource tags. * These additional resource tags will be added before emitting metrics by the telemetry reporter. */ public class SinglePointMetric implements MetricKeyable { private final MetricKey key; + private final Metric.Builder metricBuilder; - private SinglePointMetric(MetricKey key) { + private SinglePointMetric(MetricKey key, Metric.Builder metricBuilder) { this.key = key; + this.metricBuilder = metricBuilder; } @Override @@ -33,5 +45,102 @@ public MetricKey key() { return key; } - // TODO: Implement methods for serializing/deserializing metrics in required format. + public Metric.Builder builder() { + return metricBuilder; + } + + /* + Methods to construct gauge metric type. + */ + public static SinglePointMetric gauge(MetricKey metricKey, Number value, Instant timestamp) { + NumberDataPoint.Builder point = point(timestamp, value); + return gauge(metricKey, point); + } + + public static SinglePointMetric gauge(MetricKey metricKey, double value, Instant timestamp) { + NumberDataPoint.Builder point = point(timestamp, value); + return gauge(metricKey, point); + } + + /* + Methods to construct sum metric type. + */ + + public static SinglePointMetric sum(MetricKey metricKey, double value, boolean monotonic, Instant timestamp) { + return sum(metricKey, value, monotonic, timestamp, null); + } + + public static SinglePointMetric sum(MetricKey metricKey, double value, boolean monotonic, Instant timestamp, + Instant startTimestamp) { + NumberDataPoint.Builder point = point(timestamp, value); + if (startTimestamp != null) { + point.setStartTimeUnixNano(toTimeUnixNanos(startTimestamp)); + } + + return sum(metricKey, AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, monotonic, point); + } + + public static SinglePointMetric deltaSum(MetricKey metricKey, double value, boolean monotonic, + Instant timestamp, Instant startTimestamp) { + NumberDataPoint.Builder point = point(timestamp, value) + .setStartTimeUnixNano(toTimeUnixNanos(startTimestamp)); + + return sum(metricKey, AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, monotonic, point); + } + + /* + Helper methods to support metric construction. + */ + private static SinglePointMetric sum(MetricKey metricKey, AggregationTemporality aggregationTemporality, + boolean monotonic, NumberDataPoint.Builder point) { + point.addAllAttributes(asAttributes(metricKey.tags())); + + Metric.Builder metric = Metric.newBuilder().setName(metricKey.name()); + metric + .getSumBuilder() + .setAggregationTemporality(aggregationTemporality) + .setIsMonotonic(monotonic) + .addDataPoints(point); + return new SinglePointMetric(metricKey, metric); + } + + private static SinglePointMetric gauge(MetricKey metricKey, NumberDataPoint.Builder point) { + point.addAllAttributes(asAttributes(metricKey.tags())); + + Metric.Builder metric = Metric.newBuilder().setName(metricKey.name()); + metric.getGaugeBuilder().addDataPoints(point); + return new SinglePointMetric(metricKey, metric); + } + + private static NumberDataPoint.Builder point(Instant timestamp, Number value) { + if (value instanceof Long || value instanceof Integer) { + return point(timestamp, value.longValue()); + } + + return point(timestamp, value.doubleValue()); + } + + private static NumberDataPoint.Builder point(Instant timestamp, long value) { + return NumberDataPoint.newBuilder() + .setTimeUnixNano(toTimeUnixNanos(timestamp)) + .setAsInt(value); + } + + private static NumberDataPoint.Builder point(Instant timestamp, double value) { + return NumberDataPoint.newBuilder() + .setTimeUnixNano(toTimeUnixNanos(timestamp)) + .setAsDouble(value); + } + + private static Iterable asAttributes(Map labels) { + return labels.entrySet().stream().map( + entry -> KeyValue.newBuilder() + .setKey(entry.getKey()) + .setValue(AnyValue.newBuilder().setStringValue(entry.getValue())).build() + )::iterator; + } + + private static long toTimeUnixNanos(Instant t) { + return TimeUnit.SECONDS.toNanos(t.getEpochSecond()) + t.getNano(); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ConfigUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/ConfigUtils.java index 23de638ed9506..43834464d5be5 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/ConfigUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/ConfigUtils.java @@ -144,4 +144,25 @@ public static String configMapToRedactedString(Map map, ConfigDe bld.append("}"); return bld.toString(); } + + /** + * Finds and returns a boolean configuration option from the configuration map or the default value if the option is + * not set. + * + * @param configs Map with the configuration options + * @param key Configuration option for which the boolean value will be returned + * @param defaultValue The default value that will be used when the key is not present + * @return A boolean value of the configuration option of the default value + */ + public static boolean getBoolean(final Map configs, final String key, final boolean defaultValue) { + final Object value = configs.getOrDefault(key, defaultValue); + if (value instanceof Boolean) { + return (boolean) value; + } else if (value instanceof String) { + return Boolean.parseBoolean((String) value); + } else { + log.error("Invalid value (" + value + ") on configuration '" + key + "'. The default value '" + defaultValue + "' will be used instead. Please specify a true/false value."); + return defaultValue; + } + } } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java index 48bf3b7199e14..4e28bb35c669c 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/FlattenedIterator.java @@ -33,7 +33,7 @@ public FlattenedIterator(Iterator outerIterator, Function> inn } @Override - public I makeNext() { + protected I makeNext() { while (innerIterator == null || !innerIterator.hasNext()) { if (outerIterator.hasNext()) innerIterator = innerIteratorFunction.apply(outerIterator.next()); diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java index 6a0913d3c2da1..6e1c3cefbb1d6 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java @@ -1021,6 +1021,17 @@ public static void flushDirIfExists(Path path) throws IOException { } } + /** + * Flushes dirty file with swallowing {@link NoSuchFileException} + */ + public static void flushFileIfExists(Path path) throws IOException { + try (FileChannel fileChannel = FileChannel.open(path, StandardOpenOption.READ)) { + fileChannel.force(true); + } catch (NoSuchFileException e) { + log.warn("Failed to flush file {}", path, e); + } + } + /** * Closes all the provided closeables. * @throws IOException if any of the close methods throws an IOException. @@ -1543,7 +1554,7 @@ public static Iterator covariantCast(Iterator iterator) { * Checks if a string is null, empty or whitespace only. * @param str a string to be checked * @return true if the string is null, empty or whitespace only; otherwise, return false. - */ + */ public static boolean isBlank(String str) { return str == null || str.trim().isEmpty(); } diff --git a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json index 24e4f07489b50..ace268db77a6f 100644 --- a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json +++ b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json @@ -15,9 +15,9 @@ // Version 1 adds Zk broker epoch to the request if the broker is migrating from Zk mode to KRaft mode. -// Version 2 adds the PreviousBrokerEpoch for the KIP-966 +// Version 2 adds LogDirs for KIP-858 -// Version 3 adds LogDirs for KIP-858 +// Version 3 adds the PreviousBrokerEpoch for the KIP-966 { "apiKey":62, "type": "request", @@ -58,9 +58,9 @@ "about": "The rack which this broker is in." }, { "name": "IsMigratingZkBroker", "type": "bool", "versions": "1+", "default": "false", "about": "If the required configurations for ZK migration are present, this value is set to true" }, - { "name": "PreviousBrokerEpoch", "type": "int64", "versions": "2+", "default": "-1", - "about": "The epoch before a clean shutdown." }, - { "name": "LogDirs", "type": "[]uuid", "versions": "3+", - "about": "Log directories configured in this broker which are available." } + { "name": "LogDirs", "type": "[]uuid", "versions": "2+", + "about": "Log directories configured in this broker which are available." }, + { "name": "PreviousBrokerEpoch", "type": "int64", "versions": "3+", "default": "-1", "ignorable": true, + "about": "The epoch before a clean shutdown." } ] } diff --git a/clients/src/main/resources/common/message/ConsumerGroupDescribeResponse.json b/clients/src/main/resources/common/message/ConsumerGroupDescribeResponse.json index b0c98b78c1a14..f6839b2250c22 100644 --- a/clients/src/main/resources/common/message/ConsumerGroupDescribeResponse.json +++ b/clients/src/main/resources/common/message/ConsumerGroupDescribeResponse.json @@ -50,7 +50,7 @@ { "name": "Members", "type": "[]Member", "versions": "0+", "about": "The members.", "fields": [ - { "name": "MemberId", "type": "uuid", "versions": "0+", + { "name": "MemberId", "type": "string", "versions": "0+", "about": "The member ID." }, { "name": "InstanceId", "type": "string", "versions": "0+", "nullableVersions": "0+", "default": "null", "about": "The member instance ID." }, diff --git a/clients/src/main/resources/common/message/ConsumerGroupHeartbeatRequest.json b/clients/src/main/resources/common/message/ConsumerGroupHeartbeatRequest.json index e86a79fcd6800..95c26421ce26a 100644 --- a/clients/src/main/resources/common/message/ConsumerGroupHeartbeatRequest.json +++ b/clients/src/main/resources/common/message/ConsumerGroupHeartbeatRequest.json @@ -18,10 +18,6 @@ "type": "request", "listeners": ["zkBroker", "broker"], "name": "ConsumerGroupHeartbeatRequest", - // The ConsumerGroupHeartbeat API is added as part of KIP-848 and is still - // under development. Hence, the API is not exposed by default by brokers - // unless explicitly enabled. - "latestVersionUnstable": true, "validVersions": "0", "flexibleVersions": "0+", "fields": [ diff --git a/clients/src/main/resources/common/message/GetTelemetrySubscriptionsRequest.json b/clients/src/main/resources/common/message/GetTelemetrySubscriptionsRequest.json index 1020ae717c02d..3f2c5f99e4c00 100644 --- a/clients/src/main/resources/common/message/GetTelemetrySubscriptionsRequest.json +++ b/clients/src/main/resources/common/message/GetTelemetrySubscriptionsRequest.json @@ -20,10 +20,6 @@ "name": "GetTelemetrySubscriptionsRequest", "validVersions": "0", "flexibleVersions": "0+", - // The Telemetry APIs are added as part of KIP-714 and are still under - // development. Hence, the APIs are not exposed by default unless explicitly - // enabled. - "latestVersionUnstable": true, "fields": [ { "name": "ClientInstanceId", "type": "uuid", "versions": "0+", diff --git a/clients/src/main/resources/common/message/ListClientMetricsResourcesRequest.json b/clients/src/main/resources/common/message/ListClientMetricsResourcesRequest.json new file mode 100644 index 0000000000000..b54dce6b7c749 --- /dev/null +++ b/clients/src/main/resources/common/message/ListClientMetricsResourcesRequest.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 74, + "type": "request", + "listeners": ["broker"], + "name": "ListClientMetricsResourcesRequest", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + ] +} + \ No newline at end of file diff --git a/clients/src/main/resources/common/message/ListClientMetricsResourcesResponse.json b/clients/src/main/resources/common/message/ListClientMetricsResourcesResponse.json new file mode 100644 index 0000000000000..6d3321c17b61e --- /dev/null +++ b/clients/src/main/resources/common/message/ListClientMetricsResourcesResponse.json @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 74, + "type": "response", + "name": "ListClientMetricsResourcesResponse", + "validVersions": "0", + "flexibleVersions": "0+", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+" }, + { "name": "ClientMetricsResources", "type": "[]ClientMetricsResource", "versions": "0+", "fields": [ + { "name": "Name", "type": "string", "versions": "0+" } + ]} + ] +} diff --git a/clients/src/main/resources/common/message/OffsetCommitRequest.json b/clients/src/main/resources/common/message/OffsetCommitRequest.json index c11b56c95448a..27edbba5272ac 100644 --- a/clients/src/main/resources/common/message/OffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/OffsetCommitRequest.json @@ -34,9 +34,6 @@ // // Version 9 is the first version that can be used with the new consumer group protocol (KIP-848). The // request is the same as version 8. - // Version 9 is added as part of KIP-848 and is still under development. Hence, the last version of the - // API is not exposed by default by brokers unless explicitly enabled. - "latestVersionUnstable": true, "validVersions": "0-9", "flexibleVersions": "8+", "fields": [ diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json index 33dab5c957c18..85a5c2d399b21 100644 --- a/clients/src/main/resources/common/message/OffsetFetchRequest.json +++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json @@ -36,10 +36,6 @@ // // Version 9 is the first version that can be used with the new consumer group protocol (KIP-848). It adds // the MemberId and MemberEpoch fields. Those are filled in and validated when the new consumer protocol is used. - // - // Version 9 is added as part of KIP-848 and is still under development. Hence, the last version of the - // API is not exposed by default by brokers unless explicitly enabled. - "latestVersionUnstable": true, "validVersions": "0-9", "flexibleVersions": "6+", "fields": [ diff --git a/clients/src/main/resources/common/message/PushTelemetryRequest.json b/clients/src/main/resources/common/message/PushTelemetryRequest.json index b01c458f0450f..b91cc7d94f7da 100644 --- a/clients/src/main/resources/common/message/PushTelemetryRequest.json +++ b/clients/src/main/resources/common/message/PushTelemetryRequest.json @@ -20,10 +20,6 @@ "name": "PushTelemetryRequest", "validVersions": "0", "flexibleVersions": "0+", - // The Telemetry APIs are added as part of KIP-714 and are still under - // development. Hence, the APIs are not exposed by default unless explicitly - // enabled. - "latestVersionUnstable": true, "fields": [ { "name": "ClientInstanceId", "type": "uuid", "versions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json index e876caa2bac19..1b90dee6a7ad8 100644 --- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json +++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json @@ -38,6 +38,9 @@ "about": "The controller id." }, { "name": "isKRaftController", "type": "bool", "versions": "8+", "default": "false", "about": "If KRaft controller id is used during migration. See KIP-866" }, + { "name": "Type", "type": "int8", "versions": "8+", + "default": 0, "tag": 0, "taggedVersions": "8+", + "about": "Indicates if this request is a Full metadata snapshot (2), Incremental (1), or Unknown (0). Using during ZK migration, see KIP-866"}, { "name": "ControllerEpoch", "type": "int32", "versions": "0+", "about": "The controller epoch." }, { "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1", diff --git a/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java b/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java index c672540cf9eb8..2da4bbeba3a01 100644 --- a/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/ClusterConnectionStatesTest.java @@ -420,6 +420,25 @@ public void testTimedOutConnections() { assertEquals(0, connectionStates.nodesWithConnectionSetupTimeout(time.milliseconds()).size()); } + @Test + public void testSkipLastAttemptedIp() throws UnknownHostException { + setupMultipleIPs(); + + assertTrue(ClientUtils.resolve(hostTwoIps, multipleIPHostResolver).size() > 1); + + // Connect to the first IP + connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps); + InetAddress addr1 = connectionStates.currentAddress(nodeId1); + + // Disconnect, which will trigger re-resolution with the first IP still first + connectionStates.disconnected(nodeId1, time.milliseconds()); + + // Connect again, the first IP should get skipped + connectionStates.connecting(nodeId1, time.milliseconds(), hostTwoIps); + InetAddress addr2 = connectionStates.currentAddress(nodeId1); + assertNotSame(addr1, addr2); + } + private void setupMultipleIPs() { this.connectionStates = new ClusterConnectionStates(reconnectBackoffMs, reconnectBackoffMax, connectionSetupTimeoutMs, connectionSetupTimeoutMaxMs, new LogContext(), this.multipleIPHostResolver); diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataCacheTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataCacheTest.java index 9c50ef3116136..f99eb04dfcb97 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataCacheTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataCacheTest.java @@ -149,4 +149,40 @@ public void testMergeWithThatPreExistingPartitionIsRetainedPostMerge() { assertEquals(topic2, cluster.topicName(topic2Id)); } + @Test + public void testTopicNamesCacheBuiltFromTopicIds() { + Map topicIds = new HashMap<>(); + topicIds.put("topic1", Uuid.randomUuid()); + topicIds.put("topic2", Uuid.randomUuid()); + + MetadataCache cache = new MetadataCache("clusterId", + Collections.singletonMap(6, new Node(6, "localhost", 2077)), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet(), + null, + topicIds); + + Map expectedNamesCache = + topicIds.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, + Map.Entry::getKey)); + assertEquals(expectedNamesCache, cache.topicNames()); + } + + @Test + public void testEmptyTopicNamesCacheBuiltFromTopicIds() { + Map topicIds = new HashMap<>(); + + MetadataCache cache = new MetadataCache("clusterId", + Collections.singletonMap(6, new Node(6, "localhost", 2077)), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet(), + null, + topicIds); + assertEquals(Collections.emptyMap(), cache.topicNames()); + } + } diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index abf44f4c45bbe..747049fd192f2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -26,20 +26,29 @@ import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsRequestData; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsResponseData; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.message.PushTelemetryRequestData; +import org.apache.kafka.common.message.PushTelemetryResponseData; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.PushTelemetryRequest; +import org.apache.kafka.common.requests.PushTelemetryResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.test.DelayedReceive; @@ -71,6 +80,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class NetworkClientTest { @@ -992,32 +1007,48 @@ public void testReconnectAfterAddressChange() { return (mockHostResolver.useNewAddresses() && newAddresses.contains(inetAddress)) || (!mockHostResolver.useNewAddresses() && initialAddresses.contains(inetAddress)); }); + + ClientTelemetrySender mockClientTelemetrySender = mock(ClientTelemetrySender.class); + when(mockClientTelemetrySender.timeToNextUpdate(anyLong())).thenReturn(0L); + NetworkClient client = new NetworkClient(metadataUpdater, null, selector, "mock", Integer.MAX_VALUE, reconnectBackoffMsTest, reconnectBackoffMaxMsTest, 64 * 1024, 64 * 1024, defaultRequestTimeoutMs, connectionSetupTimeoutMsTest, connectionSetupTimeoutMaxMsTest, - time, false, new ApiVersions(), null, new LogContext(), mockHostResolver); + time, false, new ApiVersions(), null, new LogContext(), mockHostResolver, mockClientTelemetrySender); // Connect to one the initial addresses, then change the addresses and disconnect client.ready(node, time.milliseconds()); time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertTrue(client.isReady(node, time.milliseconds())); + // First poll should try to update the node but couldn't because node remains in connecting state + // i.e. connection handling is completed after telemetry update. + assertNull(client.telemetryConnectedNode()); + + client.poll(0, time.milliseconds()); + assertEquals(node, client.telemetryConnectedNode()); mockHostResolver.changeAddresses(); selector.serverDisconnect(node.idString()); client.poll(0, time.milliseconds()); assertFalse(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); time.sleep(reconnectBackoffMaxMsTest); client.ready(node, time.milliseconds()); time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertTrue(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); + + client.poll(0, time.milliseconds()); + assertEquals(node, client.telemetryConnectedNode()); // We should have tried to connect to one initial address and one new address, and resolved DNS twice assertEquals(1, initialAddressConns.get()); assertEquals(1, newAddressConns.get()); assertEquals(2, mockHostResolver.resolutionCount()); + verify(mockClientTelemetrySender, times(5)).timeToNextUpdate(anyLong()); } @Test @@ -1036,16 +1067,21 @@ public void testFailedConnectionToFirstAddress() { // Refuse first connection attempt return initialAddressConns.get() > 1; }); + + ClientTelemetrySender mockClientTelemetrySender = mock(ClientTelemetrySender.class); + when(mockClientTelemetrySender.timeToNextUpdate(anyLong())).thenReturn(0L); + NetworkClient client = new NetworkClient(metadataUpdater, null, selector, "mock", Integer.MAX_VALUE, reconnectBackoffMsTest, reconnectBackoffMaxMsTest, 64 * 1024, 64 * 1024, defaultRequestTimeoutMs, connectionSetupTimeoutMsTest, connectionSetupTimeoutMaxMsTest, - time, false, new ApiVersions(), null, new LogContext(), mockHostResolver); + time, false, new ApiVersions(), null, new LogContext(), mockHostResolver, mockClientTelemetrySender); // First connection attempt should fail client.ready(node, time.milliseconds()); time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertFalse(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); // Second connection attempt should succeed time.sleep(reconnectBackoffMaxMsTest); @@ -1053,12 +1089,18 @@ public void testFailedConnectionToFirstAddress() { time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertTrue(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); + + // Next client poll after handling connection setup should update telemetry node. + client.poll(0, time.milliseconds()); + assertEquals(node, client.telemetryConnectedNode()); // We should have tried to connect to two of the initial addresses, none of the new address, and should // only have resolved DNS once assertEquals(2, initialAddressConns.get()); assertEquals(0, newAddressConns.get()); assertEquals(1, mockHostResolver.resolutionCount()); + verify(mockClientTelemetrySender, times(3)).timeToNextUpdate(anyLong()); } @Test @@ -1077,21 +1119,30 @@ public void testFailedConnectionToFirstAddressAfterReconnect() { // Refuse first connection attempt to the new addresses return initialAddresses.contains(inetAddress) || newAddressConns.get() > 1; }); + + ClientTelemetrySender mockClientTelemetrySender = mock(ClientTelemetrySender.class); + when(mockClientTelemetrySender.timeToNextUpdate(anyLong())).thenReturn(0L); + NetworkClient client = new NetworkClient(metadataUpdater, null, selector, "mock", Integer.MAX_VALUE, reconnectBackoffMsTest, reconnectBackoffMaxMsTest, 64 * 1024, 64 * 1024, defaultRequestTimeoutMs, connectionSetupTimeoutMsTest, connectionSetupTimeoutMaxMsTest, - time, false, new ApiVersions(), null, new LogContext(), mockHostResolver); + time, false, new ApiVersions(), null, new LogContext(), mockHostResolver, mockClientTelemetrySender); // Connect to one the initial addresses, then change the addresses and disconnect client.ready(node, time.milliseconds()); time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertTrue(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); + // Next client poll after handling connection setup should update telemetry node. + client.poll(0, time.milliseconds()); + assertEquals(node, client.telemetryConnectedNode()); mockHostResolver.changeAddresses(); selector.serverDisconnect(node.idString()); client.poll(0, time.milliseconds()); assertFalse(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); // First connection attempt to new addresses should fail time.sleep(reconnectBackoffMaxMsTest); @@ -1099,6 +1150,7 @@ public void testFailedConnectionToFirstAddressAfterReconnect() { time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertFalse(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); // Second connection attempt to new addresses should succeed time.sleep(reconnectBackoffMaxMsTest); @@ -1106,12 +1158,18 @@ public void testFailedConnectionToFirstAddressAfterReconnect() { time.sleep(connectionSetupTimeoutMaxMsTest); client.poll(0, time.milliseconds()); assertTrue(client.isReady(node, time.milliseconds())); + assertNull(client.telemetryConnectedNode()); + + // Next client poll after handling connection setup should update telemetry node. + client.poll(0, time.milliseconds()); + assertEquals(node, client.telemetryConnectedNode()); // We should have tried to connect to one of the initial addresses and two of the new addresses (the first one // failed), and resolved DNS twice, once for each set of addresses assertEquals(1, initialAddressConns.get()); assertEquals(2, newAddressConns.get()); assertEquals(2, mockHostResolver.resolutionCount()); + verify(mockClientTelemetrySender, times(6)).timeToNextUpdate(anyLong()); } @Test @@ -1168,6 +1226,61 @@ public void testConnectionDoesNotRemainStuckInCheckingApiVersionsStateIfChannelN assertTrue(client.connectionFailed(node)); } + @Test + public void testTelemetryRequest() { + ClientTelemetrySender mockClientTelemetrySender = mock(ClientTelemetrySender.class); + when(mockClientTelemetrySender.timeToNextUpdate(anyLong())).thenReturn(0L); + + NetworkClient client = new NetworkClient(metadataUpdater, null, selector, "mock", Integer.MAX_VALUE, + reconnectBackoffMsTest, reconnectBackoffMaxMsTest, 64 * 1024, 64 * 1024, + defaultRequestTimeoutMs, connectionSetupTimeoutMsTest, connectionSetupTimeoutMaxMsTest, + time, true, new ApiVersions(), null, new LogContext(), new DefaultHostResolver(), mockClientTelemetrySender); + + // Send the ApiVersionsRequest + client.ready(node, time.milliseconds()); + client.poll(0, time.milliseconds()); + assertNull(client.telemetryConnectedNode()); + assertTrue(client.hasInFlightRequests(node.idString())); + delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(), TestUtils.defaultApiVersionsResponse( + ApiMessageType.ListenerType.BROKER)); + // handle ApiVersionsResponse + client.poll(0, time.milliseconds()); + // the ApiVersionsRequest is gone + assertFalse(client.hasInFlightRequests(node.idString())); + selector.clear(); + + GetTelemetrySubscriptionsRequest.Builder getRequest = new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData(), true); + when(mockClientTelemetrySender.createRequest()).thenReturn(Optional.of(getRequest)); + + GetTelemetrySubscriptionsResponse getResponse = new GetTelemetrySubscriptionsResponse(new GetTelemetrySubscriptionsResponseData()); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(getResponse, ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS.latestVersion(), 1); + selector.completeReceive(new NetworkReceive(node.idString(), buffer)); + + // Initiate poll to send GetTelemetrySubscriptions request + client.poll(0, time.milliseconds()); + assertTrue(client.isReady(node, time.milliseconds())); + assertEquals(node, client.telemetryConnectedNode()); + verify(mockClientTelemetrySender, times(1)).handleResponse(any(GetTelemetrySubscriptionsResponse.class)); + selector.clear(); + + PushTelemetryRequest.Builder pushRequest = new PushTelemetryRequest.Builder( + new PushTelemetryRequestData(), true); + when(mockClientTelemetrySender.createRequest()).thenReturn(Optional.of(pushRequest)); + + PushTelemetryResponse pushResponse = new PushTelemetryResponse(new PushTelemetryResponseData()); + ByteBuffer pushBuffer = RequestTestUtils.serializeResponseWithHeader(pushResponse, ApiKeys.PUSH_TELEMETRY.latestVersion(), 2); + selector.completeReceive(new NetworkReceive(node.idString(), pushBuffer)); + + // Initiate poll to send PushTelemetry request + client.poll(0, time.milliseconds()); + assertTrue(client.isReady(node, time.milliseconds())); + assertEquals(node, client.telemetryConnectedNode()); + verify(mockClientTelemetrySender, times(1)).handleResponse(any(PushTelemetryResponse.class)); + verify(mockClientTelemetrySender, times(4)).timeToNextUpdate(anyLong()); + verify(mockClientTelemetrySender, times(2)).createRequest(); + } + private RequestHeader parseHeader(ByteBuffer buffer) { buffer.getInt(); // skip size return RequestHeader.parse(buffer.slice()); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 19653017a985b..19bff2ea5e87a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -33,8 +33,8 @@ import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; -import org.apache.kafka.common.TopicCollection; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicCollection; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.Uuid; @@ -110,13 +110,15 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.FindCoordinatorResponseData; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; -import org.apache.kafka.common.message.LeaveGroupRequestData; import org.apache.kafka.common.message.InitProducerIdResponseData; +import org.apache.kafka.common.message.LeaveGroupRequestData; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; +import org.apache.kafka.common.message.ListClientMetricsResourcesResponseData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.message.ListOffsetsResponseData; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; @@ -176,12 +178,16 @@ import org.apache.kafka.common.requests.ElectLeadersResponse; import org.apache.kafka.common.requests.FindCoordinatorRequest; import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; import org.apache.kafka.common.requests.IncrementalAlterConfigsResponse; import org.apache.kafka.common.requests.InitProducerIdRequest; import org.apache.kafka.common.requests.InitProducerIdResponse; import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.LeaveGroupRequest; import org.apache.kafka.common.requests.LeaveGroupResponse; +import org.apache.kafka.common.requests.ListClientMetricsResourcesRequest; +import org.apache.kafka.common.requests.ListClientMetricsResourcesResponse; import org.apache.kafka.common.requests.ListGroupsRequest; import org.apache.kafka.common.requests.ListGroupsResponse; import org.apache.kafka.common.requests.ListOffsetsRequest; @@ -222,6 +228,7 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -7060,6 +7067,48 @@ public void testFenceProducers() throws Exception { } } + @Test + public void testClientInstanceId() { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + Uuid expected = Uuid.randomUuid(); + + GetTelemetrySubscriptionsResponseData responseData = + new GetTelemetrySubscriptionsResponseData().setClientInstanceId(expected).setErrorCode(Errors.NONE.code()); + + env.kafkaClient().prepareResponse( + request -> request instanceof GetTelemetrySubscriptionsRequest, + new GetTelemetrySubscriptionsResponse(responseData)); + + Uuid result = env.adminClient().clientInstanceId(Duration.ofMillis(10)); + assertEquals(expected, result); + } + } + + @Test + public void testClientInstanceIdInvalidTimeout() { + Properties props = new Properties(); + props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + + KafkaAdminClient admin = (KafkaAdminClient) AdminClient.create(props); + Exception exception = assertThrows(IllegalArgumentException.class, () -> admin.clientInstanceId(Duration.ofMillis(-1))); + assertEquals("The timeout cannot be negative.", exception.getMessage()); + + admin.close(); + } + + @Test + public void testClientInstanceIdNoTelemetryReporterRegistered() { + Properties props = new Properties(); + props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(AdminClientConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); + + KafkaAdminClient admin = (KafkaAdminClient) AdminClient.create(props); + Exception exception = assertThrows(IllegalStateException.class, () -> admin.clientInstanceId(Duration.ofMillis(0))); + assertEquals("Telemetry is not enabled. Set config `enable.metrics.push` to `true`.", exception.getMessage()); + + admin.close(); + } + private UnregisterBrokerResponse prepareUnregisterBrokerResponse(Errors error, int throttleTimeMs) { return new UnregisterBrokerResponse(new UnregisterBrokerResponseData() .setErrorCode(error.code()) @@ -7090,6 +7139,68 @@ private static MemberDescription convertToMemberDescriptions(DescribedGroupMembe assignment); } + @Test + public void testListClientMetricsResources() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + List expected = Arrays.asList( + new ClientMetricsResourceListing("one"), + new ClientMetricsResourceListing("two") + ); + + ListClientMetricsResourcesResponseData responseData = + new ListClientMetricsResourcesResponseData().setErrorCode(Errors.NONE.code()); + + responseData.clientMetricsResources() + .add(new ListClientMetricsResourcesResponseData.ClientMetricsResource().setName("one")); + responseData.clientMetricsResources() + .add((new ListClientMetricsResourcesResponseData.ClientMetricsResource()).setName("two")); + + env.kafkaClient().prepareResponse( + request -> request instanceof ListClientMetricsResourcesRequest, + new ListClientMetricsResourcesResponse(responseData)); + + ListClientMetricsResourcesResult result = env.adminClient().listClientMetricsResources(); + assertEquals(new HashSet<>(expected), new HashSet<>(result.all().get())); + } + } + + @Test + public void testListClientMetricsResourcesEmpty() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + List expected = Collections.emptyList(); + + ListClientMetricsResourcesResponseData responseData = + new ListClientMetricsResourcesResponseData().setErrorCode(Errors.NONE.code()); + + env.kafkaClient().prepareResponse( + request -> request instanceof ListClientMetricsResourcesRequest, + new ListClientMetricsResourcesResponse(responseData)); + + ListClientMetricsResourcesResult result = env.adminClient().listClientMetricsResources(); + assertEquals(new HashSet<>(expected), new HashSet<>(result.all().get())); + } + } + + @Test + public void testListClientMetricsResourcesNotSupported() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + env.kafkaClient().prepareResponse( + request -> request instanceof ListClientMetricsResourcesRequest, + prepareListClientMetricsResourcesResponse(Errors.UNSUPPORTED_VERSION)); + + ListClientMetricsResourcesResult result = env.adminClient().listClientMetricsResources(); + + // Validate response + assertNotNull(result.all()); + TestUtils.assertFutureThrows(result.all(), Errors.UNSUPPORTED_VERSION.exception().getClass()); + } + } + + private static ListClientMetricsResourcesResponse prepareListClientMetricsResourcesResponse(Errors error) { + return new ListClientMetricsResourcesResponse(new ListClientMetricsResourcesResponseData() + .setErrorCode(error.code())); + } + @SafeVarargs private static void assertCollectionIs(Collection collection, T... elements) { for (T element : elements) { diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index 515b02e696543..f8fc440b28904 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -90,11 +90,15 @@ public class MockAdminClient extends AdminClient { private final String clusterId; private final List> brokerLogDirs; private final List> brokerConfigs; + private final Map> clientMetricsConfigs; private Node controller; private int timeoutNextRequests = 0; private final int defaultPartitions; private final int defaultReplicationFactor; + private boolean telemetryDisabled = false; + private Uuid clientInstanceId; + private int injectTimeoutExceptionCounter; private KafkaException listConsumerGroupOffsetsException; @@ -238,6 +242,7 @@ private MockAdminClient( this.defaultReplicationFactor = defaultReplicationFactor; this.brokerLogDirs = brokerLogDirs; this.brokerConfigs = new ArrayList<>(); + this.clientMetricsConfigs = new HashMap<>(); for (int i = 0; i < brokers.size(); i++) { final Map config = new HashMap<>(); config.put("default.replication.factor", String.valueOf(defaultReplicationFactor)); @@ -823,6 +828,13 @@ synchronized private Config getResourceDescription(ConfigResource resource) { } throw new UnknownTopicOrPartitionException("Resource " + resource + " not found."); } + case CLIENT_METRICS: { + String resourceName = resource.name(); + if (resourceName.isEmpty()) { + throw new InvalidRequestException("Empty resource name"); + } + return toConfigObject(clientMetricsConfigs.get(resourceName)); + } default: throw new UnsupportedOperationException("Not implemented yet"); } @@ -916,6 +928,34 @@ synchronized private Throwable handleIncrementalResourceAlteration( topicMetadata.configs = newMap; return null; } + case CLIENT_METRICS: { + String resourceName = resource.name(); + + if (resourceName.isEmpty()) { + return new InvalidRequestException("Empty resource name"); + } + + if (!clientMetricsConfigs.containsKey(resourceName)) { + clientMetricsConfigs.put(resourceName, new HashMap<>()); + } + + HashMap newMap = new HashMap<>(clientMetricsConfigs.get(resourceName)); + for (AlterConfigOp op : ops) { + switch (op.opType()) { + case SET: + newMap.put(op.configEntry().name(), op.configEntry().value()); + break; + case DELETE: + newMap.remove(op.configEntry().name()); + break; + default: + return new InvalidRequestException( + "Unsupported op type " + op.opType()); + } + } + clientMetricsConfigs.put(resourceName, newMap); + return null; + } default: return new UnsupportedOperationException(); } @@ -1264,6 +1304,11 @@ public FenceProducersResult fenceProducers(Collection transactionalIds, throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public ListClientMetricsResourcesResult listClientMetricsResources(ListClientMetricsResourcesOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override synchronized public void close(Duration timeout) {} @@ -1312,9 +1357,38 @@ synchronized public void setMockMetrics(MetricName name, Metric metric) { mockMetrics.put(name, metric); } + public void disableTelemetry() { + telemetryDisabled = true; + } + + /** + * @param injectTimeoutExceptionCounter use -1 for infinite + */ + public void injectTimeoutException(final int injectTimeoutExceptionCounter) { + this.injectTimeoutExceptionCounter = injectTimeoutExceptionCounter; + } + + public void setClientInstanceId(final Uuid instanceId) { + clientInstanceId = instanceId; + } + @Override public Uuid clientInstanceId(Duration timeout) { - throw new UnsupportedOperationException("Not implemented yet"); + if (telemetryDisabled) { + throw new IllegalStateException(); + } + if (clientInstanceId == null) { + throw new UnsupportedOperationException("clientInstanceId not set"); + } + if (injectTimeoutExceptionCounter != 0) { + // -1 is used as "infinite" + if (injectTimeoutExceptionCounter > 0) { + --injectTimeoutExceptionCounter; + } + throw new TimeoutException(); + } + + return clientInstanceId; } @Override diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerConfigTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerConfigTest.java index 58181b68535d3..df3ba98dd0c56 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerConfigTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/ConsumerConfigTest.java @@ -39,7 +39,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; public class ConsumerConfigTest { @@ -66,18 +65,24 @@ public void testOverrideClientId() { @Test public void testOverrideEnableAutoCommit() { - ConsumerConfig config = new ConsumerConfig(properties); - boolean overrideEnableAutoCommit = config.maybeOverrideEnableAutoCommit(); - assertFalse(overrideEnableAutoCommit); - - properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); - config = new ConsumerConfig(properties); - try { - config.maybeOverrideEnableAutoCommit(); - fail("Should have thrown an exception"); - } catch (InvalidConfigurationException e) { - // expected - } + // Verify that our default properties (no 'enable.auto.commit' or 'group.id') are valid. + assertEquals(false, new ConsumerConfig(properties).getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); + + // Verify that explicitly disabling 'enable.auto.commit' still works. + properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.FALSE.toString()); + assertEquals(false, new ConsumerConfig(properties).getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); + + // Verify that enabling 'enable.auto.commit' but without 'group.id' fails. + properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, Boolean.TRUE.toString()); + assertThrows(InvalidConfigurationException.class, () -> new ConsumerConfig(properties)); + + // Verify that then adding 'group.id' to the mix allows it to pass OK. + properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group"); + assertEquals(true, new ConsumerConfig(properties).getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); + + // Now remove the 'enable.auto.commit' flag and verify that it is set to true (the default). + properties.remove(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + assertEquals(true, new ConsumerConfig(properties).getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index b4c3dba56fbb3..657d3145e89d2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -16,35 +16,23 @@ */ package org.apache.kafka.clients.consumer; -import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.GroupRebalanceConfig; import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.NodeApiVersions; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerInterceptors; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; -import org.apache.kafka.clients.consumer.internals.ConsumerMetrics; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; -import org.apache.kafka.clients.consumer.internals.Deserializers; -import org.apache.kafka.clients.consumer.internals.FetchConfig; -import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; -import org.apache.kafka.clients.consumer.internals.Fetcher; import org.apache.kafka.clients.consumer.internals.MockRebalanceListener; -import org.apache.kafka.clients.consumer.internals.OffsetFetcher; import org.apache.kafka.clients.consumer.internals.SubscriptionState; -import org.apache.kafka.clients.consumer.internals.TopicMetadataFetcher; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.Node; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.errors.AuthenticationException; @@ -68,6 +56,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.SyncGroupResponseData; +import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -98,6 +87,8 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; @@ -106,10 +97,11 @@ import org.apache.kafka.test.MockMetricsReporter; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.MockedStatic; +import org.mockito.internal.stubbing.answers.CallsRealMethods; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; import java.time.Duration; @@ -124,6 +116,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; @@ -142,12 +135,15 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.management.MBeanServer; +import javax.management.ObjectName; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.apache.kafka.clients.consumer.KafkaConsumer.DEFAULT_REASON; +import static org.apache.kafka.clients.consumer.internals.LegacyKafkaConsumer.DEFAULT_REASON; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; +import static org.apache.kafka.common.utils.Utils.propsToMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -157,8 +153,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Note to future authors in this class. If you close the consumer, close with DURATION.ZERO to reduce the duration of @@ -181,13 +182,11 @@ public class KafkaConsumerTest { private final int sessionTimeoutMs = 10000; private final int defaultApiTimeoutMs = 60000; - private final int requestTimeoutMs = defaultApiTimeoutMs / 2; private final int heartbeatIntervalMs = 1000; // Set auto commit interval lower than heartbeat so we don't need to deal with // a concurrent heartbeat request private final int autoCommitIntervalMs = 500; - private final int throttleMs = 10; private final String groupId = "mock-group"; private final String memberId = "memberId"; @@ -222,42 +221,68 @@ public void cleanup() { } } - @Test - public void testMetricsReporterAutoGeneratedClientId() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testMetricsReporterAutoGeneratedClientId(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); - consumer = new KafkaConsumer<>(props, new StringDeserializer(), new StringDeserializer()); + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); - MockMetricsReporter mockMetricsReporter = (MockMetricsReporter) consumer.metrics.reporters().get(0); + assertEquals(3, consumer.metricsRegistry().reporters().size()); - assertEquals(consumer.getClientId(), mockMetricsReporter.clientId); - assertEquals(2, consumer.metrics.reporters().size()); + MockMetricsReporter mockMetricsReporter = (MockMetricsReporter) consumer.metricsRegistry().reporters().stream() + .filter(reporter -> reporter instanceof MockMetricsReporter).findFirst().get(); + assertEquals(consumer.clientId(), mockMetricsReporter.clientId); } - @Test + @ParameterizedTest + @EnumSource(GroupProtocol.class) @SuppressWarnings("deprecation") - public void testDisableJmxReporter() { + public void testDisableJmxAndClientTelemetryReporter(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.AUTO_INCLUDE_JMX_REPORTER_CONFIG, "false"); - consumer = new KafkaConsumer<>(props, new StringDeserializer(), new StringDeserializer()); - assertTrue(consumer.metrics.reporters().isEmpty()); + props.setProperty(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + assertTrue(consumer.metricsRegistry().reporters().isEmpty()); } - @Test - public void testExplicitlyEnableJmxReporter() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testExplicitlyOnlyEnableJmxReporter(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, "org.apache.kafka.common.metrics.JmxReporter"); - consumer = new KafkaConsumer<>(props, new StringDeserializer(), new StringDeserializer()); - assertEquals(1, consumer.metrics.reporters().size()); + props.setProperty(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + assertEquals(1, consumer.metricsRegistry().reporters().size()); + assertTrue(consumer.metricsRegistry().reporters().get(0) instanceof JmxReporter); + } + + @ParameterizedTest + @EnumSource(GroupProtocol.class) + @SuppressWarnings("deprecation") + public void testExplicitlyOnlyEnableClientTelemetryReporter(GroupProtocol groupProtocol) { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ConsumerConfig.AUTO_INCLUDE_JMX_REPORTER_CONFIG, "false"); + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + assertEquals(1, consumer.metricsRegistry().reporters().size()); + assertTrue(consumer.metricsRegistry().reporters().get(0) instanceof ClientTelemetryReporter); } - @Test + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testPollReturnsRecords() { - consumer = setUpConsumerWithRecordsToPoll(tp0, 5); + public void testPollReturnsRecords(GroupProtocol groupProtocol) { + consumer = setUpConsumerWithRecordsToPoll(groupProtocol, tp0, 5); ConsumerRecords records = (ConsumerRecords) consumer.poll(Duration.ZERO); @@ -266,14 +291,17 @@ public void testPollReturnsRecords() { assertEquals(records.records(tp0).size(), 5); } - @Test + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testSecondPollWithDeserializationErrorThrowsRecordDeserializationException() { + public void testSecondPollWithDeserializationErrorThrowsRecordDeserializationException(GroupProtocol groupProtocol) { int invalidRecordNumber = 4; int invalidRecordOffset = 3; StringDeserializer deserializer = mockErrorDeserializer(invalidRecordNumber); - consumer = setUpConsumerWithRecordsToPoll(tp0, 5, deserializer); + consumer = setUpConsumerWithRecordsToPoll(groupProtocol, tp0, 5, deserializer); ConsumerRecords records = (ConsumerRecords) consumer.poll(Duration.ZERO); assertEquals(invalidRecordNumber - 1, records.count()); @@ -317,18 +345,23 @@ public String deserialize(String topic, Headers headers, ByteBuffer data) { }; } - private KafkaConsumer setUpConsumerWithRecordsToPoll(TopicPartition tp, int recordCount) { - return setUpConsumerWithRecordsToPoll(tp, recordCount, new StringDeserializer()); + private KafkaConsumer setUpConsumerWithRecordsToPoll(GroupProtocol groupProtocol, + TopicPartition tp, + int recordCount) { + return setUpConsumerWithRecordsToPoll(groupProtocol, tp, recordCount, new StringDeserializer()); } - private KafkaConsumer setUpConsumerWithRecordsToPoll(TopicPartition tp, int recordCount, Deserializer deserializer) { + private KafkaConsumer setUpConsumerWithRecordsToPoll(GroupProtocol groupProtocol, + TopicPartition tp, + int recordCount, + Deserializer deserializer) { Cluster cluster = TestUtils.singletonCluster(tp.topic(), 1); Node node = cluster.nodes().get(0); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, Optional.of(deserializer), false); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp), null); @@ -337,9 +370,11 @@ public String deserialize(String topic, Headers headers, ByteBuffer data) { return consumer; } - @Test - public void testConstructorClose() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testConstructorClose(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "invalid-23-8409-adsfsdj"); props.setProperty(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); @@ -347,7 +382,7 @@ public void testConstructorClose() { final int oldInitCount = MockMetricsReporter.INIT_COUNT.get(); final int oldCloseCount = MockMetricsReporter.CLOSE_COUNT.get(); try { - new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + newConsumer(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); fail("should have caught an exception and returned"); } catch (KafkaException e) { assertEquals(oldInitCount + 1, MockMetricsReporter.INIT_COUNT.get()); @@ -356,44 +391,53 @@ public void testConstructorClose() { } } - @Test - public void testOsDefaultSocketBufferSizes() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testOsDefaultSocketBufferSizes(GroupProtocol groupProtocol) { Map config = new HashMap<>(); + config.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ConsumerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); config.put(ConsumerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); - consumer = new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + consumer = newConsumer(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); } - @Test - public void testInvalidSocketSendBufferSize() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testInvalidSocketSendBufferSize(GroupProtocol groupProtocol) { Map config = new HashMap<>(); + config.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ConsumerConfig.SEND_BUFFER_CONFIG, -2); assertThrows(KafkaException.class, - () -> new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())); + () -> newConsumer(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())); } - @Test - public void testInvalidSocketReceiveBufferSize() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testInvalidSocketReceiveBufferSize(GroupProtocol groupProtocol) { Map config = new HashMap<>(); + config.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ConsumerConfig.RECEIVE_BUFFER_CONFIG, -2); assertThrows(KafkaException.class, - () -> new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())); + () -> newConsumer(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())); } - @Test - public void shouldIgnoreGroupInstanceIdForEmptyGroupId() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void shouldIgnoreGroupInstanceIdForEmptyGroupId(GroupProtocol groupProtocol) { Map config = new HashMap<>(); + config.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "instance_id"); - consumer = new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + consumer = newConsumer(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); } - @Test - public void testSubscription() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscription(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); consumer.subscribe(singletonList(topic)); assertEquals(singleton(topic), consumer.subscription()); @@ -412,93 +456,107 @@ public void testSubscription() { assertTrue(consumer.assignment().isEmpty()); } - @Test - public void testSubscriptionOnNullTopicCollection() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionOnNullTopicCollection(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); assertThrows(IllegalArgumentException.class, () -> consumer.subscribe((List) null)); } - @Test - public void testSubscriptionOnNullTopic() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionOnNullTopic(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); assertThrows(IllegalArgumentException.class, () -> consumer.subscribe(singletonList(null))); } - @Test - public void testSubscriptionOnEmptyTopic() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionOnEmptyTopic(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); String emptyTopic = " "; assertThrows(IllegalArgumentException.class, () -> consumer.subscribe(singletonList(emptyTopic))); } - @Test - public void testSubscriptionOnNullPattern() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionOnNullPattern(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); assertThrows(IllegalArgumentException.class, () -> consumer.subscribe((Pattern) null)); } - @Test - public void testSubscriptionOnEmptyPattern() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionOnEmptyPattern(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); assertThrows(IllegalArgumentException.class, () -> consumer.subscribe(Pattern.compile(""))); } - @Test - public void testSubscriptionWithEmptyPartitionAssignment() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSubscriptionWithEmptyPartitionAssignment(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, ""); props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId); - consumer = newConsumer(props); + consumer = newConsumer(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); assertThrows(IllegalStateException.class, () -> consumer.subscribe(singletonList(topic))); } - @Test - public void testSeekNegative() { - consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testSeekNegative(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); consumer.assign(singleton(new TopicPartition("nonExistTopic", 0))); assertThrows(IllegalArgumentException.class, () -> consumer.seek(new TopicPartition("nonExistTopic", 0), -1)); } - @Test - public void testAssignOnNullTopicPartition() { - consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testAssignOnNullTopicPartition(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); assertThrows(IllegalArgumentException.class, () -> consumer.assign(null)); } - @Test - public void testAssignOnEmptyTopicPartition() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testAssignOnEmptyTopicPartition(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); consumer.assign(Collections.emptyList()); assertTrue(consumer.subscription().isEmpty()); assertTrue(consumer.assignment().isEmpty()); } - @Test - public void testAssignOnNullTopicInPartition() { - consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testAssignOnNullTopicInPartition(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(null, 0)))); } - @Test - public void testAssignOnEmptyTopicInPartition() { - consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testAssignOnEmptyTopicInPartition(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(" ", 0)))); } - @Test - public void testInterceptorConstructorClose() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testInterceptorConstructorClose(GroupProtocol groupProtocol) { try { Properties props = new Properties(); // test with client ID assigned by KafkaConsumer + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, MockConsumerInterceptor.class.getName()); - consumer = new KafkaConsumer<>( + consumer = newConsumer( props, new StringDeserializer(), new StringDeserializer()); assertEquals(1, MockConsumerInterceptor.INIT_COUNT.get()); assertEquals(0, MockConsumerInterceptor.CLOSE_COUNT.get()); @@ -515,12 +573,14 @@ public void testInterceptorConstructorClose() { } } - @Test - public void testInterceptorConstructorConfigurationWithExceptionShouldCloseRemainingInstances() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testInterceptorConstructorConfigurationWithExceptionShouldCloseRemainingInstances(GroupProtocol groupProtocol) { final int targetInterceptor = 3; try { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, MockConsumerInterceptor.class.getName() + ", " + MockConsumerInterceptor.class.getName() + ", " @@ -528,10 +588,8 @@ public void testInterceptorConstructorConfigurationWithExceptionShouldCloseRemai MockConsumerInterceptor.setThrowOnConfigExceptionThreshold(targetInterceptor); - assertThrows(KafkaException.class, () -> { - new KafkaConsumer<>( - props, new StringDeserializer(), new StringDeserializer()); - }); + assertThrows(KafkaException.class, () -> newConsumer( + props, new StringDeserializer(), new StringDeserializer())); assertEquals(3, MockConsumerInterceptor.CONFIG_COUNT.get()); assertEquals(3, MockConsumerInterceptor.CLOSE_COUNT.get()); @@ -541,9 +599,10 @@ public void testInterceptorConstructorConfigurationWithExceptionShouldCloseRemai } } - @Test - public void testPause() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testPause(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); consumer.assign(singletonList(tp0)); assertEquals(singleton(tp0), consumer.assignment()); @@ -559,27 +618,32 @@ public void testPause() { assertTrue(consumer.paused().isEmpty()); } - @Test - public void testConsumerJmxPrefix() throws Exception { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testConsumerJmxPrefix(GroupProtocol groupProtocol) throws Exception { Map config = new HashMap<>(); + config.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ConsumerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); config.put(ConsumerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); config.put("client.id", "client-1"); - consumer = new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + consumer = newConsumer(config, new ByteArrayDeserializer(), new ByteArrayDeserializer()); MBeanServer server = ManagementFactory.getPlatformMBeanServer(); - MetricName testMetricName = consumer.metrics.metricName("test-metric", + MetricName testMetricName = consumer.metricsRegistry().metricName("test-metric", "grp1", "test metric"); - consumer.metrics.addMetric(testMetricName, new Avg()); + consumer.metricsRegistry().addMetric(testMetricName, new Avg()); assertNotNull(server.getObjectInstance(new ObjectName("kafka.consumer:type=grp1,client-id=client-1"))); } - private KafkaConsumer newConsumer(String groupId) { - return newConsumer(groupId, Optional.empty()); + private KafkaConsumer newConsumer(GroupProtocol groupProtocol, String groupId) { + return newConsumer(groupProtocol, groupId, Optional.empty()); } - private KafkaConsumer newConsumer(String groupId, Optional enableAutoCommit) { + private KafkaConsumer newConsumer(GroupProtocol groupProtocol, + String groupId, + Optional enableAutoCommit) { Properties props = new Properties(); + props.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "my.consumer"); props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); @@ -587,22 +651,38 @@ private KafkaConsumer newConsumer(String groupId, Optional props.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommit.toString())); - return newConsumer(props); + return newConsumer(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + } + + private KafkaConsumer newConsumer(Properties props) { + return newConsumer(props, null, null); + } + + private KafkaConsumer newConsumer(Map configs, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + return new KafkaConsumer<>(new ConsumerConfig(ConsumerConfig.appendDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, valueDeserializer); } - private KafkaConsumer newConsumer(Properties props) { - return new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + private KafkaConsumer newConsumer(Properties props, + Deserializer keyDeserializer, + Deserializer valueDeserializer) { + return newConsumer(propsToMap(props), keyDeserializer, valueDeserializer); } - @Test - public void verifyHeartbeatSent() throws Exception { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void verifyHeartbeatSent(GroupProtocol groupProtocol) throws Exception { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -624,15 +704,18 @@ public void verifyHeartbeatSent() throws Exception { assertTrue(heartbeatReceived.get()); } - @Test - public void verifyHeartbeatSentWhenFetchedDataReady() throws Exception { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void verifyHeartbeatSentWhenFetchedDataReady(GroupProtocol groupProtocol) throws Exception { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -654,15 +737,16 @@ public void verifyHeartbeatSentWhenFetchedDataReady() throws Exception { assertTrue(heartbeatReceived.get()); } - @Test - public void verifyPollTimesOutDuringMetadataUpdate() { + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void verifyPollTimesOutDuringMetadataUpdate(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); // Since we would enable the heartbeat thread after received join-response which could // send the sync-group on behalf of the consumer if it is enqueued, we may still complete @@ -677,16 +761,19 @@ public void verifyPollTimesOutDuringMetadataUpdate() { assertEquals(0, requests.stream().filter(request -> request.apiKey().equals(ApiKeys.FETCH)).count()); } + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("deprecation") - @Test - public void verifyDeprecatedPollDoesNotTimeOutDuringMetadataUpdate() { + public void verifyDeprecatedPollDoesNotTimeOutDuringMetadataUpdate(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -699,15 +786,16 @@ public void verifyDeprecatedPollDoesNotTimeOutDuringMetadataUpdate() { assertEquals(FetchRequest.Builder.class, aClass); } - @Test + @ParameterizedTest + @EnumSource(GroupProtocol.class) @SuppressWarnings("unchecked") - public void verifyNoCoordinatorLookupForManualAssignmentWithSeek() { + public void verifyNoCoordinatorLookupForManualAssignmentWithSeek(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, null, groupInstanceId, false); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, null, groupInstanceId, false); consumer.assign(singleton(tp0)); consumer.seekToBeginning(singleton(tp0)); @@ -721,8 +809,11 @@ public void verifyNoCoordinatorLookupForManualAssignmentWithSeek() { assertEquals(55L, consumer.position(tp0)); } - @Test - public void verifyNoCoordinatorLookupForManualAssignmentWithOffsetCommit() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void verifyNoCoordinatorLookupForManualAssignmentWithOffsetCommit(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -730,7 +821,7 @@ public void verifyNoCoordinatorLookupForManualAssignmentWithOffsetCommit() { Node node = metadata.fetch().nodes().get(0); // create a consumer with groupID with manual assignment - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(singleton(tp0)); // 1st coordinator error should cause coordinator unknown @@ -757,8 +848,11 @@ public void verifyNoCoordinatorLookupForManualAssignmentWithOffsetCommit() { assertEquals(55, consumer.committed(Collections.singleton(tp0), Duration.ZERO).get(tp0).offset()); } - @Test - public void testFetchProgressWithMissingPartitionPosition() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testFetchProgressWithMissingPartitionPosition(GroupProtocol groupProtocol) { // Verifies that we can make progress on one partition while we are awaiting // a reset on another partition. @@ -767,7 +861,7 @@ public void testFetchProgressWithMissingPartitionPosition() { initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumerNoAutoCommit(time, client, subscription, metadata); + consumer = newConsumerNoAutoCommit(groupProtocol, time, client, subscription, metadata); consumer.assign(Arrays.asList(tp0, tp1)); consumer.seekToEnd(singleton(tp0)); consumer.seekToBeginning(singleton(tp1)); @@ -814,8 +908,11 @@ private void initMetadata(MockClient mockClient, Map partitionC mockClient.updateMetadata(initialMetadata); } - @Test - public void testMissingOffsetNoResetPolicy() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testMissingOffsetNoResetPolicy(GroupProtocol groupProtocol) { SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.NONE); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -823,7 +920,7 @@ public void testMissingOffsetNoResetPolicy() { initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, false); consumer.assign(singletonList(tp0)); @@ -835,8 +932,11 @@ public void testMissingOffsetNoResetPolicy() { assertThrows(NoOffsetForPartitionException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testResetToCommittedOffset() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testResetToCommittedOffset(GroupProtocol groupProtocol) { SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.NONE); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -844,7 +944,7 @@ public void testResetToCommittedOffset() { initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, false); consumer.assign(singletonList(tp0)); @@ -857,8 +957,11 @@ public void testResetToCommittedOffset() { assertEquals(539L, consumer.position(tp0)); } - @Test - public void testResetUsingAutoResetPolicy() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testResetUsingAutoResetPolicy(GroupProtocol groupProtocol) { SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.LATEST); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -866,7 +969,7 @@ public void testResetUsingAutoResetPolicy() { initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, false); consumer.assign(singletonList(tp0)); @@ -881,15 +984,16 @@ public void testResetUsingAutoResetPolicy() { assertEquals(50L, consumer.position(tp0)); } - @Test - public void testOffsetIsValidAfterSeek() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testOffsetIsValidAfterSeek(GroupProtocol groupProtocol) { SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.LATEST); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupId, Optional.empty(), false); consumer.assign(singletonList(tp0)); consumer.seek(tp0, 20L); @@ -897,8 +1001,11 @@ public void testOffsetIsValidAfterSeek() { assertEquals(subscription.validPosition(tp0).offset, 20L); } - @Test - public void testCommitsFetchedDuringAssign() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCommitsFetchedDuringAssign(GroupProtocol groupProtocol) { long offset1 = 10000; long offset2 = 20000; @@ -908,7 +1015,7 @@ public void testCommitsFetchedDuringAssign() { initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(singletonList(tp0)); // lookup coordinator @@ -933,22 +1040,31 @@ public void testCommitsFetchedDuringAssign() { assertEquals(offset2, consumer.committed(Collections.singleton(tp1)).get(tp1).offset()); } - @Test - public void testFetchStableOffsetThrowInCommitted() { - assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer().committed(Collections.singleton(tp0))); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testFetchStableOffsetThrowInCommitted(GroupProtocol groupProtocol) { + assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer(groupProtocol).committed(Collections.singleton(tp0))); } - @Test - public void testFetchStableOffsetThrowInPoll() { - assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer().poll(Duration.ZERO)); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testFetchStableOffsetThrowInPoll(GroupProtocol groupProtocol) { + assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer(groupProtocol).poll(Duration.ZERO)); } - @Test - public void testFetchStableOffsetThrowInPosition() { - assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer().position(tp0)); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testFetchStableOffsetThrowInPosition(GroupProtocol groupProtocol) { + assertThrows(UnsupportedVersionException.class, () -> setupThrowableConsumer(groupProtocol).position(tp0)); } - private KafkaConsumer setupThrowableConsumer() { + private KafkaConsumer setupThrowableConsumer(GroupProtocol groupProtocol) { long offset1 = 10000; ConsumerMetadata metadata = createMetadata(subscription); @@ -960,7 +1076,7 @@ public void testFetchStableOffsetThrowInPosition() { Node node = metadata.fetch().nodes().get(0); consumer = newConsumer( - time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, true); + groupProtocol, time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, true); consumer.assign(singletonList(tp0)); client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); @@ -971,8 +1087,11 @@ public void testFetchStableOffsetThrowInPosition() { return consumer; } - @Test - public void testNoCommittedOffsets() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testNoCommittedOffsets(GroupProtocol groupProtocol) { long offset1 = 10000; ConsumerMetadata metadata = createMetadata(subscription); @@ -981,7 +1100,7 @@ public void testNoCommittedOffsets() { initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(Arrays.asList(tp0, tp1)); // lookup coordinator @@ -996,15 +1115,18 @@ public void testNoCommittedOffsets() { assertNull(committed.get(tp1)); } - @Test - public void testAutoCommitSentBeforePositionUpdate() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testAutoCommitSentBeforePositionUpdate(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1027,8 +1149,11 @@ public void testAutoCommitSentBeforePositionUpdate() { assertTrue(commitReceived.get()); } - @Test - public void testRegexSubscription() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testRegexSubscription(GroupProtocol groupProtocol) { String unmatchedTopic = "unmatched"; ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1040,7 +1165,7 @@ public void testRegexSubscription() { initMetadata(client, partitionCounts); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); prepareRebalance(client, node, singleton(topic), assignor, singletonList(tp0), null); consumer.subscribe(Pattern.compile(topic), getConsumerRebalanceListener(consumer)); @@ -1053,8 +1178,11 @@ public void testRegexSubscription() { assertEquals(singleton(tp0), consumer.assignment()); } - @Test - public void testChangingRegexSubscription() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testChangingRegexSubscription(GroupProtocol groupProtocol) { String otherTopic = "other"; TopicPartition otherTopicPartition = new TopicPartition(otherTopic, 0); @@ -1068,7 +1196,7 @@ public void testChangingRegexSubscription() { initMetadata(client, partitionCounts); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); Node coordinator = prepareRebalance(client, node, singleton(topic), assignor, singletonList(tp0), null); consumer.subscribe(Pattern.compile(topic), getConsumerRebalanceListener(consumer)); @@ -1087,15 +1215,18 @@ public void testChangingRegexSubscription() { assertEquals(singleton(otherTopic), consumer.subscription()); } - @Test - public void testWakeupWithFetchDataAvailable() throws Exception { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testWakeupWithFetchDataAvailable(GroupProtocol groupProtocol) throws Exception { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1125,15 +1256,18 @@ public void testWakeupWithFetchDataAvailable() throws Exception { exec.awaitTermination(5L, TimeUnit.SECONDS); } - @Test - public void testPollThrowsInterruptExceptionIfInterrupted() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testPollThrowsInterruptExceptionIfInterrupted(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1150,15 +1284,16 @@ public void testPollThrowsInterruptExceptionIfInterrupted() { } } - @Test - public void fetchResponseWithUnexpectedPartitionIsIgnored() { + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void fetchResponseWithUnexpectedPartitionIsIgnored(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singletonList(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1182,9 +1317,12 @@ public void fetchResponseWithUnexpectedPartitionIsIgnored() { * Upon unsubscribing from subscribed topics the consumer subscription and assignment * are both updated right away but its consumed offsets are not auto committed. */ - @Test + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testSubscriptionChangesWithAutoCommitEnabled() { + public void testSubscriptionChangesWithAutoCommitEnabled(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1197,7 +1335,7 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); // initial subscription consumer.subscribe(Arrays.asList(topic, topic2), getConsumerRebalanceListener(consumer)); @@ -1295,8 +1433,11 @@ public void testSubscriptionChangesWithAutoCommitEnabled() { * Upon unsubscribing from subscribed topics, the assigned partitions immediately * change but if auto-commit is disabled the consumer offsets are not committed. */ - @Test - public void testSubscriptionChangesWithAutoCommitDisabled() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testSubscriptionChangesWithAutoCommitDisabled(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1308,7 +1449,7 @@ public void testSubscriptionChangesWithAutoCommitDisabled() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); initializeSubscriptionWithSingleTopic(consumer, getConsumerRebalanceListener(consumer)); @@ -1349,8 +1490,11 @@ public void testSubscriptionChangesWithAutoCommitDisabled() { client.requests().clear(); } - @Test - public void testUnsubscribeShouldTriggerPartitionsRevokedWithValidGeneration() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testUnsubscribeShouldTriggerPartitionsRevokedWithValidGeneration(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1358,7 +1502,7 @@ public void testUnsubscribeShouldTriggerPartitionsRevokedWithValidGeneration() { Node node = metadata.fetch().nodes().get(0); CooperativeStickyAssignor assignor = new CooperativeStickyAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); initializeSubscriptionWithSingleTopic(consumer, getExceptionConsumerRebalanceListener()); @@ -1372,8 +1516,11 @@ public void testUnsubscribeShouldTriggerPartitionsRevokedWithValidGeneration() { assertEquals(partitionRevoked + singleTopicPartition, unsubscribeException.getCause().getMessage()); } - @Test - public void testUnsubscribeShouldTriggerPartitionsLostWithNoGeneration() throws Exception { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testUnsubscribeShouldTriggerPartitionsLostWithNoGeneration(GroupProtocol groupProtocol) throws Exception { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1381,7 +1528,7 @@ public void testUnsubscribeShouldTriggerPartitionsLostWithNoGeneration() throws Node node = metadata.fetch().nodes().get(0); CooperativeStickyAssignor assignor = new CooperativeStickyAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); initializeSubscriptionWithSingleTopic(consumer, getExceptionConsumerRebalanceListener()); Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1407,9 +1554,12 @@ private void initializeSubscriptionWithSingleTopic(KafkaConsumer consumer, assertEquals(Collections.emptySet(), consumer.assignment()); } - @Test + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testManualAssignmentChangeWithAutoCommitEnabled() { + public void testManualAssignmentChangeWithAutoCommitEnabled(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1421,7 +1571,7 @@ public void testManualAssignmentChangeWithAutoCommitEnabled() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); // lookup coordinator client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); @@ -1462,8 +1612,11 @@ public void testManualAssignmentChangeWithAutoCommitEnabled() { client.requests().clear(); } - @Test - public void testManualAssignmentChangeWithAutoCommitDisabled() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testManualAssignmentChangeWithAutoCommitDisabled(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1475,7 +1628,7 @@ public void testManualAssignmentChangeWithAutoCommitDisabled() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); // lookup coordinator client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); @@ -1517,8 +1670,11 @@ public void testManualAssignmentChangeWithAutoCommitDisabled() { client.requests().clear(); } - @Test - public void testOffsetOfPausedPartitions() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testOffsetOfPausedPartitions(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1527,7 +1683,7 @@ public void testOffsetOfPausedPartitions() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); // lookup coordinator client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); @@ -1567,38 +1723,47 @@ public void testOffsetOfPausedPartitions() { consumer.unsubscribe(); } - @Test - public void testPollWithNoSubscription() { - consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testPollWithNoSubscription(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); assertThrows(IllegalStateException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testPollWithEmptySubscription() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testPollWithEmptySubscription(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); consumer.subscribe(Collections.emptyList()); assertThrows(IllegalStateException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testPollWithEmptyUserAssignment() { - consumer = newConsumer(groupId); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testPollWithEmptyUserAssignment(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, groupId); consumer.assign(Collections.emptySet()); assertThrows(IllegalStateException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testGracefulClose() throws Exception { + // TODO: this test references RPCs to be sent that are not part of the CONSUMER group protocol. + // We are deferring any attempts at generalizing this test for both group protocols to the future. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testGracefulClose(GroupProtocol groupProtocol) throws Exception { Map response = new HashMap<>(); response.put(tp0, Errors.NONE); OffsetCommitResponse commitResponse = offsetCommitResponse(response); LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(new LeaveGroupResponseData().setErrorCode(Errors.NONE.code())); FetchResponse closeResponse = FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>()); - consumerCloseTest(5000, Arrays.asList(commitResponse, leaveGroupResponse, closeResponse), 0, false); + consumerCloseTest(groupProtocol, 5000, Arrays.asList(commitResponse, leaveGroupResponse, closeResponse), 0, false); } - @Test - public void testCloseTimeoutDueToNoResponseForCloseFetchRequest() throws Exception { + // TODO: this test references RPCs to be sent that are not part of the CONSUMER group protocol. + // We are deferring any attempts at generalizing this test for both group protocols to the future. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCloseTimeoutDueToNoResponseForCloseFetchRequest(GroupProtocol groupProtocol) throws Exception { Map response = new HashMap<>(); response.put(tp0, Errors.NONE); OffsetCommitResponse commitResponse = offsetCommitResponse(response); @@ -1610,39 +1775,54 @@ public void testCloseTimeoutDueToNoResponseForCloseFetchRequest() throws Excepti // than configured timeout. final int closeTimeoutMs = 5000; final int waitForCloseCompletionMs = closeTimeoutMs + 1000; - consumerCloseTest(closeTimeoutMs, serverResponsesWithoutCloseResponse, waitForCloseCompletionMs, false); + consumerCloseTest(groupProtocol, closeTimeoutMs, serverResponsesWithoutCloseResponse, waitForCloseCompletionMs, false); } - @Test - public void testCloseTimeout() throws Exception { - consumerCloseTest(5000, Collections.emptyList(), 5000, false); + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCloseTimeout(GroupProtocol groupProtocol) throws Exception { + consumerCloseTest(groupProtocol, 5000, Collections.emptyList(), 5000, false); } - @Test - public void testLeaveGroupTimeout() throws Exception { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testLeaveGroupTimeout(GroupProtocol groupProtocol) throws Exception { Map response = new HashMap<>(); response.put(tp0, Errors.NONE); OffsetCommitResponse commitResponse = offsetCommitResponse(response); - consumerCloseTest(5000, singletonList(commitResponse), 5000, false); + consumerCloseTest(groupProtocol, 5000, singletonList(commitResponse), 5000, false); } - @Test - public void testCloseNoWait() throws Exception { - consumerCloseTest(0, Collections.emptyList(), 0, false); + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCloseNoWait(GroupProtocol groupProtocol) throws Exception { + consumerCloseTest(groupProtocol, 0, Collections.emptyList(), 0, false); } - @Test - public void testCloseInterrupt() throws Exception { - consumerCloseTest(Long.MAX_VALUE, Collections.emptyList(), 0, true); + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCloseInterrupt(GroupProtocol groupProtocol) throws Exception { + consumerCloseTest(groupProtocol, Long.MAX_VALUE, Collections.emptyList(), 0, true); } - @Test - public void testCloseShouldBeIdempotent() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCloseShouldBeIdempotent(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = spy(new MockClient(time, metadata)); initMetadata(client, singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); consumer.close(Duration.ZERO); consumer.close(Duration.ZERO); @@ -1651,47 +1831,48 @@ public void testCloseShouldBeIdempotent() { verify(client).close(); } - @Test - public void testOperationsBySubscribingConsumerWithDefaultGroupId() { - try { - newConsumer(null, Optional.of(Boolean.TRUE)); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testOperationsBySubscribingConsumerWithDefaultGroupId(GroupProtocol groupProtocol) { + try (KafkaConsumer consumer = newConsumer(groupProtocol, null, Optional.of(Boolean.TRUE))) { fail("Expected an InvalidConfigurationException"); - } catch (KafkaException e) { - assertEquals(InvalidConfigurationException.class, e.getCause().getClass()); + } catch (InvalidConfigurationException e) { + // OK, expected } - try { - newConsumer((String) null).subscribe(Collections.singleton(topic)); + try (KafkaConsumer consumer = newConsumer(groupProtocol, (String) null)) { + consumer.subscribe(Collections.singleton(topic)); fail("Expected an InvalidGroupIdException"); } catch (InvalidGroupIdException e) { // OK, expected } - try { - newConsumer((String) null).committed(Collections.singleton(tp0)).get(tp0); + try (KafkaConsumer consumer = newConsumer(groupProtocol, (String) null)) { + consumer.committed(Collections.singleton(tp0)).get(tp0); fail("Expected an InvalidGroupIdException"); } catch (InvalidGroupIdException e) { // OK, expected } - try { - newConsumer((String) null).commitAsync(); + try (KafkaConsumer consumer = newConsumer(groupProtocol, (String) null)) { + consumer.commitAsync(); fail("Expected an InvalidGroupIdException"); } catch (InvalidGroupIdException e) { // OK, expected } - try { - newConsumer((String) null).commitSync(); + try (KafkaConsumer consumer = newConsumer(groupProtocol, (String) null)) { + consumer.commitSync(); fail("Expected an InvalidGroupIdException"); } catch (InvalidGroupIdException e) { // OK, expected } } - @Test - public void testOperationsByAssigningConsumerWithDefaultGroupId() { - KafkaConsumer consumer = newConsumer((String) null); + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testOperationsByAssigningConsumerWithDefaultGroupId(GroupProtocol groupProtocol) { + KafkaConsumer consumer = newConsumer(groupProtocol, null); consumer.assign(singleton(tp0)); try { @@ -1714,32 +1895,39 @@ public void testOperationsByAssigningConsumerWithDefaultGroupId() { } catch (InvalidGroupIdException e) { // OK, expected } + + consumer.close(); } - @Test - public void testMetricConfigRecordingLevelInfo() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testMetricConfigRecordingLevelInfo(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); - KafkaConsumer consumer = new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); - assertEquals(Sensor.RecordingLevel.INFO, consumer.metrics.config().recordLevel()); + KafkaConsumer consumer = newConsumer(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + assertEquals(Sensor.RecordingLevel.INFO, consumer.metricsRegistry().config().recordLevel()); consumer.close(Duration.ZERO); props.put(ConsumerConfig.METRICS_RECORDING_LEVEL_CONFIG, "DEBUG"); - KafkaConsumer consumer2 = new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); - assertEquals(Sensor.RecordingLevel.DEBUG, consumer2.metrics.config().recordLevel()); + KafkaConsumer consumer2 = newConsumer(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + assertEquals(Sensor.RecordingLevel.DEBUG, consumer2.metricsRegistry().config().recordLevel()); consumer2.close(Duration.ZERO); } - @Test + // TODO: this test references RPCs to be sent that are not part of the CONSUMER group protocol. + // We are deferring any attempts at generalizing this test for both group protocols to the future. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testShouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception { + public void testShouldAttemptToRejoinGroupAfterSyncGroupFailed(GroupProtocol groupProtocol) throws Exception { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); @@ -1797,7 +1985,8 @@ public void testShouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exceptio assertFalse(records.isEmpty()); } - private void consumerCloseTest(final long closeTimeoutMs, + private void consumerCloseTest(GroupProtocol groupProtocol, + final long closeTimeoutMs, List responses, long waitMs, boolean interrupt) throws Exception { @@ -1807,7 +1996,7 @@ private void consumerCloseTest(final long closeTimeoutMs, initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, Optional.empty()); + final KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, Optional.empty()); consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer)); Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null); @@ -1893,8 +2082,11 @@ private void consumerCloseTest(final long closeTimeoutMs, } } - @Test - public void testPartitionsForNonExistingTopic() { + // TODO: this test requires topic metadata logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testPartitionsForNonExistingTopic(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -1907,59 +2099,83 @@ public void testPartitionsForNonExistingTopic() { Collections.emptyList()); client.prepareResponse(updateResponse); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); assertEquals(Collections.emptyList(), consumer.partitionsFor("non-exist-topic")); } - @Test - public void testPartitionsForAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test requires topic metadata logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testPartitionsForAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); assertThrows(AuthenticationException.class, () -> consumer.partitionsFor("some other topic")); } - @Test - public void testBeginningOffsetsAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testBeginningOffsetsAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); assertThrows(AuthenticationException.class, () -> consumer.beginningOffsets(Collections.singleton(tp0))); } - @Test - public void testEndOffsetsAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testEndOffsetsAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); assertThrows(AuthenticationException.class, () -> consumer.endOffsets(Collections.singleton(tp0))); } - @Test - public void testPollAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testPollAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); consumer.subscribe(singleton(topic)); assertThrows(AuthenticationException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testOffsetsForTimesAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testOffsetsForTimesAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); assertThrows(AuthenticationException.class, () -> consumer.offsetsForTimes(singletonMap(tp0, 0L))); } - @Test - public void testCommitSyncAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCommitSyncAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); Map offsets = new HashMap<>(); offsets.put(tp0, new OffsetAndMetadata(10L)); assertThrows(AuthenticationException.class, () -> consumer.commitSync(offsets)); } - @Test - public void testCommittedAuthenticationFailure() { - final KafkaConsumer consumer = consumerWithPendingAuthenticationError(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testCommittedAuthenticationFailure(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerWithPendingAuthenticationError(groupProtocol); assertThrows(AuthenticationException.class, () -> consumer.committed(Collections.singleton(tp0)).get(tp0)); } - @Test - public void testMeasureCommitSyncDurationOnFailure() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testMeasureCommitSyncDurationOnFailure(GroupProtocol groupProtocol) { final KafkaConsumer consumer - = consumerWithPendingError(new MockTime(Duration.ofSeconds(1).toMillis())); + = consumerWithPendingError(groupProtocol, new MockTime(Duration.ofSeconds(1).toMillis())); try { consumer.commitSync(Collections.singletonMap(tp0, new OffsetAndMetadata(10L))); @@ -1967,12 +2183,15 @@ public void testMeasureCommitSyncDurationOnFailure() { } final Metric metric = consumer.metrics() - .get(consumer.metrics.metricName("commit-sync-time-ns-total", "consumer-metrics")); + .get(consumer.metricsRegistry().metricName("commit-sync-time-ns-total", "consumer-metrics")); assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos()); } - @Test - public void testMeasureCommitSyncDuration() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testMeasureCommitSyncDuration(GroupProtocol groupProtocol) { Time time = new MockTime(Duration.ofSeconds(1).toMillis()); SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); @@ -1980,7 +2199,7 @@ public void testMeasureCommitSyncDuration() { MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(singletonList(tp0)); @@ -1995,14 +2214,17 @@ public void testMeasureCommitSyncDuration() { consumer.commitSync(Collections.singletonMap(tp0, new OffsetAndMetadata(10L))); final Metric metric = consumer.metrics() - .get(consumer.metrics.metricName("commit-sync-time-ns-total", "consumer-metrics")); + .get(consumer.metricsRegistry().metricName("commit-sync-time-ns-total", "consumer-metrics")); assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos()); } - @Test - public void testMeasureCommittedDurationOnFailure() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testMeasureCommittedDurationOnFailure(GroupProtocol groupProtocol) { final KafkaConsumer consumer - = consumerWithPendingError(new MockTime(Duration.ofSeconds(1).toMillis())); + = consumerWithPendingError(groupProtocol, new MockTime(Duration.ofSeconds(1).toMillis())); try { consumer.committed(Collections.singleton(tp0)); @@ -2010,12 +2232,15 @@ public void testMeasureCommittedDurationOnFailure() { } final Metric metric = consumer.metrics() - .get(consumer.metrics.metricName("committed-time-ns-total", "consumer-metrics")); + .get(consumer.metricsRegistry().metricName("committed-time-ns-total", "consumer-metrics")); assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos()); } - @Test - public void testMeasureCommittedDuration() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testMeasureCommittedDuration(GroupProtocol groupProtocol) { long offset1 = 10000; Time time = new MockTime(Duration.ofSeconds(1).toMillis()); SubscriptionState subscription = new SubscriptionState(new LogContext(), @@ -2024,7 +2249,7 @@ public void testMeasureCommittedDuration() { MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(singletonList(tp0)); @@ -2040,19 +2265,22 @@ public void testMeasureCommittedDuration() { consumer.committed(Collections.singleton(tp0)).get(tp0).offset(); final Metric metric = consumer.metrics() - .get(consumer.metrics.metricName("committed-time-ns-total", "consumer-metrics")); + .get(consumer.metricsRegistry().metricName("committed-time-ns-total", "consumer-metrics")); assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos()); } - @Test - public void testRebalanceException() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testRebalanceException(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(topic), getExceptionConsumerRebalanceListener()); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); @@ -2086,13 +2314,16 @@ public void testRebalanceException() { assertTrue(subscription.assignedPartitions().isEmpty()); } - @Test - public void testReturnRecordsDuringRebalance() throws InterruptedException { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testReturnRecordsDuringRebalance(GroupProtocol groupProtocol) throws InterruptedException { Time time = new MockTime(1L); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); ConsumerPartitionAssignor assignor = new CooperativeStickyAssignor(); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1))); @@ -2211,15 +2442,18 @@ public void testReturnRecordsDuringRebalance() throws InterruptedException { consumer.close(Duration.ZERO); } - @Test - public void testGetGroupMetadata() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testGetGroupMetadata(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); final Node node = metadata.fetch().nodes().get(0); - final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + final KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); final ConsumerGroupMetadata groupMetadataOnStart = consumer.groupMetadata(); assertEquals(groupId, groupMetadataOnStart.groupId()); @@ -2241,12 +2475,15 @@ public void testGetGroupMetadata() { assertEquals(groupInstanceId, groupMetadataAfterPoll.groupInstanceId()); } - @Test - public void testInvalidGroupMetadata() throws InterruptedException { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testInvalidGroupMetadata(GroupProtocol groupProtocol) throws InterruptedException { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, new RoundRobinAssignor(), true, groupInstanceId); consumer.subscribe(singletonList(topic)); // concurrent access is illegal @@ -2268,15 +2505,18 @@ public void testInvalidGroupMetadata() throws InterruptedException { assertThrows(IllegalStateException.class, consumer::groupMetadata); } - @Test + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") @SuppressWarnings("unchecked") - public void testCurrentLag() { + public void testCurrentLag(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); // throws for unassigned partition assertThrows(IllegalStateException.class, () -> consumer.currentLag(tp0)); @@ -2321,14 +2561,17 @@ public void testCurrentLag() { assertEquals(OptionalLong.of(45L), consumer.currentLag(tp0)); } - @Test - public void testListOffsetShouldUpdateSubscriptions() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testListOffsetShouldUpdateSubscriptions(GroupProtocol groupProtocol) { final ConsumerMetadata metadata = createMetadata(subscription); final MockClient client = new MockClient(time, metadata); initMetadata(client, singletonMap(topic, 1)); - consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.assign(singleton(tp0)); @@ -2344,7 +2587,8 @@ public void testListOffsetShouldUpdateSubscriptions() { assertEquals(OptionalLong.of(40L), consumer.currentLag(tp0)); } - private KafkaConsumer consumerWithPendingAuthenticationError(final Time time) { + private KafkaConsumer consumerWithPendingAuthenticationError(GroupProtocol groupProtocol, + final Time time) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -2354,15 +2598,15 @@ private KafkaConsumer consumerWithPendingAuthenticationError(fin ConsumerPartitionAssignor assignor = new RangeAssignor(); client.createPendingAuthenticationError(node, 0); - return newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + return newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); } - private KafkaConsumer consumerWithPendingAuthenticationError() { - return consumerWithPendingAuthenticationError(new MockTime()); + private KafkaConsumer consumerWithPendingAuthenticationError(GroupProtocol groupProtocol) { + return consumerWithPendingAuthenticationError(groupProtocol, new MockTime()); } - private KafkaConsumer consumerWithPendingError(final Time time) { - return consumerWithPendingAuthenticationError(time); + private KafkaConsumer consumerWithPendingError(GroupProtocol groupProtocol, final Time time) { + return consumerWithPendingAuthenticationError(groupProtocol, time); } private ConsumerRebalanceListener getConsumerRebalanceListener(final KafkaConsumer consumer) { @@ -2512,6 +2756,7 @@ private OffsetFetchResponse offsetResponse(Map offsets, Er partitionData.put(entry.getKey(), new OffsetFetchResponse.PartitionData(entry.getValue(), Optional.empty(), "", error)); } + int throttleMs = 10; return new OffsetFetchResponse( throttleMs, Collections.singletonMap(groupId, Errors.NONE), @@ -2561,11 +2806,12 @@ private FetchResponse fetchResponse(Map fetches) { if (fetchCount == 0) { records = MemoryRecords.EMPTY; } else { - MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, - TimestampType.CREATE_TIME, fetchOffset); - for (int i = 0; i < fetchCount; i++) - builder.append(0L, ("key-" + i).getBytes(), ("value-" + i).getBytes()); - records = builder.build(); + try (MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, fetchOffset)) { + for (int i = 0; i < fetchCount; i++) + builder.append(0L, ("key-" + i).getBytes(), ("value-" + i).getBytes()); + records = builder.build(); + } } tpResponses.put(new TopicIdPartition(topicIds.get(partition.topic()), partition), new FetchResponseData.PartitionData() @@ -2582,24 +2828,49 @@ private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, return fetchResponse(Collections.singletonMap(partition, fetchInfo)); } - private KafkaConsumer newConsumer(Time time, + private KafkaConsumer newConsumer(GroupProtocol groupProtocol, + Time time, KafkaClient client, SubscriptionState subscription, ConsumerMetadata metadata, ConsumerPartitionAssignor assignor, boolean autoCommitEnabled, Optional groupInstanceId) { - return newConsumer(time, client, subscription, metadata, assignor, autoCommitEnabled, groupId, groupInstanceId, false); + return newConsumer( + groupProtocol, + time, + client, + subscription, + metadata, + assignor, + autoCommitEnabled, + groupId, + groupInstanceId, + false + ); } - private KafkaConsumer newConsumerNoAutoCommit(Time time, + private KafkaConsumer newConsumerNoAutoCommit(GroupProtocol groupProtocol, + Time time, KafkaClient client, SubscriptionState subscription, ConsumerMetadata metadata) { - return newConsumer(time, client, subscription, metadata, new RangeAssignor(), false, groupId, groupInstanceId, false); + return newConsumer( + groupProtocol, + time, + client, + subscription, + metadata, + new RangeAssignor(), + false, + groupId, + groupInstanceId, + false + ); } - private KafkaConsumer newConsumer(Time time, + private KafkaConsumer newConsumer(GroupProtocol groupProtocol, + Time time, KafkaClient client, SubscriptionState subscription, ConsumerMetadata metadata, @@ -2608,22 +2879,64 @@ private KafkaConsumer newConsumer(Time time, String groupId, Optional groupInstanceId, boolean throwOnStableOffsetNotSupported) { - return newConsumer(time, client, subscription, metadata, assignor, autoCommitEnabled, groupId, groupInstanceId, - Optional.of(new StringDeserializer()), throwOnStableOffsetNotSupported); + return newConsumer( + groupProtocol, + time, + client, + subscription, + metadata, + assignor, + autoCommitEnabled, + groupId, + groupInstanceId, + Optional.of(new StringDeserializer()), + throwOnStableOffsetNotSupported + ); } - private KafkaConsumer newConsumer(Time time, + private KafkaConsumer newConsumer(GroupProtocol groupProtocol, + Time time, KafkaClient client, - SubscriptionState subscription, + SubscriptionState subscriptions, ConsumerMetadata metadata, ConsumerPartitionAssignor assignor, boolean autoCommitEnabled, String groupId, Optional groupInstanceId, - Optional> valueDeserializer, + Optional> valueDeserializerOpt, boolean throwOnStableOffsetNotSupported) { + Deserializer keyDeserializer = new StringDeserializer(); + Deserializer valueDeserializer = valueDeserializerOpt.orElse(new StringDeserializer()); + LogContext logContext = new LogContext(); + List assignors = singletonList(assignor); + ConsumerConfig config = newConsumerConfig( + groupProtocol, + autoCommitEnabled, + groupId, + groupInstanceId, + valueDeserializer, + throwOnStableOffsetNotSupported + ); + return new KafkaConsumer<>( + logContext, + time, + config, + keyDeserializer, + valueDeserializer, + client, + subscriptions, + metadata, + assignors + ); + } + + private ConsumerConfig newConsumerConfig(GroupProtocol groupProtocol, + boolean autoCommitEnabled, + String groupId, + Optional groupInstanceId, + Deserializer valueDeserializer, + boolean throwOnStableOffsetNotSupported) { String clientId = "mock-consumer"; - String metricGroupPrefix = "consumer"; long retryBackoffMs = 100; long retryBackoffMaxMs = 1000; int minBytes = 1; @@ -2633,101 +2946,35 @@ private KafkaConsumer newConsumer(Time time, int maxPollRecords = Integer.MAX_VALUE; boolean checkCrcs = true; int rebalanceTimeoutMs = 60000; + int requestTimeoutMs = defaultApiTimeoutMs / 2; - Deserializer keyDeserializer = new StringDeserializer(); - Deserializer deserializer = valueDeserializer.orElse(new StringDeserializer()); - - List assignors = singletonList(assignor); - ConsumerInterceptors interceptors = new ConsumerInterceptors<>(Collections.emptyList()); - - Metrics metrics = new Metrics(time); - ConsumerMetrics metricsRegistry = new ConsumerMetrics(metricGroupPrefix); - - LogContext loggerFactory = new LogContext(); - ConsumerNetworkClient consumerClient = new ConsumerNetworkClient(loggerFactory, client, metadata, time, - retryBackoffMs, requestTimeoutMs, heartbeatIntervalMs); - - ConsumerCoordinator consumerCoordinator = null; - if (groupId != null) { - GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs, - rebalanceTimeoutMs, - heartbeatIntervalMs, - groupId, - groupInstanceId, - retryBackoffMs, - retryBackoffMaxMs, - true); - consumerCoordinator = new ConsumerCoordinator(rebalanceConfig, - loggerFactory, - consumerClient, - assignors, - metadata, - subscription, - metrics, - metricGroupPrefix, - time, - autoCommitEnabled, - autoCommitIntervalMs, - interceptors, - throwOnStableOffsetNotSupported, - null); - } - IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED; - FetchMetricsManager metricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics); - FetchConfig fetchConfig = new FetchConfig( - minBytes, - maxBytes, - maxWaitMs, - fetchSize, - maxPollRecords, - checkCrcs, - CommonClientConfigs.DEFAULT_CLIENT_RACK, - isolationLevel); - Fetcher fetcher = new Fetcher<>( - loggerFactory, - consumerClient, - metadata, - subscription, - fetchConfig, - new Deserializers<>(keyDeserializer, deserializer), - metricsManager, - time, - new ApiVersions()); - OffsetFetcher offsetFetcher = new OffsetFetcher(loggerFactory, - consumerClient, - metadata, - subscription, - time, - retryBackoffMs, - requestTimeoutMs, - isolationLevel, - new ApiVersions()); - TopicMetadataFetcher topicMetadataFetcher = new TopicMetadataFetcher(loggerFactory, - consumerClient, - retryBackoffMs, - retryBackoffMaxMs); - - return new KafkaConsumer<>( - loggerFactory, - clientId, - consumerCoordinator, - keyDeserializer, - deserializer, - fetcher, - offsetFetcher, - topicMetadataFetcher, - interceptors, - time, - consumerClient, - metrics, - subscription, - metadata, - retryBackoffMs, - retryBackoffMaxMs, - requestTimeoutMs, - defaultApiTimeoutMs, - assignors, - groupId); + Map configs = new HashMap<>(); + configs.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitIntervalMs); + configs.put(ConsumerConfig.CHECK_CRCS_CONFIG, checkCrcs); + configs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); + configs.put(ConsumerConfig.CLIENT_RACK_CONFIG, CommonClientConfigs.DEFAULT_CLIENT_RACK); + configs.put(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, defaultApiTimeoutMs); + configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, autoCommitEnabled); + configs.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, maxBytes); + configs.put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, maxWaitMs); + configs.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, minBytes); + configs.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + configs.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); + configs.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, heartbeatIntervalMs); + configs.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, IsolationLevel.READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)); + configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + configs.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, fetchSize); + configs.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, rebalanceTimeoutMs); + configs.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords); + configs.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs); + configs.put(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG, retryBackoffMaxMs); + configs.put(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, retryBackoffMs); + configs.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs); + configs.put(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED, throwOnStableOffsetNotSupported); + configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); + groupInstanceId.ifPresent(gi -> configs.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, gi)); + + return new ConsumerConfig(configs); } private static class FetchInfo { @@ -2748,8 +2995,11 @@ private static class FetchInfo { } } - @Test - public void testSubscriptionOnInvalidTopic() { + // TODO: this test requires rebalance logic which is not yet implemented in the CONSUMER group protocol. + // Once it is implemented, this should use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testSubscriptionOnInvalidTopic(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -2767,22 +3017,23 @@ public void testSubscriptionOnInvalidTopic() { topicMetadata); client.prepareMetadataUpdate(updateResponse); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singleton(invalidTopicName), getConsumerRebalanceListener(consumer)); assertThrows(InvalidTopicException.class, () -> consumer.poll(Duration.ZERO)); } - @Test - public void testPollTimeMetrics() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testPollTimeMetrics(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); consumer.subscribe(singletonList(topic)); // MetricName objects to check - Metrics metrics = consumer.metrics; + Metrics metrics = consumer.metricsRegistry(); MetricName lastPollSecondsAgoName = metrics.metricName("last-poll-seconds-ago", "consumer-metrics"); MetricName timeBetweenPollAvgName = metrics.metricName("time-between-poll-avg", "consumer-metrics"); MetricName timeBetweenPollMaxName = metrics.metricName("time-between-poll-max", "consumer-metrics"); @@ -2818,32 +3069,33 @@ public void testPollTimeMetrics() { assertEquals(10 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue()); } - @Test - public void testPollIdleRatio() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) +public void testPollIdleRatio(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); // MetricName object to check - Metrics metrics = consumer.metrics; + Metrics metrics = consumer.metricsRegistry(); MetricName pollIdleRatio = metrics.metricName("poll-idle-ratio-avg", "consumer-metrics"); // Test default value assertEquals(Double.NaN, consumer.metrics().get(pollIdleRatio).metricValue()); // 1st poll // Spend 50ms in poll so value = 1.0 - consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollStart(time.milliseconds()); time.sleep(50); - consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollEnd(time.milliseconds()); assertEquals(1.0d, consumer.metrics().get(pollIdleRatio).metricValue()); // 2nd poll // Spend 50m outside poll and 0ms in poll so value = 0.0 time.sleep(50); - consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds()); - consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollStart(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollEnd(time.milliseconds()); // Avg of first two data points assertEquals((1.0d + 0.0d) / 2, consumer.metrics().get(pollIdleRatio).metricValue()); @@ -2851,9 +3103,9 @@ public void testPollIdleRatio() { // 3rd poll // Spend 25ms outside poll and 25ms in poll so value = 0.5 time.sleep(25); - consumer.kafkaConsumerMetrics.recordPollStart(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollStart(time.milliseconds()); time.sleep(25); - consumer.kafkaConsumerMetrics.recordPollEnd(time.milliseconds()); + consumer.kafkaConsumerMetrics().recordPollEnd(time.milliseconds()); // Avg of three data points assertEquals((1.0d + 0.0d + 0.5d) / 3, consumer.metrics().get(pollIdleRatio).metricValue()); @@ -2861,16 +3113,17 @@ public void testPollIdleRatio() { private static boolean consumerMetricPresent(KafkaConsumer consumer, String name) { MetricName metricName = new MetricName(name, "consumer-metrics", "", Collections.emptyMap()); - return consumer.metrics.metrics().containsKey(metricName); + return consumer.metricsRegistry().metrics().containsKey(metricName); } - @Test - public void testClosingConsumerUnregistersConsumerMetrics() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) +public void testClosingConsumerUnregistersConsumerMetrics(GroupProtocol groupProtocol) { Time time = new MockTime(1L); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, new RoundRobinAssignor(), true, groupInstanceId); consumer.subscribe(singletonList(topic)); assertTrue(consumerMetricPresent(consumer, "last-poll-seconds-ago")); @@ -2882,19 +3135,23 @@ public void testClosingConsumerUnregistersConsumerMetrics() { assertFalse(consumerMetricPresent(consumer, "time-between-poll-max")); } - @Test - public void testEnforceRebalanceWithManualAssignment() { - consumer = newConsumer((String) null); + // NOTE: this test uses the enforceRebalance API which is not implemented in the CONSUMER group protocol. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testEnforceRebalanceWithManualAssignment(GroupProtocol groupProtocol) { + consumer = newConsumer(groupProtocol, null); consumer.assign(singleton(new TopicPartition("topic", 0))); assertThrows(IllegalStateException.class, consumer::enforceRebalance); } - @Test - public void testEnforceRebalanceTriggersRebalanceOnNextPoll() { + // NOTE: this test uses the enforceRebalance API which is not implemented in the CONSUMER group protocol. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testEnforceRebalanceTriggersRebalanceOnNextPoll(GroupProtocol groupProtocol) { Time time = new MockTime(1L); ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); - KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId); + KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); MockRebalanceListener countingRebalanceListener = new MockRebalanceListener(); initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1))); @@ -2918,8 +3175,10 @@ public void testEnforceRebalanceTriggersRebalanceOnNextPoll() { assertEquals(countingRebalanceListener.revokedCount, 1); } - @Test - public void testEnforceRebalanceReason() { + // NOTE: this test uses the enforceRebalance API which is not implemented in the CONSUMER group protocol. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testEnforceRebalanceReason(GroupProtocol groupProtocol) { Time time = new MockTime(1L); ConsumerMetadata metadata = createMetadata(subscription); @@ -2928,6 +3187,7 @@ public void testEnforceRebalanceReason() { Node node = metadata.fetch().nodes().get(0); consumer = newConsumer( + groupProtocol, time, client, subscription, @@ -2978,24 +3238,30 @@ private void prepareJoinGroupAndVerifyReason( ); } - @Test - public void configurableObjectsShouldSeeGeneratedClientId() { + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void configurableObjectsShouldSeeGeneratedClientId(GroupProtocol groupProtocol) { Properties props = new Properties(); + props.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, DeserializerForClientId.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, DeserializerForClientId.class.getName()); props.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptorForClientId.class.getName()); - consumer = new KafkaConsumer<>(props); - assertNotNull(consumer.getClientId()); - assertNotEquals(0, consumer.getClientId().length()); + consumer = newConsumer(props); + assertNotNull(consumer.clientId()); + assertNotEquals(0, consumer.clientId().length()); assertEquals(3, CLIENT_IDS.size()); - CLIENT_IDS.forEach(id -> assertEquals(id, consumer.getClientId())); + CLIENT_IDS.forEach(id -> assertEquals(id, consumer.clientId())); } - @Test - public void testUnusedConfigs() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testUnusedConfigs(GroupProtocol groupProtocol) { Map props = new HashMap<>(); + props.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.put(SslConfigs.SSL_PROTOCOL_CONFIG, "TLS"); ConsumerConfig config = new ConsumerConfig(ConsumerConfig.appendDeserializerToConfig(props, new StringDeserializer(), new StringDeserializer())); @@ -3006,45 +3272,103 @@ public void testUnusedConfigs() { assertTrue(config.unused().contains(SslConfigs.SSL_PROTOCOL_CONFIG)); } - @Test - public void testAssignorNameConflict() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testAssignorNameConflict(GroupProtocol groupProtocol) { Map configs = new HashMap<>(); + configs.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name()); configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); configs.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, Arrays.asList(RangeAssignor.class.getName(), ConsumerPartitionAssignorTest.TestConsumerPartitionAssignor.class.getName())); assertThrows(KafkaException.class, - () -> new KafkaConsumer<>(configs, new StringDeserializer(), new StringDeserializer())); + () -> newConsumer(configs, new StringDeserializer(), new StringDeserializer())); } - @Test - public void testOffsetsForTimesTimeout() { - final KafkaConsumer consumer = consumerForCheckingTimeoutException(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testOffsetsForTimesTimeout(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerForCheckingTimeoutException(groupProtocol); assertEquals( "Failed to get offsets by times in 60000ms", assertThrows(org.apache.kafka.common.errors.TimeoutException.class, () -> consumer.offsetsForTimes(singletonMap(tp0, 0L))).getMessage() ); } - @Test - public void testBeginningOffsetsTimeout() { - final KafkaConsumer consumer = consumerForCheckingTimeoutException(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testBeginningOffsetsTimeout(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerForCheckingTimeoutException(groupProtocol); assertEquals( "Failed to get offsets by times in 60000ms", assertThrows(org.apache.kafka.common.errors.TimeoutException.class, () -> consumer.beginningOffsets(singletonList(tp0))).getMessage() ); } - @Test - public void testEndOffsetsTimeout() { - final KafkaConsumer consumer = consumerForCheckingTimeoutException(); + // TODO: this test triggers a bug with the CONSUMER group protocol implementation. + // The bug will be investigated and fixed so this test can use both group protocols. + @ParameterizedTest + @EnumSource(value = GroupProtocol.class, names = "GENERIC") + public void testEndOffsetsTimeout(GroupProtocol groupProtocol) { + final KafkaConsumer consumer = consumerForCheckingTimeoutException(groupProtocol); assertEquals( "Failed to get offsets by times in 60000ms", assertThrows(org.apache.kafka.common.errors.TimeoutException.class, () -> consumer.endOffsets(singletonList(tp0))).getMessage() ); } - private KafkaConsumer consumerForCheckingTimeoutException() { + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testClientInstanceId() { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + + ClientTelemetryReporter clientTelemetryReporter = mock(ClientTelemetryReporter.class); + clientTelemetryReporter.configure(any()); + + MockedStatic mockedCommonClientConfigs = mockStatic(CommonClientConfigs.class, new CallsRealMethods()); + mockedCommonClientConfigs.when(() -> CommonClientConfigs.telemetryReporter(anyString(), any())).thenReturn(Optional.of(clientTelemetryReporter)); + + ClientTelemetrySender clientTelemetrySender = mock(ClientTelemetrySender.class); + Uuid expectedUuid = Uuid.randomUuid(); + when(clientTelemetryReporter.telemetrySender()).thenReturn(clientTelemetrySender); + when(clientTelemetrySender.clientInstanceId(any())).thenReturn(Optional.of(expectedUuid)); + + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + Uuid uuid = consumer.clientInstanceId(Duration.ofMillis(0)); + assertEquals(expectedUuid, uuid); + + mockedCommonClientConfigs.close(); + } + + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testClientInstanceIdInvalidTimeout() { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + Exception exception = assertThrows(IllegalArgumentException.class, () -> consumer.clientInstanceId(Duration.ofMillis(-1))); + assertEquals("The timeout cannot be negative.", exception.getMessage()); + } + + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testClientInstanceIdNoTelemetryReporterRegistered() { + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ConsumerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); + + consumer = newConsumer(props, new StringDeserializer(), new StringDeserializer()); + Exception exception = assertThrows(IllegalStateException.class, () -> consumer.clientInstanceId(Duration.ofMillis(0))); + assertEquals("Telemetry is not enabled. Set config `enable.metrics.push` to `true`.", exception.getMessage()); + } + + private KafkaConsumer consumerForCheckingTimeoutException(GroupProtocol groupProtocol) { ConsumerMetadata metadata = createMetadata(subscription); MockClient client = new MockClient(time, metadata); @@ -3052,7 +3376,7 @@ private KafkaConsumer consumerForCheckingTimeoutException() { ConsumerPartitionAssignor assignor = new RangeAssignor(); - final KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId); + final KafkaConsumer consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, false, groupInstanceId); for (int i = 0; i < 10; i++) { client.prepareResponse( @@ -3069,6 +3393,32 @@ private KafkaConsumer consumerForCheckingTimeoutException() { return consumer; } + @ParameterizedTest + @EnumSource(GroupProtocol.class) + public void testCommittedThrowsTimeoutExceptionForNoResponse(GroupProtocol groupProtocol) { + Time time = new MockTime(Duration.ofSeconds(1).toMillis()); + + ConsumerMetadata metadata = createMetadata(subscription); + MockClient client = new MockClient(time, metadata); + + initMetadata(client, Collections.singletonMap(topic, 2)); + Node node = metadata.fetch().nodes().get(0); + + consumer = newConsumer(groupProtocol, time, client, subscription, metadata, assignor, true, groupInstanceId); + consumer.assign(singletonList(tp0)); + + // lookup coordinator + client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node); + Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); + + // try to get committed offsets for one topic-partition - but it is disconnected so there's no response and it will time out + client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, 0L), Errors.NONE), coordinator, true); + org.apache.kafka.common.errors.TimeoutException timeoutException = assertThrows(org.apache.kafka.common.errors.TimeoutException.class, + () -> consumer.committed(Collections.singleton(tp0), Duration.ofMillis(1000L))); + assertEquals("Timeout of 1000ms expired before the last committed offset for partitions [test-0] could be determined. " + + "Try tuning default.api.timeout.ms larger to relax the threshold.", timeoutException.getMessage()); + } + private static final List CLIENT_IDS = new ArrayList<>(); public static class DeserializerForClientId implements Deserializer { @Override diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java new file mode 100644 index 0000000000000..12b833109ecb8 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java @@ -0,0 +1,1108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.consumer.RetriableCommitFailedException; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; +import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; +import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; +import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.SubscriptionChangeApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.UnsubscribeApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.GroupAuthorizationException; +import org.apache.kafka.common.errors.InvalidGroupIdException; +import org.apache.kafka.common.errors.NetworkException; +import org.apache.kafka.common.errors.RetriableException; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.message.OffsetCommitResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.JoinGroupRequest; +import org.apache.kafka.common.requests.ListOffsetsRequest; +import org.apache.kafka.common.requests.OffsetCommitResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.Timer; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.opentest4j.AssertionFailedError; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Arrays.asList; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED; +import static org.apache.kafka.common.utils.Utils.mkEntry; +import static org.apache.kafka.common.utils.Utils.mkMap; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AsyncKafkaConsumerTest { + + private AsyncKafkaConsumer consumer; + private FetchCollector fetchCollector; + private ConsumerTestBuilder.AsyncKafkaConsumerTestBuilder testBuilder; + private ApplicationEventHandler applicationEventHandler; + private SubscriptionState subscriptions; + + @BeforeEach + public void setup() { + // By default, the consumer is part of a group and autoCommit is enabled. + setup(ConsumerTestBuilder.createDefaultGroupInformation(), true); + } + + private void setup(Optional groupInfo, boolean enableAutoCommit) { + testBuilder = new ConsumerTestBuilder.AsyncKafkaConsumerTestBuilder(groupInfo, enableAutoCommit, true); + applicationEventHandler = testBuilder.applicationEventHandler; + consumer = testBuilder.consumer; + fetchCollector = testBuilder.fetchCollector; + subscriptions = testBuilder.subscriptions; + } + + @AfterEach + public void cleanup() { + if (testBuilder != null) { + shutDown(); + } + } + + private void shutDown() { + prepAutocommitOnClose(); + testBuilder.close(); + } + + private void resetWithEmptyGroupId() { + // Create a consumer that is not configured as part of a group. + cleanup(); + setup(Optional.empty(), false); + } + + private void resetWithAutoCommitEnabled() { + cleanup(); + setup(ConsumerTestBuilder.createDefaultGroupInformation(), true); + } + + @Test + public void testSuccessfulStartupShutdown() { + assertDoesNotThrow(() -> consumer.close()); + } + + @Test + public void testSuccessfulStartupShutdownWithAutoCommit() { + resetWithAutoCommitEnabled(); + TopicPartition tp = new TopicPartition("topic", 0); + consumer.assign(singleton(tp)); + consumer.seek(tp, 100); + prepAutocommitOnClose(); + } + + @Test + public void testInvalidGroupId() { + // Create consumer without group id + resetWithEmptyGroupId(); + assertThrows(InvalidGroupIdException.class, () -> consumer.committed(new HashSet<>())); + } + + @Test + public void testFailOnClosedConsumer() { + consumer.close(); + final IllegalStateException res = assertThrows(IllegalStateException.class, consumer::assignment); + assertEquals("This consumer has already been closed.", res.getMessage()); + } + + @Test + public void testCommitAsync_NullCallback() throws InterruptedException { + CompletableFuture future = new CompletableFuture<>(); + Map offsets = new HashMap<>(); + offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); + offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); + + doReturn(future).when(consumer).commit(offsets, false); + consumer.commitAsync(offsets, null); + future.complete(null); + TestUtils.waitForCondition(future::isDone, + 2000, + "commit future should complete"); + + assertFalse(future.isCompletedExceptionally()); + } + + @ParameterizedTest + @MethodSource("commitExceptionSupplier") + public void testCommitAsync_UserSuppliedCallback(Exception exception) { + CompletableFuture future = new CompletableFuture<>(); + + Map offsets = new HashMap<>(); + offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); + + doReturn(future).when(consumer).commit(offsets, false); + MockCommitCallback callback = new MockCommitCallback(); + assertDoesNotThrow(() -> consumer.commitAsync(offsets, callback)); + + if (exception == null) { + future.complete(null); + consumer.maybeInvokeCommitCallbacks(); + assertNull(callback.exception); + } else { + future.completeExceptionally(exception); + consumer.maybeInvokeCommitCallbacks(); + assertSame(exception.getClass(), callback.exception.getClass()); + } + } + + private static Stream commitExceptionSupplier() { + return Stream.of( + null, // For the successful completion scenario + new KafkaException("Test exception"), + new GroupAuthorizationException("Group authorization exception")); + } + + @Test + public void testFencedInstanceException() { + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(consumer).commit(new HashMap<>(), false); + assertDoesNotThrow(() -> consumer.commitAsync()); + future.completeExceptionally(Errors.FENCED_INSTANCE_ID.exception()); + } + + @Test + public void testCommitted() { + Map offsets = mockTopicPartitionOffset(); + CompletableFuture> committedFuture = new CompletableFuture<>(); + committedFuture.complete(offsets); + + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + assertDoesNotThrow(() -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + } + } + + @Test + public void testCommittedLeaderEpochUpdate() { + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + final TopicPartition t2 = new TopicPartition("t0", 4); + HashMap topicPartitionOffsets = new HashMap<>(); + topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L, Optional.of(2), "")); + topicPartitionOffsets.put(t1, null); + topicPartitionOffsets.put(t2, new OffsetAndMetadata(20L, Optional.of(3), "")); + + CompletableFuture> committedFuture = new CompletableFuture<>(); + committedFuture.complete(topicPartitionOffsets); + + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + assertDoesNotThrow(() -> consumer.committed(topicPartitionOffsets.keySet(), Duration.ofMillis(1000))); + } + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t0, 2); + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t2, 3); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + } + + @Test + public void testCommitted_ExceptionThrown() { + Map offsets = mockTopicPartitionOffset(); + CompletableFuture> committedFuture = new CompletableFuture<>(); + committedFuture.completeExceptionally(new KafkaException("Test exception")); + + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + assertThrows(KafkaException.class, () -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + } + } + + @Test + public void testWakeupBeforeCallingPoll() { + final String topicName = "foo"; + final int partition = 3; + final TopicPartition tp = new TopicPartition(topicName, partition); + doReturn(Fetch.empty()).when(fetchCollector).collectFetch(any(FetchBuffer.class)); + Map offsets = mkMap(mkEntry(tp, new OffsetAndMetadata(1))); + doReturn(offsets).when(applicationEventHandler).addAndGet(any(OffsetFetchApplicationEvent.class), any(Timer.class)); + consumer.assign(singleton(tp)); + + consumer.wakeup(); + + assertThrows(WakeupException.class, () -> consumer.poll(Duration.ZERO)); + assertDoesNotThrow(() -> consumer.poll(Duration.ZERO)); + } + + @Test + public void testWakeupAfterEmptyFetch() { + final String topicName = "foo"; + final int partition = 3; + final TopicPartition tp = new TopicPartition(topicName, partition); + doAnswer(invocation -> { + consumer.wakeup(); + return Fetch.empty(); + }).when(fetchCollector).collectFetch(any(FetchBuffer.class)); + Map offsets = mkMap(mkEntry(tp, new OffsetAndMetadata(1))); + doReturn(offsets).when(applicationEventHandler).addAndGet(any(OffsetFetchApplicationEvent.class), any(Timer.class)); + consumer.assign(singleton(tp)); + + assertThrows(WakeupException.class, () -> consumer.poll(Duration.ofMinutes(1))); + assertDoesNotThrow(() -> consumer.poll(Duration.ZERO)); + } + + @Test + public void testWakeupAfterNonEmptyFetch() { + final String topicName = "foo"; + final int partition = 3; + final TopicPartition tp = new TopicPartition(topicName, partition); + final List> records = asList( + new ConsumerRecord<>(topicName, partition, 2, "key1", "value1"), + new ConsumerRecord<>(topicName, partition, 3, "key2", "value2") + ); + doAnswer(invocation -> { + consumer.wakeup(); + return Fetch.forPartition(tp, records, true); + }).when(fetchCollector).collectFetch(Mockito.any(FetchBuffer.class)); + Map offsets = mkMap(mkEntry(tp, new OffsetAndMetadata(1))); + doReturn(offsets).when(applicationEventHandler).addAndGet(any(OffsetFetchApplicationEvent.class), any(Timer.class)); + consumer.assign(singleton(tp)); + + // since wakeup() is called when the non-empty fetch is returned the wakeup should be ignored + assertDoesNotThrow(() -> consumer.poll(Duration.ofMinutes(1))); + // the previously ignored wake-up should not be ignored in the next call + assertThrows(WakeupException.class, () -> consumer.poll(Duration.ZERO)); + } + + @Test + public void testClearWakeupTriggerAfterPoll() { + final String topicName = "foo"; + final int partition = 3; + final TopicPartition tp = new TopicPartition(topicName, partition); + final List> records = asList( + new ConsumerRecord<>(topicName, partition, 2, "key1", "value1"), + new ConsumerRecord<>(topicName, partition, 3, "key2", "value2") + ); + doReturn(Fetch.forPartition(tp, records, true)) + .when(fetchCollector).collectFetch(any(FetchBuffer.class)); + Map offsets = mkMap(mkEntry(tp, new OffsetAndMetadata(1))); + doReturn(offsets).when(applicationEventHandler).addAndGet(any(OffsetFetchApplicationEvent.class), any(Timer.class)); + consumer.assign(singleton(tp)); + + consumer.poll(Duration.ZERO); + + assertDoesNotThrow(() -> consumer.poll(Duration.ZERO)); + } + + @Test + public void testEnsureCallbackExecutedByApplicationThread() { + final String currentThread = Thread.currentThread().getName(); + ExecutorService backgroundExecutor = Executors.newSingleThreadExecutor(); + MockCommitCallback callback = new MockCommitCallback(); + CountDownLatch latch = new CountDownLatch(1); // Initialize the latch with a count of 1 + try { + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(consumer).commit(new HashMap<>(), false); + assertDoesNotThrow(() -> consumer.commitAsync(new HashMap<>(), callback)); + // Simulating some background work + backgroundExecutor.submit(() -> { + future.complete(null); + latch.countDown(); + }); + latch.await(); + assertEquals(1, consumer.callbacks()); + consumer.maybeInvokeCommitCallbacks(); + assertEquals(currentThread, callback.completionThread); + } catch (Exception e) { + fail("Not expecting an exception"); + } finally { + backgroundExecutor.shutdown(); + } + } + + @Test + public void testEnsureCommitSyncExecutedCommitAsyncCallbacks() { + MockCommitCallback callback = new MockCommitCallback(); + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(consumer).commit(new HashMap<>(), false); + assertDoesNotThrow(() -> consumer.commitAsync(new HashMap<>(), callback)); + future.completeExceptionally(new NetworkException("Test exception")); + assertMockCommitCallbackInvoked(() -> consumer.commitSync(), + callback, + Errors.NETWORK_EXCEPTION); + } + + @Test + @SuppressWarnings("deprecation") + public void testPollLongThrowsException() { + Exception e = assertThrows(UnsupportedOperationException.class, () -> consumer.poll(0L)); + assertEquals("Consumer.poll(long) is not supported when \"group.protocol\" is \"consumer\". " + + "This method is deprecated and will be removed in the next major release.", e.getMessage()); + } + + @Test + public void testCommitSyncLeaderEpochUpdate() { + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + HashMap topicPartitionOffsets = new HashMap<>(); + topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L, Optional.of(2), "")); + topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L, Optional.of(1), "")); + + consumer.assign(Arrays.asList(t0, t1)); + + CompletableFuture commitFuture = new CompletableFuture<>(); + commitFuture.complete(null); + + try (MockedConstruction ignored = commitEventMocker(commitFuture)) { + assertDoesNotThrow(() -> consumer.commitSync(topicPartitionOffsets)); + } + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t0, 2); + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t1, 1); + verify(applicationEventHandler).add(ArgumentMatchers.isA(CommitApplicationEvent.class)); + } + + @Test + public void testCommitAsyncLeaderEpochUpdate() { + MockCommitCallback callback = new MockCommitCallback(); + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + HashMap topicPartitionOffsets = new HashMap<>(); + topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L, Optional.of(2), "")); + topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L, Optional.of(1), "")); + + consumer.assign(Arrays.asList(t0, t1)); + + CompletableFuture commitFuture = new CompletableFuture<>(); + commitFuture.complete(null); + + try (MockedConstruction ignored = commitEventMocker(commitFuture)) { + assertDoesNotThrow(() -> consumer.commitAsync(topicPartitionOffsets, callback)); + } + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t0, 2); + verify(testBuilder.metadata).updateLastSeenEpochIfNewer(t1, 1); + verify(applicationEventHandler).add(ArgumentMatchers.isA(CommitApplicationEvent.class)); + } + + @Test + public void testEnsurePollExecutedCommitAsyncCallbacks() { + MockCommitCallback callback = new MockCommitCallback(); + CompletableFuture future = new CompletableFuture<>(); + consumer.assign(Collections.singleton(new TopicPartition("foo", 0))); + doReturn(future).when(consumer).commit(new HashMap<>(), false); + assertDoesNotThrow(() -> consumer.commitAsync(new HashMap<>(), callback)); + future.complete(null); + assertMockCommitCallbackInvoked(() -> consumer.poll(Duration.ZERO), + callback, + null); + } + + @Test + public void testEnsureShutdownExecutedCommitAsyncCallbacks() { + MockCommitCallback callback = new MockCommitCallback(); + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(consumer).commit(new HashMap<>(), false); + assertDoesNotThrow(() -> consumer.commitAsync(new HashMap<>(), callback)); + future.complete(null); + assertMockCommitCallbackInvoked(() -> consumer.close(), + callback, + null); + } + + private void assertMockCommitCallbackInvoked(final Executable task, + final MockCommitCallback callback, + final Errors errors) { + assertDoesNotThrow(task); + assertEquals(1, callback.invoked); + if (errors == null) + assertNull(callback.exception); + else if (errors.exception() instanceof RetriableException) + assertTrue(callback.exception instanceof RetriableCommitFailedException); + } + + private static class MockCommitCallback implements OffsetCommitCallback { + public int invoked = 0; + public Exception exception = null; + public String completionThread; + + @Override + public void onComplete(Map offsets, Exception exception) { + invoked++; + this.completionThread = Thread.currentThread().getName(); + this.exception = exception; + } + } + /** + * This is a rather ugly bit of code. Not my choice :( + * + *

    + * + * Inside the {@link org.apache.kafka.clients.consumer.Consumer#committed(Set, Duration)} call we create an + * instance of {@link OffsetFetchApplicationEvent} that holds the partitions and internally holds a + * {@link CompletableFuture}. We want to test different behaviours of the {@link Future#get()}, such as + * returning normally, timing out, throwing an error, etc. By mocking the construction of the event object that + * is created, we can affect that behavior. + */ + private static MockedConstruction offsetFetchEventMocker(CompletableFuture> future) { + // This "answer" is where we pass the future to be invoked by the ConsumerUtils.getResult() method + Answer> getInvocationAnswer = invocation -> { + // This argument captures the actual argument value that was passed to the event's get() method, so we + // just "forward" that value to our mocked call + Timer timer = invocation.getArgument(0); + return ConsumerUtils.getResult(future, timer); + }; + + MockedConstruction.MockInitializer mockInitializer = (mock, ctx) -> { + // When the event's get() method is invoked, we call the "answer" method just above + when(mock.get(any())).thenAnswer(getInvocationAnswer); + + // When the event's type() method is invoked, we have to return the type as it will be null in the mock + when(mock.type()).thenReturn(ApplicationEvent.Type.FETCH_COMMITTED_OFFSET); + + // This is needed for the WakeupTrigger code that keeps track of the active task + when(mock.future()).thenReturn(future); + }; + + return mockConstruction(OffsetFetchApplicationEvent.class, mockInitializer); + } + + private static MockedConstruction commitEventMocker(CompletableFuture future) { + Answer getInvocationAnswer = invocation -> { + Timer timer = invocation.getArgument(0); + return ConsumerUtils.getResult(future, timer); + }; + + MockedConstruction.MockInitializer mockInitializer = (mock, ctx) -> { + when(mock.get(any())).thenAnswer(getInvocationAnswer); + when(mock.type()).thenReturn(ApplicationEvent.Type.COMMIT); + when(mock.future()).thenReturn(future); + }; + + return mockConstruction(CommitApplicationEvent.class, mockInitializer); + } + + @Test + public void testAssign() { + final TopicPartition tp = new TopicPartition("foo", 3); + consumer.assign(singleton(tp)); + assertTrue(consumer.subscription().isEmpty()); + assertTrue(consumer.assignment().contains(tp)); + verify(applicationEventHandler).add(any(AssignmentChangeApplicationEvent.class)); + verify(applicationEventHandler).add(any(NewTopicsMetadataUpdateRequestEvent.class)); + } + + @Test + public void testAssignOnNullTopicPartition() { + assertThrows(IllegalArgumentException.class, () -> consumer.assign(null)); + } + + @Test + public void testAssignOnEmptyTopicPartition() { + consumer.assign(Collections.emptyList()); + assertTrue(consumer.subscription().isEmpty()); + assertTrue(consumer.assignment().isEmpty()); + } + + @Test + public void testAssignOnNullTopicInPartition() { + assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(null, 0)))); + } + + @Test + public void testAssignOnEmptyTopicInPartition() { + assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(" ", 0)))); + } + + @Test + public void testBeginningOffsetsFailsIfNullPartitions() { + assertThrows(NullPointerException.class, () -> consumer.beginningOffsets(null, + Duration.ofMillis(1))); + } + + @Test + public void testBeginningOffsets() { + Map expectedOffsetsAndTimestamp = + mockOffsetAndTimestamp(); + Set partitions = expectedOffsetsAndTimestamp.keySet(); + doReturn(expectedOffsetsAndTimestamp).when(applicationEventHandler).addAndGet(any(), any()); + Map result = + assertDoesNotThrow(() -> consumer.beginningOffsets(partitions, + Duration.ofMillis(1))); + Map expectedOffsets = expectedOffsetsAndTimestamp.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); + assertEquals(expectedOffsets, result); + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + ArgumentMatchers.isA(Timer.class)); + } + + @Test + public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailure() { + Set partitions = mockTopicPartitionOffset().keySet(); + Throwable eventProcessingFailure = new KafkaException("Unexpected failure " + + "processing List Offsets event"); + doThrow(eventProcessingFailure).when(applicationEventHandler).addAndGet(any(), any()); + Throwable consumerError = assertThrows(KafkaException.class, + () -> consumer.beginningOffsets(partitions, + Duration.ofMillis(1))); + assertEquals(eventProcessingFailure, consumerError); + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); + } + + @Test + public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { + doThrow(new TimeoutException()).when(applicationEventHandler).addAndGet(any(), any()); + assertThrows(TimeoutException.class, + () -> consumer.beginningOffsets( + Collections.singletonList(new TopicPartition("t1", 0)), + Duration.ofMillis(1))); + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + ArgumentMatchers.isA(Timer.class)); + } + + @Test + public void testOffsetsForTimesOnNullPartitions() { + assertThrows(NullPointerException.class, () -> consumer.offsetsForTimes(null, + Duration.ofMillis(1))); + } + + @Test + public void testOffsetsForTimesFailsOnNegativeTargetTimes() { + assertThrows(IllegalArgumentException.class, + () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( + "topic1", 1), ListOffsetsRequest.EARLIEST_TIMESTAMP), + Duration.ofMillis(1))); + + assertThrows(IllegalArgumentException.class, + () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( + "topic1", 1), ListOffsetsRequest.LATEST_TIMESTAMP), + Duration.ofMillis(1))); + + assertThrows(IllegalArgumentException.class, + () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( + "topic1", 1), ListOffsetsRequest.MAX_TIMESTAMP), + Duration.ofMillis(1))); + } + + @Test + public void testOffsetsForTimes() { + Map expectedResult = mockOffsetAndTimestamp(); + Map timestampToSearch = mockTimestampToSearch(); + + doReturn(expectedResult).when(applicationEventHandler).addAndGet(any(), any()); + Map result = + assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, Duration.ofMillis(1))); + assertEquals(expectedResult, result); + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + ArgumentMatchers.isA(Timer.class)); + } + + // This test ensures same behaviour as the current consumer when offsetsForTimes is called + // with 0 timeout. It should return map with all requested partitions as keys, with null + // OffsetAndTimestamp as value. + @Test + public void testOffsetsForTimesWithZeroTimeout() { + TopicPartition tp = new TopicPartition("topic1", 0); + Map expectedResult = + Collections.singletonMap(tp, null); + Map timestampToSearch = Collections.singletonMap(tp, 5L); + + Map result = + assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, + Duration.ofMillis(0))); + assertEquals(expectedResult, result); + verify(applicationEventHandler, never()).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + ArgumentMatchers.isA(Timer.class)); + } + + @Test + public void testWakeup_committed() { + consumer.wakeup(); + assertThrows(WakeupException.class, () -> consumer.committed(mockTopicPartitionOffset().keySet())); + assertNoPendingWakeup(consumer.wakeupTrigger()); + } + + @Test + public void testRefreshCommittedOffsetsSuccess() { + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.singletonMap(partition, new OffsetAndMetadata(10L)); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); + } + + @Test + public void testRefreshCommittedOffsetsSuccessButNoCommittedOffsetsFound() { + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.emptyMap(); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); + } + + @Test + public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { + testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(true); + } + + @Test + public void testRefreshCommittedOffsetsNotCalledIfNoGroupId() { + // Create consumer without group id so committed offsets are not used for updating positions + resetWithEmptyGroupId(); + testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(false); + } + + @Test + public void testSubscribeGeneratesEvent() { + String topic = "topic1"; + consumer.subscribe(singletonList(topic)); + assertEquals(singleton(topic), consumer.subscription()); + assertTrue(consumer.assignment().isEmpty()); + verify(applicationEventHandler).add(ArgumentMatchers.isA(SubscriptionChangeApplicationEvent.class)); + } + + @Test + public void testUnsubscribeGeneratesUnsubscribeEvent() { + consumer.unsubscribe(); + + // Verify the unsubscribe event was generated and mock its completion. + final ArgumentCaptor captor = ArgumentCaptor.forClass(UnsubscribeApplicationEvent.class); + verify(applicationEventHandler).add(captor.capture()); + UnsubscribeApplicationEvent unsubscribeApplicationEvent = captor.getValue(); + unsubscribeApplicationEvent.future().complete(null); + + assertTrue(consumer.subscription().isEmpty()); + assertTrue(consumer.assignment().isEmpty()); + } + + @Test + public void testSubscribeToEmptyListActsAsUnsubscribe() { + consumer.subscribe(Collections.emptyList()); + assertTrue(consumer.subscription().isEmpty()); + assertTrue(consumer.assignment().isEmpty()); + verify(applicationEventHandler).add(ArgumentMatchers.isA(UnsubscribeApplicationEvent.class)); + } + + @Test + public void testSubscribeToNullTopicCollection() { + assertThrows(IllegalArgumentException.class, () -> consumer.subscribe((List) null)); + } + + @Test + public void testSubscriptionOnNullTopic() { + assertThrows(IllegalArgumentException.class, () -> consumer.subscribe(singletonList(null))); + } + + @Test + public void testSubscriptionOnEmptyTopic() { + String emptyTopic = " "; + assertThrows(IllegalArgumentException.class, () -> consumer.subscribe(singletonList(emptyTopic))); + } + + @Test + public void testGroupMetadataAfterCreationWithGroupIdIsNull() { + final Properties props = requiredConsumerProperties(); + final ConsumerConfig config = new ConsumerConfig(props); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer())) { + + assertFalse(config.unused().contains(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG)); + assertFalse(config.unused().contains(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED)); + final Throwable exception = assertThrows(InvalidGroupIdException.class, consumer::groupMetadata); + assertEquals( + "To use the group management or offset commit APIs, you must " + + "provide a valid " + ConsumerConfig.GROUP_ID_CONFIG + " in the consumer configuration.", + exception.getMessage() + ); + } + } + + @Test + public void testGroupMetadataAfterCreationWithGroupIdIsNotNull() { + final String groupId = "consumerGroupA"; + final ConsumerConfig config = new ConsumerConfig(requiredConsumerPropertiesAndGroupId(groupId)); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer())) { + + final ConsumerGroupMetadata groupMetadata = consumer.groupMetadata(); + + assertEquals(groupId, groupMetadata.groupId()); + assertEquals(Optional.empty(), groupMetadata.groupInstanceId()); + assertEquals(JoinGroupRequest.UNKNOWN_GENERATION_ID, groupMetadata.generationId()); + assertEquals(JoinGroupRequest.UNKNOWN_MEMBER_ID, groupMetadata.memberId()); + } + } + + @Test + public void testGroupMetadataAfterCreationWithGroupIdIsNotNullAndGroupInstanceIdSet() { + final String groupId = "consumerGroupA"; + final String groupInstanceId = "groupInstanceId1"; + final Properties props = requiredConsumerPropertiesAndGroupId(groupId); + props.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, groupInstanceId); + final ConsumerConfig config = new ConsumerConfig(props); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer())) { + + final ConsumerGroupMetadata groupMetadata = consumer.groupMetadata(); + + assertEquals(groupId, groupMetadata.groupId()); + assertEquals(Optional.of(groupInstanceId), groupMetadata.groupInstanceId()); + assertEquals(JoinGroupRequest.UNKNOWN_GENERATION_ID, groupMetadata.generationId()); + assertEquals(JoinGroupRequest.UNKNOWN_MEMBER_ID, groupMetadata.memberId()); + } + } + + @Test + public void testGroupMetadataUpdateSingleCall() { + final String groupId = "consumerGroupA"; + final ConsumerConfig config = new ConsumerConfig(requiredConsumerPropertiesAndGroupId(groupId)); + final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer(), backgroundEventQueue)) { + final int generation = 1; + final String memberId = "newMemberId"; + final ConsumerGroupMetadata expectedGroupMetadata = new ConsumerGroupMetadata( + groupId, + generation, + memberId, + Optional.empty() + ); + final GroupMetadataUpdateEvent groupMetadataUpdateEvent = new GroupMetadataUpdateEvent( + generation, + memberId + ); + backgroundEventQueue.add(groupMetadataUpdateEvent); + consumer.assign(singletonList(new TopicPartition("topic", 0))); + consumer.poll(Duration.ZERO); + + final ConsumerGroupMetadata actualGroupMetadata = consumer.groupMetadata(); + + assertEquals(expectedGroupMetadata, actualGroupMetadata); + + final ConsumerGroupMetadata secondActualGroupMetadataWithoutUpdate = consumer.groupMetadata(); + + assertEquals(expectedGroupMetadata, secondActualGroupMetadataWithoutUpdate); + } + } + + @Test + public void testBackgroundError() { + final String groupId = "consumerGroupA"; + final ConsumerConfig config = new ConsumerConfig(requiredConsumerPropertiesAndGroupId(groupId)); + final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer(), backgroundEventQueue)) { + final KafkaException expectedException = new KafkaException("Nobody expects the Spanish Inquisition"); + final ErrorBackgroundEvent errorBackgroundEvent = new ErrorBackgroundEvent(expectedException); + backgroundEventQueue.add(errorBackgroundEvent); + consumer.assign(singletonList(new TopicPartition("topic", 0))); + + final KafkaException exception = assertThrows(KafkaException.class, () -> consumer.poll(Duration.ZERO)); + + assertEquals(expectedException.getMessage(), exception.getMessage()); + } + } + + @Test + public void testMultipleBackgroundErrors() { + final String groupId = "consumerGroupA"; + final ConsumerConfig config = new ConsumerConfig(requiredConsumerPropertiesAndGroupId(groupId)); + final LinkedBlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer(), backgroundEventQueue)) { + final KafkaException expectedException1 = new KafkaException("Nobody expects the Spanish Inquisition"); + final ErrorBackgroundEvent errorBackgroundEvent1 = new ErrorBackgroundEvent(expectedException1); + backgroundEventQueue.add(errorBackgroundEvent1); + final KafkaException expectedException2 = new KafkaException("Spam, Spam, Spam"); + final ErrorBackgroundEvent errorBackgroundEvent2 = new ErrorBackgroundEvent(expectedException2); + backgroundEventQueue.add(errorBackgroundEvent2); + consumer.assign(singletonList(new TopicPartition("topic", 0))); + + final KafkaException exception = assertThrows(KafkaException.class, () -> consumer.poll(Duration.ZERO)); + + assertEquals(expectedException1.getMessage(), exception.getMessage()); + assertTrue(backgroundEventQueue.isEmpty()); + } + } + + @Test + public void testGroupIdNull() { + final Properties props = requiredConsumerProperties(); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 10000); + props.put(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED, true); + final ConsumerConfig config = new ConsumerConfig(props); + + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer())) { + assertFalse(config.unused().contains(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG)); + assertFalse(config.unused().contains(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED)); + } catch (final Exception exception) { + throw new AssertionFailedError("The following exception was not expected:", exception); + } + } + + @Test + public void testGroupIdNotNullAndValid() { + final Properties props = requiredConsumerPropertiesAndGroupId("consumerGroupA"); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 10000); + props.put(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED, true); + final ConsumerConfig config = new ConsumerConfig(props); + + try (final AsyncKafkaConsumer consumer = + new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer())) { + assertTrue(config.unused().contains(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG)); + assertTrue(config.unused().contains(THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED)); + } catch (final Exception exception) { + throw new AssertionFailedError("The following exception was not expected:", exception); + } + } + + @Test + public void testGroupIdEmpty() { + testInvalidGroupId(""); + } + + @Test + public void testGroupIdOnlyWhitespaces() { + testInvalidGroupId(" "); + } + + private void testInvalidGroupId(final String groupId) { + final Properties props = requiredConsumerPropertiesAndGroupId(groupId); + final ConsumerConfig config = new ConsumerConfig(props); + + final Exception exception = assertThrows( + KafkaException.class, + () -> new AsyncKafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer()) + ); + + assertEquals("Failed to construct kafka consumer", exception.getMessage()); + } + + private Properties requiredConsumerPropertiesAndGroupId(final String groupId) { + final Properties props = requiredConsumerProperties(); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + return props; + } + + private Properties requiredConsumerProperties() { + final Properties props = new Properties(); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9091"); + return props; + } + + private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean committedOffsetsEnabled) { + // Uncompleted future that will time out if used + CompletableFuture> committedFuture = new CompletableFuture<>(); + + consumer.assign(singleton(new TopicPartition("t1", 1))); + + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + // Poll with 0 timeout to run a single iteration of the poll loop + consumer.poll(Duration.ofMillis(0)); + + verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); + + if (committedOffsetsEnabled) { + // Verify there was an OffsetFetch event and no ResetPositions event + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler, + never()).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); + } else { + // Verify there was not any OffsetFetch event but there should be a ResetPositions + verify(applicationEventHandler, + never()).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); + } + } + } + + private void testRefreshCommittedOffsetsSuccess(Set partitions, + Map committedOffsets) { + CompletableFuture> committedFuture = new CompletableFuture<>(); + committedFuture.complete(committedOffsets); + consumer.assign(partitions); + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + // Poll with 0 timeout to run a single iteration of the poll loop + consumer.poll(Duration.ofMillis(0)); + + verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); + } + } + + @Test + public void testLongPollWaitIsLimited() { + String topicName = "topic1"; + consumer.subscribe(singletonList(topicName)); + + assertEquals(singleton(topicName), consumer.subscription()); + assertTrue(consumer.assignment().isEmpty()); + + final int partition = 3; + final TopicPartition tp = new TopicPartition(topicName, partition); + final List> records = asList( + new ConsumerRecord<>(topicName, partition, 2, "key1", "value1"), + new ConsumerRecord<>(topicName, partition, 3, "key2", "value2") + ); + + // On the first iteration, return no data; on the second, return two records + doAnswer(invocation -> { + // Mock the subscription being assigned as the first fetch is collected + subscriptions.assignFromSubscribed(Collections.singleton(tp)); + return Fetch.empty(); + }).doAnswer(invocation -> { + return Fetch.forPartition(tp, records, true); + }).when(fetchCollector).collectFetch(any(FetchBuffer.class)); + + // And then poll for up to 10000ms, which should return 2 records without timing out + ConsumerRecords returnedRecords = consumer.poll(Duration.ofMillis(10000)); + assertEquals(2, returnedRecords.count()); + + assertEquals(singleton(topicName), consumer.subscription()); + assertEquals(singleton(tp), consumer.assignment()); + } + + private void assertNoPendingWakeup(final WakeupTrigger wakeupTrigger) { + assertNull(wakeupTrigger.getPendingTask()); + } + + private HashMap mockTopicPartitionOffset() { + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + HashMap topicPartitionOffsets = new HashMap<>(); + topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L)); + topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L)); + return topicPartitionOffsets; + } + + private HashMap mockOffsetAndTimestamp() { + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + HashMap offsetAndTimestamp = new HashMap<>(); + offsetAndTimestamp.put(t0, new OffsetAndTimestamp(5L, 1L)); + offsetAndTimestamp.put(t1, new OffsetAndTimestamp(6L, 3L)); + return offsetAndTimestamp; + } + + private HashMap mockTimestampToSearch() { + final TopicPartition t0 = new TopicPartition("t0", 2); + final TopicPartition t1 = new TopicPartition("t0", 3); + HashMap timestampToSearch = new HashMap<>(); + timestampToSearch.put(t0, 1L); + timestampToSearch.put(t1, 2L); + return timestampToSearch; + } + + private void prepAutocommitOnClose() { + Node node = testBuilder.metadata.fetch().nodes().get(0); + testBuilder.client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "group-id", node)); + if (!testBuilder.subscriptions.allConsumed().isEmpty()) { + List topicPartitions = new ArrayList<>(testBuilder.subscriptions.assignedPartitionsList()); + testBuilder.client.prepareResponse(mockAutocommitResponse( + topicPartitions, + (short) 1, + Errors.NONE).responseBody()); + } + } + + private ClientResponse mockAutocommitResponse(final List topicPartitions, + final short apiKeyVersion, + final Errors error) { + OffsetCommitResponseData responseData = new OffsetCommitResponseData(); + List responseTopics = new ArrayList<>(); + topicPartitions.forEach(tp -> { + responseTopics.add(new OffsetCommitResponseData.OffsetCommitResponseTopic() + .setName(tp.topic()) + .setPartitions(Collections.singletonList( + new OffsetCommitResponseData.OffsetCommitResponsePartition() + .setErrorCode(error.code()) + .setPartitionIndex(tp.partition())))); + }); + responseData.setTopics(responseTopics); + OffsetCommitResponse response = mock(OffsetCommitResponse.class); + when(response.data()).thenReturn(responseData); + return new ClientResponse( + new RequestHeader(ApiKeys.OFFSET_COMMIT, apiKeyVersion, "", 1), + null, + "-1", + testBuilder.time.milliseconds(), + testBuilder.time.milliseconds(), + false, + null, + null, + new OffsetCommitResponse(responseData) + ); + } +} + diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index 4751cdea09d12..74d69d8d42aeb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -19,6 +19,8 @@ import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.DisconnectException; @@ -35,6 +37,7 @@ import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -52,11 +55,14 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Collections.singleton; import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID; @@ -81,15 +87,16 @@ public class CommitRequestManagerTest { private MockTime time; private CoordinatorRequestManager coordinatorRequestManager; private Properties props; + private BackgroundEventHandler backgroundEventHandler; @BeforeEach public void setup() { this.logContext = new LogContext(); this.time = new MockTime(0); - this.subscriptionState = mock(SubscriptionState.class); + this.subscriptionState = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST); this.coordinatorRequestManager = mock(CoordinatorRequestManager.class); this.groupState = new GroupState(DEFAULT_GROUP_ID, Optional.empty()); - + this.backgroundEventHandler = spy(new BackgroundEventHandler(logContext, new LinkedBlockingQueue<>())); this.props = new Properties(); this.props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); this.props.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); @@ -120,16 +127,23 @@ public void testPoll_EnsureManualCommitSent() { @Test public void testPoll_EnsureAutocommitSent() { + TopicPartition tp = new TopicPartition("t1", 1); + subscriptionState.assignFromUser(Collections.singleton(tp)); + subscriptionState.seek(tp, 100); CommitRequestManager commitRequestManger = create(true, 100); assertPoll(0, commitRequestManger); Map offsets = new HashMap<>(); - offsets.put(new TopicPartition("t1", 0), new OffsetAndMetadata(0)); + offsets.put(tp, new OffsetAndMetadata(0)); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); - when(subscriptionState.allConsumed()).thenReturn(offsets); time.sleep(100); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); - assertPoll(1, commitRequestManger); + List pollResults = assertPoll(1, commitRequestManger); + pollResults.forEach(v -> v.onComplete(mockOffsetCommitResponse( + "t1", + 1, + (short) 1, + Errors.NONE))); } @Test @@ -183,40 +197,59 @@ public void testPoll_EnsureEmptyPendingRequestAfterPoll() { @Test public void testAutocommit_ResendAutocommitAfterException() { CommitRequestManager commitRequestManger = create(true, 100); + TopicPartition tp = new TopicPartition("topic", 1); + subscriptionState.assignFromUser(Collections.singleton(tp)); + subscriptionState.seek(tp, 100); time.sleep(100); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); - List> futures = assertPoll(1, commitRequestManger); + List futures = assertPoll(1, commitRequestManger); time.sleep(99); - // complete the autocommit request (exceptionally) - futures.get(0).complete(mockOffsetCommitResponse( + // complete the autocommit request (exceptionally), and expect an exponential backoff of 100ms + futures.get(0).onComplete(mockOffsetCommitResponse( "topic", 1, (short) 1, Errors.COORDINATOR_LOAD_IN_PROGRESS)); - // we can then autocommit again + // Expecting to wait for 100ms but only waited 99ms. No result is expected here. + time.sleep(99); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); assertPoll(0, commitRequestManger); - time.sleep(1); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); - assertPoll(1, commitRequestManger); + + // Complete the full backoff of 100ms, and now expecting a result. + time.sleep(1); + futures = assertPoll(1, commitRequestManger); assertEmptyPendingRequests(commitRequestManger); + futures.get(0).onComplete(mockOffsetCommitResponse( + "topic", + 1, + (short) 1, + Errors.NONE)); } @Test public void testAutocommit_EnsureOnlyOneInflightRequest() { + TopicPartition t1p = new TopicPartition("topic1", 0); + subscriptionState.assignFromUser(singleton(t1p)); + CommitRequestManager commitRequestManger = create(true, 100); time.sleep(100); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); - List> futures = assertPoll(1, commitRequestManger); - time.sleep(100); + // Nothing consumed therefore no commit request is sent + assertPoll(0, commitRequestManger); + time.sleep(10); + subscriptionState.seekUnvalidated(t1p, new SubscriptionState.FetchPosition(100L)); + List futures = assertPoll(1, commitRequestManger); + + time.sleep(90); commitRequestManger.updateAutoCommitTimer(time.milliseconds()); // We want to make sure we don't resend autocommit if the previous request has not been completed assertPoll(0, commitRequestManger); assertEmptyPendingRequests(commitRequestManger); // complete the unsent request and re-poll - futures.get(0).complete(buildOffsetCommitClientResponse(new OffsetCommitResponse(0, new HashMap<>()), Errors.NONE)); + futures.get(0).onComplete(buildOffsetCommitClientResponse(new OffsetCommitResponse(0, new HashMap<>()), Errors.NONE)); assertPoll(1, commitRequestManger); } @@ -351,7 +384,8 @@ private void testRetriable(final CommitRequestManager commitRequestManger, final List>> futures) { futures.forEach(f -> assertFalse(f.isDone())); - time.sleep(500); + // The manager should backoff for 100ms + time.sleep(100); commitRequestManger.poll(time.milliseconds()); futures.forEach(f -> assertFalse(f.isDone())); } @@ -371,7 +405,6 @@ private static Stream offsetCommitExceptionSupplier() { Arguments.of(Errors.INVALID_COMMIT_OFFSET_SIZE, false), Arguments.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, true), Arguments.of(Errors.COORDINATOR_NOT_AVAILABLE, true), - Arguments.of(Errors.NOT_COORDINATOR, true), Arguments.of(Errors.REQUEST_TIMED_OUT, true), Arguments.of(Errors.FENCED_INSTANCE_ID, false), Arguments.of(Errors.TOPIC_AUTHORIZATION_FAILED, false)); @@ -389,7 +422,6 @@ private static Stream offsetFetchExceptionSupplier() { Arguments.of(Errors.INVALID_COMMIT_OFFSET_SIZE, false), Arguments.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, false), Arguments.of(Errors.COORDINATOR_NOT_AVAILABLE, false), - Arguments.of(Errors.NOT_COORDINATOR, true), Arguments.of(Errors.REQUEST_TIMED_OUT, false), Arguments.of(Errors.FENCED_INSTANCE_ID, false), Arguments.of(Errors.TOPIC_AUTHORIZATION_FAILED, false)); @@ -473,13 +505,13 @@ private void sendAndVerifyOffsetCommitRequests( assertEquals(0, res.unsentRequests.size()); } - private List> assertPoll( + private List assertPoll( final int numRes, final CommitRequestManager manager) { return assertPoll(true, numRes, manager); } - private List> assertPoll( + private List assertPoll( final boolean coordinatorDiscovered, final int numRes, final CommitRequestManager manager) { @@ -491,22 +523,27 @@ private List> assertPoll( NetworkClientDelegate.PollResult res = manager.poll(time.milliseconds()); assertEquals(numRes, res.unsentRequests.size()); - return res.unsentRequests.stream().map(NetworkClientDelegate.UnsentRequest::future).collect(Collectors.toList()); + return res.unsentRequests.stream().map(NetworkClientDelegate.UnsentRequest::handler).collect(Collectors.toList()); } private CommitRequestManager create(final boolean autoCommitEnabled, final long autoCommitInterval) { props.setProperty(AUTO_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(autoCommitInterval)); props.setProperty(ENABLE_AUTO_COMMIT_CONFIG, String.valueOf(autoCommitEnabled)); + + if (autoCommitEnabled) + props.setProperty(GROUP_ID_CONFIG, TestUtils.randomString(10)); + return spy(new CommitRequestManager( - this.time, - this.logContext, - this.subscriptionState, - new ConsumerConfig(props), - this.coordinatorRequestManager, - this.groupState, - retryBackoffMs, - retryBackoffMaxMs, - 0)); + this.time, + this.logContext, + this.subscriptionState, + new ConsumerConfig(props), + this.coordinatorRequestManager, + this.backgroundEventHandler, + this.groupState, + retryBackoffMs, + retryBackoffMaxMs, + 0)); } private ClientResponse buildOffsetFetchClientResponse( diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 40995b7f4f444..ba0d3bacef4d9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -3568,8 +3568,6 @@ public void testPrepareJoinAndRejoinAfterFailedRebalance() { assertFalse(client.hasPendingResponses()); assertEquals(1, client.inFlightRequestCount()); - System.out.println(client.requests()); - // Retry join should then succeed client.respond(joinGroupFollowerResponse(generationId, memberId, "leader", Errors.NONE)); client.prepareResponse(syncGroupResponse(partitions, Errors.NONE)); @@ -3702,7 +3700,8 @@ private void supportStableFlag(final short upperVersion, final boolean expectThr autoCommitIntervalMs, null, true, - null); + null, + Optional.empty()); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.OFFSET_FETCH.id, (short) 0, upperVersion)); @@ -3865,7 +3864,8 @@ private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanc autoCommitIntervalMs, null, false, - null); + null, + Optional.empty()); } private Collection getRevoked(final List owned, @@ -4091,7 +4091,7 @@ private void createRackAwareCoordinator(String rackId, MockPartitionAssignor ass coordinator = new ConsumerCoordinator(rebalanceConfig, new LogContext(), consumerClient, Collections.singletonList(assignor), metadata, subscriptions, - metrics, consumerId + groupId, time, false, autoCommitIntervalMs, null, false, rackId); + metrics, consumerId + groupId, time, false, autoCommitIntervalMs, null, false, rackId, Optional.empty()); } private static MetadataResponse rackAwareMetadata(int numNodes, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index 812999b3ca340..a1370918e4dcb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -28,10 +28,15 @@ import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; +import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FindCoordinatorRequestData; +import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.FindCoordinatorResponse; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetCommitRequest; +import org.apache.kafka.common.requests.OffsetCommitResponse; import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.utils.Time; import org.apache.kafka.test.TestCondition; @@ -42,6 +47,7 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -49,6 +55,9 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_HEARTBEAT_INTERVAL_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REQUEST_TIMEOUT_MS; import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -74,9 +83,11 @@ public class ConsumerNetworkThreadTest { private BlockingQueue applicationEventsQueue; private ApplicationEventProcessor applicationEventProcessor; private OffsetsRequestManager offsetsRequestManager; - private CommitRequestManager commitManager; + private CommitRequestManager commitRequestManager; + private CoordinatorRequestManager coordinatorRequestManager; private ConsumerNetworkThread consumerNetworkThread; private MockClient client; + private SubscriptionState subscriptions; @BeforeEach public void setup() { @@ -87,9 +98,11 @@ public void setup() { client = testBuilder.client; applicationEventsQueue = testBuilder.applicationEventQueue; applicationEventProcessor = testBuilder.applicationEventProcessor; - commitManager = testBuilder.commitRequestManager.orElseThrow(IllegalStateException::new); + commitRequestManager = testBuilder.commitRequestManager.orElseThrow(IllegalStateException::new); offsetsRequestManager = testBuilder.offsetsRequestManager; + coordinatorRequestManager = testBuilder.coordinatorRequestManager.orElseThrow(IllegalStateException::new); consumerNetworkThread = testBuilder.consumerNetworkThread; + subscriptions = testBuilder.subscriptions; consumerNetworkThread.initializeResources(); } @@ -104,7 +117,6 @@ public void testStartupAndTearDown() throws InterruptedException { // The consumer is closed in ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder.close() // which is called from tearDown(). consumerNetworkThread.start(); - TestCondition isStarted = () -> consumerNetworkThread.isRunning(); TestCondition isClosed = () -> !(consumerNetworkThread.isRunning() || consumerNetworkThread.isAlive()); @@ -113,6 +125,7 @@ public void testStartupAndTearDown() throws InterruptedException { TestUtils.waitForCondition(isStarted, "The consumer network thread did not start within " + DEFAULT_MAX_WAIT_MS + " ms"); + prepareTearDown(); consumerNetworkThread.close(Duration.ofMillis(DEFAULT_MAX_WAIT_MS)); TestUtils.waitForCondition(isClosed, @@ -193,8 +206,8 @@ public void testAssignmentChangeEvent() { consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(AssignmentChangeApplicationEvent.class)); verify(networkClient, times(1)).poll(anyLong(), anyLong()); - verify(commitManager, times(1)).updateAutoCommitTimer(currentTimeMs); - verify(commitManager, times(1)).maybeAutoCommit(offset); + verify(commitRequestManager, times(1)).updateAutoCommitTimer(currentTimeMs); + verify(commitRequestManager, times(1)).maybeAutoCommit(offset); } @Test @@ -226,10 +239,20 @@ void testPollResultTimer() { assertEquals(10, networkClient.addAll(failure)); } + @Test + void testMaximumTimeToWait() { + // Initial value before runOnce has been called + assertEquals(ConsumerNetworkThread.MAX_POLL_TIMEOUT_MS, consumerNetworkThread.maximumTimeToWait()); + consumerNetworkThread.runOnce(); + // After runOnce has been called, it takes the default heartbeat interval from the heartbeat request manager + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, consumerNetworkThread.maximumTimeToWait()); + } + @Test void testRequestManagersArePolledOnce() { consumerNetworkThread.runOnce(); testBuilder.requestManagers.entries().forEach(rmo -> rmo.ifPresent(rm -> verify(rm, times(1)).poll(anyLong()))); + testBuilder.requestManagers.entries().forEach(rmo -> rmo.ifPresent(rm -> verify(rm, times(1)).maximumTimeToWait(anyLong()))); verify(networkClient, times(1)).poll(anyLong(), anyLong()); } @@ -244,6 +267,10 @@ void testEnsureMetadataUpdateOnPoll() { @Test void testEnsureEventsAreCompleted() { + Node node = metadata.fetch().nodes().get(0); + coordinatorRequestManager.markCoordinatorUnknown("test", time.milliseconds()); + client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "group-id", node)); + prepareOffsetCommitRequest(new HashMap<>(), Errors.NONE, false); CompletableApplicationEvent event1 = spy(new CommitApplicationEvent(Collections.emptyMap())); ApplicationEvent event2 = new CommitApplicationEvent(Collections.emptyMap()); CompletableFuture future = new CompletableFuture<>(); @@ -258,6 +285,85 @@ void testEnsureEventsAreCompleted() { assertTrue(applicationEventsQueue.isEmpty()); } + @Test + void testCoordinatorConnectionOnClose() { + TopicPartition tp = new TopicPartition("topic", 0); + subscriptions.assignFromUser(singleton(new TopicPartition("topic", 0))); + subscriptions.seekUnvalidated(tp, new SubscriptionState.FetchPosition(100)); + Node node = metadata.fetch().nodes().get(0); + coordinatorRequestManager.markCoordinatorUnknown("test", time.milliseconds()); + client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "group-id", node)); + prepareOffsetCommitRequest(singletonMap(tp, 100L), Errors.NONE, false); + consumerNetworkThread.cleanup(); + assertTrue(coordinatorRequestManager.coordinator().isPresent()); + assertFalse(client.hasPendingResponses()); + assertFalse(client.hasInFlightRequests()); + } + + @Test + void testAutoCommitOnClose() { + TopicPartition tp = new TopicPartition("topic", 0); + Node node = metadata.fetch().nodes().get(0); + subscriptions.assignFromUser(singleton(tp)); + subscriptions.seek(tp, 100); + coordinatorRequestManager.markCoordinatorUnknown("test", time.milliseconds()); + client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "group-id", node)); + prepareOffsetCommitRequest(singletonMap(tp, 100L), Errors.NONE, false); + consumerNetworkThread.maybeAutocommitOnClose(time.timer(1000)); + assertTrue(coordinatorRequestManager.coordinator().isPresent()); + verify(commitRequestManager).createCommitAllConsumedRequest(); + + assertFalse(client.hasPendingResponses()); + assertFalse(client.hasInFlightRequests()); + } + + private void prepareTearDown() { + Node node = metadata.fetch().nodes().get(0); + client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "group-id", node)); + prepareOffsetCommitRequest(new HashMap<>(), Errors.NONE, false); + } + + private void prepareOffsetCommitRequest(final Map expectedOffsets, + final Errors error, + final boolean disconnected) { + Map errors = partitionErrors(expectedOffsets.keySet(), error); + client.prepareResponse(offsetCommitRequestMatcher(expectedOffsets), offsetCommitResponse(errors), disconnected); + } + + private Map partitionErrors(final Collection partitions, + final Errors error) { + final Map errors = new HashMap<>(); + for (TopicPartition partition : partitions) { + errors.put(partition, error); + } + return errors; + } + + private OffsetCommitResponse offsetCommitResponse(final Map responseData) { + return new OffsetCommitResponse(responseData); + } + + private MockClient.RequestMatcher offsetCommitRequestMatcher(final Map expectedOffsets) { + return body -> { + OffsetCommitRequest req = (OffsetCommitRequest) body; + Map offsets = req.offsets(); + if (offsets.size() != expectedOffsets.size()) + return false; + + for (Map.Entry expectedOffset : expectedOffsets.entrySet()) { + if (!offsets.containsKey(expectedOffset.getKey())) { + return false; + } else { + Long actualOffset = offsets.get(expectedOffset.getKey()); + if (!actualOffset.equals(expectedOffset.getValue())) { + return false; + } + } + } + return true; + }; + } + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index b36f1958f7d02..53917f6ff178f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -27,7 +27,6 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.MetadataResponse; @@ -38,6 +37,7 @@ import org.apache.kafka.common.utils.Time; import java.io.Closeable; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -69,7 +69,7 @@ public class ConsumerTestBuilder implements Closeable { static final double DEFAULT_HEARTBEAT_JITTER_MS = 0.0; final LogContext logContext = new LogContext(); - final Time time = new MockTime(0); + final Time time; public final BlockingQueue applicationEventQueue; public final BlockingQueue backgroundEventQueue; final ConsumerConfig config; @@ -86,12 +86,12 @@ public class ConsumerTestBuilder implements Closeable { final Optional commitRequestManager; final Optional heartbeatRequestManager; final Optional membershipManager; + final Optional heartbeatState; final Optional heartbeatRequestState; final TopicMetadataRequestManager topicMetadataRequestManager; final FetchRequestManager fetchRequestManager; final RequestManagers requestManagers; public final ApplicationEventProcessor applicationEventProcessor; - public final BackgroundEventProcessor backgroundEventProcessor; public final BackgroundEventHandler backgroundEventHandler; final MockClient client; final Optional groupInfo; @@ -101,7 +101,12 @@ public ConsumerTestBuilder() { } public ConsumerTestBuilder(Optional groupInfo) { + this(groupInfo, true, true); + } + + public ConsumerTestBuilder(Optional groupInfo, boolean enableAutoCommit, boolean enableAutoTick) { this.groupInfo = groupInfo; + this.time = enableAutoTick ? new MockTime(1) : new MockTime(); this.applicationEventQueue = new LinkedBlockingQueue<>(); this.backgroundEventQueue = new LinkedBlockingQueue<>(); this.backgroundEventHandler = spy(new BackgroundEventHandler(logContext, backgroundEventQueue)); @@ -124,6 +129,9 @@ public ConsumerTestBuilder(Optional groupInfo) { properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_REQUEST_TIMEOUT_MS); properties.put(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG, DEFAULT_MAX_POLL_INTERVAL_MS); + if (!enableAutoCommit) + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + groupInfo.ifPresent(gi -> { properties.put(GROUP_ID_CONFIG, gi.groupState.groupId); gi.groupState.groupInstanceId.ifPresent(groupInstanceId -> properties.put(GROUP_INSTANCE_ID_CONFIG, groupInstanceId)); @@ -166,6 +174,8 @@ public ConsumerTestBuilder(Optional groupInfo) { backgroundEventHandler, logContext)); + this.topicMetadataRequestManager = spy(new TopicMetadataRequestManager(logContext, config)); + if (groupInfo.isPresent()) { GroupInformation gi = groupInfo.get(); CoordinatorRequestManager coordinator = spy(new CoordinatorRequestManager( @@ -181,16 +191,25 @@ public ConsumerTestBuilder(Optional groupInfo) { subscriptions, config, coordinator, + backgroundEventHandler, groupState)); MembershipManager mm = spy( new MembershipManagerImpl( gi.groupState.groupId, - gi.groupState.groupInstanceId.orElse(null), - null, + gi.groupState.groupInstanceId, + Optional.empty(), + subscriptions, + commit, + metadata, logContext ) ); - HeartbeatRequestManager.HeartbeatRequestState state = spy(new HeartbeatRequestManager.HeartbeatRequestState(logContext, + HeartbeatRequestManager.HeartbeatState heartbeatState = spy(new HeartbeatRequestManager.HeartbeatState( + subscriptions, + mm, + DEFAULT_MAX_POLL_INTERVAL_MS)); + HeartbeatRequestManager.HeartbeatRequestState heartbeatRequestState = spy(new HeartbeatRequestManager.HeartbeatRequestState( + logContext, time, gi.heartbeatIntervalMs, retryBackoffMs, @@ -198,23 +217,24 @@ public ConsumerTestBuilder(Optional groupInfo) { gi.heartbeatJitterMs)); HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( logContext, - time, config, coordinator, - subscriptions, mm, - state, + heartbeatState, + heartbeatRequestState, backgroundEventHandler)); this.coordinatorRequestManager = Optional.of(coordinator); this.commitRequestManager = Optional.of(commit); this.heartbeatRequestManager = Optional.of(heartbeat); - this.heartbeatRequestState = Optional.of(state); + this.heartbeatState = Optional.of(heartbeatState); + this.heartbeatRequestState = Optional.of(heartbeatRequestState); this.membershipManager = Optional.of(mm); } else { this.coordinatorRequestManager = Optional.empty(); this.commitRequestManager = Optional.empty(); this.heartbeatRequestManager = Optional.empty(); + this.heartbeatState = Optional.empty(); this.heartbeatRequestState = Optional.empty(); this.membershipManager = Optional.empty(); } @@ -229,29 +249,26 @@ public ConsumerTestBuilder(Optional groupInfo) { metricsManager, networkClientDelegate, apiVersions)); - this.topicMetadataRequestManager = spy(new TopicMetadataRequestManager(logContext, - config)); this.requestManagers = new RequestManagers(logContext, offsetsRequestManager, topicMetadataRequestManager, fetchRequestManager, coordinatorRequestManager, commitRequestManager, - heartbeatRequestManager); + heartbeatRequestManager, + membershipManager); this.applicationEventProcessor = spy(new ApplicationEventProcessor( logContext, applicationEventQueue, requestManagers, metadata) ); - this.backgroundEventProcessor = spy(new BackgroundEventProcessor(logContext, backgroundEventQueue)); } @Override public void close() { closeQuietly(requestManagers, RequestManagers.class.getSimpleName()); closeQuietly(applicationEventProcessor, ApplicationEventProcessor.class.getSimpleName()); - closeQuietly(backgroundEventProcessor, BackgroundEventProcessor.class.getSimpleName()); } public static class ConsumerNetworkThreadTestBuilder extends ConsumerTestBuilder { @@ -275,7 +292,7 @@ public ConsumerNetworkThreadTestBuilder(Optional groupInfo) { @Override public void close() { - closeQuietly(consumerNetworkThread, ConsumerNetworkThread.class.getSimpleName()); + consumerNetworkThread.close(); } } @@ -283,12 +300,8 @@ public static class ApplicationEventHandlerTestBuilder extends ConsumerTestBuild public final ApplicationEventHandler applicationEventHandler; - public ApplicationEventHandlerTestBuilder() { - this(createDefaultGroupInformation()); - } - - public ApplicationEventHandlerTestBuilder(Optional groupInfo) { - super(groupInfo); + public ApplicationEventHandlerTestBuilder(Optional groupInfo, boolean enableAutoCommit, boolean enableAutoTick) { + super(groupInfo, enableAutoCommit, enableAutoTick); this.applicationEventHandler = spy(new ApplicationEventHandler( logContext, time, @@ -304,26 +317,28 @@ public void close() { } } - public static class PrototypeAsyncConsumerTestBuilder extends ApplicationEventHandlerTestBuilder { + public static class AsyncKafkaConsumerTestBuilder extends ApplicationEventHandlerTestBuilder { - final PrototypeAsyncConsumer consumer; + final AsyncKafkaConsumer consumer; - public PrototypeAsyncConsumerTestBuilder(Optional groupInfo) { - super(groupInfo); + final FetchCollector fetchCollector; + + public AsyncKafkaConsumerTestBuilder(Optional groupInfo, boolean enableAutoCommit, boolean enableAutoTick) { + super(groupInfo, enableAutoCommit, enableAutoTick); String clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); List assignors = ConsumerPartitionAssignor.getAssignorInstances( config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) ); Deserializers deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); - FetchCollector fetchCollector = new FetchCollector<>(logContext, + this.fetchCollector = spy(new FetchCollector<>(logContext, metadata, subscriptions, fetchConfig, deserializers, metricsManager, - time); - this.consumer = spy(new PrototypeAsyncConsumer<>( + time)); + this.consumer = spy(new AsyncKafkaConsumer<>( logContext, clientId, deserializers, @@ -346,6 +361,10 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupInfo) { public void close() { consumer.close(); } + + public void close(final Duration timeout) { + consumer.close(timeout); + } } public static class GroupInformation { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchBufferTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchBufferTest.java index 319f009f4d360..967a5bb2aab8b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchBufferTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchBufferTest.java @@ -26,9 +26,11 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.Arrays; import java.util.HashSet; import java.util.Properties; @@ -171,6 +173,20 @@ public void testAddAllAndRetainAll() { } } + @Test + public void testWakeup() throws Exception { + try (FetchBuffer fetchBuffer = new FetchBuffer(logContext)) { + final Thread waitingThread = new Thread(() -> { + final Timer timer = time.timer(Duration.ofMinutes(1)); + fetchBuffer.awaitNotEmpty(timer); + }); + waitingThread.start(); + fetchBuffer.wakeup(); + waitingThread.join(Duration.ofSeconds(30).toMillis()); + assertFalse(waitingThread.isAlive()); + } + } + private CompletedFetch completedFetch(TopicPartition tp) { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData(); FetchMetricsAggregator metricsAggregator = new FetchMetricsAggregator(metricsManager, allPartitions); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 978b0b6345e93..e6f5689a001ce 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -189,6 +189,7 @@ public class FetcherTest { private MemoryRecords records; private MemoryRecords nextRecords; + private MemoryRecords moreRecords; private MemoryRecords emptyRecords; private MemoryRecords partialRecords; private ExecutorService executorService; @@ -197,6 +198,7 @@ public class FetcherTest { public void setup() { records = buildRecords(1L, 3, 1); nextRecords = buildRecords(4L, 2, 4); + moreRecords = buildRecords(6L, 3, 6); emptyRecords = buildRecords(0L, 0, 0); partialRecords = buildRecords(4L, 1, 0); partialRecords.buffer().putInt(Records.SIZE_OFFSET, 10000); @@ -1124,6 +1126,66 @@ public void testFetchMaxPollRecords() { assertEquals(5, recordsToTest.get(1).offset()); } + /** + * KAFKA-15836: + * Test that max.poll.records is honoured when consuming from multiple topic-partitions and the + * fetched records are not aligned on max.poll.records boundaries. + * + * tp0 has records 1,2,3; tp1 has records 6,7,8 + * max.poll.records is 2 + * + * poll 1 should return 1,2 + * poll 2 should return 3,6 + * poll 3 should return 7,8 + * + * Or it can be 6,7; then 8,1; then 2,3 because the order of topic-partitions returned is non-deterministic. + */ + @Test + public void testFetchMaxPollRecordsUnaligned() { + final int maxPollRecords = 2; + buildFetcher(maxPollRecords); + + Set tps = new HashSet<>(); + tps.add(tp0); + tps.add(tp1); + assignFromUser(tps); + subscriptions.seek(tp0, 1); + subscriptions.seek(tp1, 6); + + client.prepareResponse(fetchResponse2(tidp0, records, 100L, tidp1, moreRecords, 100L)); + client.prepareResponse(fullFetchResponse(tidp0, emptyRecords, Errors.NONE, 100L, 0)); + + // Send fetch request because we do not have pending fetch responses to process. + // The first fetch response will return 3 records for tp0 and 3 more for tp1. + assertEquals(1, sendFetches()); + // The poll returns 2 records from one of the topic-partitions (non-deterministic). + // This leaves 1 record pending from that topic-partition, and the remaining 3 from the other. + pollAndValidateMaxPollRecordsNotExceeded(maxPollRecords); + + // See if we need to send another fetch, which we do not because we have records in hand. + assertEquals(0, sendFetches()); + // The poll returns 2 more records, 1 from the topic-partition we've already been + // processing, and 1 more from the other topic-partition. This means we have processed + // all records from the former, and 2 remain from the latter. + pollAndValidateMaxPollRecordsNotExceeded(maxPollRecords); + + // See if we need to send another fetch, which we do because we've processed all of the records + // from one of the topic-partitions. The fetch response does not contain any more records. + assertEquals(1, sendFetches()); + // The poll returns the final 2 records. + pollAndValidateMaxPollRecordsNotExceeded(maxPollRecords); + } + + private void pollAndValidateMaxPollRecordsNotExceeded(int maxPollRecords) { + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchRecords(); + int fetchedRecords = 0; + for (List> recordsToTest : recordsByPartition.values()) { + fetchedRecords += recordsToTest.size(); + } + assertEquals(maxPollRecords, fetchedRecords); + } + /** * Test the scenario where a partition with fetched but not consumed records (i.e. max.poll.records is * less than the number of fetched records) is unassigned and a different partition is assigned. This is a @@ -3695,6 +3757,28 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); } + private FetchResponse fetchResponse2(TopicIdPartition tp1, MemoryRecords records1, long hw1, + TopicIdPartition tp2, MemoryRecords records2, long hw2) { + Map partitions = new HashMap<>(); + partitions.put(tp1, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.topicPartition().partition()) + .setErrorCode(Errors.NONE.code()) + .setHighWatermark(hw1) + .setLastStableOffset(FetchResponse.INVALID_LAST_STABLE_OFFSET) + .setLogStartOffset(0) + .setRecords(records1)); + partitions.put(tp2, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.topicPartition().partition()) + .setErrorCode(Errors.NONE.code()) + .setHighWatermark(hw2) + .setLastStableOffset(FetchResponse.INVALID_LAST_STABLE_OFFSET) + .setLogStartOffset(0) + .setRecords(records2)); + return FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + /** * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index d9355ce36a9e8..b584a5fde5c8d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -17,10 +17,14 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -43,6 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.BlockingQueue; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_INSTANCE_ID; @@ -68,9 +73,11 @@ public class HeartbeatRequestManagerTest { private HeartbeatRequestManager heartbeatRequestManager; private MembershipManager membershipManager; private HeartbeatRequestManager.HeartbeatRequestState heartbeatRequestState; + private HeartbeatRequestManager.HeartbeatState heartbeatState; private final String memberId = "member-id"; private final int memberEpoch = 1; private BackgroundEventHandler backgroundEventHandler; + private BlockingQueue backgroundEventQueue; @BeforeEach public void setUp() { @@ -78,12 +85,14 @@ public void setUp() { } private void setUp(Optional groupInfo) { - testBuilder = new ConsumerTestBuilder(groupInfo); + testBuilder = new ConsumerTestBuilder(groupInfo, true, false); time = testBuilder.time; coordinatorRequestManager = testBuilder.coordinatorRequestManager.orElseThrow(IllegalStateException::new); heartbeatRequestManager = testBuilder.heartbeatRequestManager.orElseThrow(IllegalStateException::new); heartbeatRequestState = testBuilder.heartbeatRequestState.orElseThrow(IllegalStateException::new); + heartbeatState = testBuilder.heartbeatState.orElseThrow(IllegalStateException::new); backgroundEventHandler = testBuilder.backgroundEventHandler; + backgroundEventQueue = testBuilder.backgroundEventQueue; subscriptions = testBuilder.subscriptions; membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new); @@ -111,10 +120,13 @@ public void cleanup() { @Test public void testHeartbeatOnStartup() { - // The initial heartbeatInterval is set to 0 - resetWithZeroHeartbeatInterval(Optional.empty()); - NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(0, result.unsentRequests.size()); + + resetWithZeroHeartbeatInterval(Optional.empty()); + mockStableMember(); + assertEquals(0, heartbeatRequestManager.maximumTimeToWait(time.milliseconds())); + result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); // Ensure we do not resend the request without the first request being completed @@ -122,19 +134,48 @@ public void testHeartbeatOnStartup() { assertEquals(0, result2.unsentRequests.size()); } + @ParameterizedTest + @ApiKeyVersionsSource(apiKey = ApiKeys.CONSUMER_GROUP_HEARTBEAT) + public void testFirstHeartbeatIncludesRequiredInfoToJoinGroupAndGetAssignments(short version) { + resetWithZeroHeartbeatInterval(Optional.of(DEFAULT_GROUP_INSTANCE_ID)); + String topic = "topic1"; + subscriptions.subscribe(Collections.singleton(topic), Optional.empty()); + membershipManager.onSubscriptionUpdated(); + + // Create a ConsumerHeartbeatRequest and verify the payload + assertEquals(0, heartbeatRequestManager.maximumTimeToWait(time.milliseconds())); + NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(1, pollResult.unsentRequests.size()); + NetworkClientDelegate.UnsentRequest request = pollResult.unsentRequests.get(0); + assertTrue(request.requestBuilder() instanceof ConsumerGroupHeartbeatRequest.Builder); + + ConsumerGroupHeartbeatRequest heartbeatRequest = + (ConsumerGroupHeartbeatRequest) request.requestBuilder().build(version); + + // Should include epoch 0 to join and no member ID. + assertTrue(heartbeatRequest.data().memberId().isEmpty()); + assertEquals(0, heartbeatRequest.data().memberEpoch()); + + // Should include subscription and group basic info to start getting assignments. + assertEquals(Collections.singletonList(topic), heartbeatRequest.data().subscribedTopicNames()); + assertEquals(DEFAULT_MAX_POLL_INTERVAL_MS, heartbeatRequest.data().rebalanceTimeoutMs()); + assertEquals(DEFAULT_GROUP_ID, heartbeatRequest.data().groupId()); + assertEquals(DEFAULT_GROUP_INSTANCE_ID, heartbeatRequest.data().instanceId()); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) - public void testSendHeartbeatOnMemberState(final boolean shouldSendHeartbeat) { + public void testSkippingHeartbeat(final boolean shouldSkipHeartbeat) { // The initial heartbeatInterval is set to 0 resetWithZeroHeartbeatInterval(Optional.empty()); // Mocking notInGroup - when(membershipManager.shouldSendHeartbeat()).thenReturn(shouldSendHeartbeat); + when(membershipManager.shouldSkipHeartbeat()).thenReturn(shouldSkipHeartbeat); when(heartbeatRequestState.canSendRequest(anyLong())).thenReturn(true); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); - if (shouldSendHeartbeat) { + if (!shouldSkipHeartbeat) { assertEquals(1, result.unsentRequests.size()); assertEquals(0, result.timeUntilNextPollMs); } else { @@ -144,27 +185,43 @@ public void testSendHeartbeatOnMemberState(final boolean shouldSendHeartbeat) { } } - @ParameterizedTest - @MethodSource("stateProvider") - public void testTimerNotDue(final MemberState state) { - when(membershipManager.state()).thenReturn(state); + @Test + public void testTimerNotDue() { + mockStableMember(); time.sleep(100); // time elapsed < heartbeatInterval, no heartbeat should be sent NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(0, result.unsentRequests.size()); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS - 100, result.timeUntilNextPollMs); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS - 100, heartbeatRequestManager.maximumTimeToWait(time.milliseconds())); - if (membershipManager.shouldSendHeartbeat()) { - assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS - 100, result.timeUntilNextPollMs); - } else { - assertEquals(Long.MAX_VALUE, result.timeUntilNextPollMs); - } + // Member in state where it should not send Heartbeat anymore + when(subscriptions.hasAutoAssignedPartitions()).thenReturn(true); + membershipManager.transitionToFatal(); + result = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(Long.MAX_VALUE, result.timeUntilNextPollMs); + } + + @Test + public void testHeartbeatOutsideInterval() { + when(membershipManager.shouldSkipHeartbeat()).thenReturn(false); + when(membershipManager.shouldHeartbeatNow()).thenReturn(true); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + + // Heartbeat should be sent + assertEquals(1, result.unsentRequests.size()); + // Interval timer reset + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, result.timeUntilNextPollMs); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, heartbeatRequestManager.maximumTimeToWait(time.milliseconds())); + // Membership manager updated (to transition out of the heartbeating state) + verify(membershipManager).onHeartbeatRequestSent(); } @Test public void testNetworkTimeout() { // The initial heartbeatInterval is set to 0 resetWithZeroHeartbeatInterval(Optional.empty()); + mockStableMember(); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); - when(membershipManager.shouldSendHeartbeat()).thenReturn(true); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); // Mimic network timeout @@ -184,13 +241,13 @@ public void testNetworkTimeout() { public void testFailureOnFatalException() { // The initial heartbeatInterval is set to 0 resetWithZeroHeartbeatInterval(Optional.empty()); + mockStableMember(); when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); - when(membershipManager.shouldSendHeartbeat()).thenReturn(true); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); result.unsentRequests.get(0).handler().onFailure(time.milliseconds(), new KafkaException("fatal")); - verify(membershipManager).transitionToFailed(); + verify(membershipManager).transitionToFatal(); verify(backgroundEventHandler).add(any()); } @@ -200,6 +257,7 @@ public void testNoCoordinator() { NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(Long.MAX_VALUE, result.timeUntilNextPollMs); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, heartbeatRequestManager.maximumTimeToWait(time.milliseconds())); assertEquals(0, result.unsentRequests.size()); } @@ -208,6 +266,7 @@ public void testNoCoordinator() { public void testValidateConsumerGroupHeartbeatRequest(final short version) { // The initial heartbeatInterval is set to 0, but we're testing resetWithZeroHeartbeatInterval(Optional.of(DEFAULT_GROUP_INSTANCE_ID)); + mockStableMember(); List subscribedTopics = Collections.singletonList("topic"); subscriptions.subscribe(new HashSet<>(subscribedTopics), Optional.empty()); @@ -217,7 +276,7 @@ public void testValidateConsumerGroupHeartbeatRequest(final short version) { new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() .setMemberId(memberId) .setMemberEpoch(memberEpoch)); - membershipManager.updateState(result.data()); + membershipManager.onHeartbeatResponseReceived(result.data()); // Create a ConsumerHeartbeatRequest and verify the payload NetworkClientDelegate.PollResult pollResult = heartbeatRequestManager.poll(time.milliseconds()); @@ -238,16 +297,105 @@ public void testValidateConsumerGroupHeartbeatRequest(final short version) { assertNull(heartbeatRequest.data().subscribedTopicRegex()); } + @Test + public void testConsumerGroupMetadataFirstUpdate() { + final GroupMetadataUpdateEvent groupMetadataUpdateEvent = makeFirstGroupMetadataUpdate(memberId, memberEpoch); + + final GroupMetadataUpdateEvent expectedGroupMetadataUpdateEvent = new GroupMetadataUpdateEvent( + memberEpoch, + memberId + ); + assertEquals(expectedGroupMetadataUpdateEvent, groupMetadataUpdateEvent); + } + + @Test + public void testConsumerGroupMetadataUpdateWithSameUpdate() { + makeFirstGroupMetadataUpdate(memberId, memberEpoch); + + time.sleep(2000); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + + assertEquals(1, result.unsentRequests.size()); + NetworkClientDelegate.UnsentRequest request = result.unsentRequests.get(0); + ClientResponse responseWithSameUpdate = createHeartbeatResponse(request, Errors.NONE); + request.handler().onComplete(responseWithSameUpdate); + assertEquals(0, backgroundEventQueue.size()); + } + + @Test + public void testConsumerGroupMetadataUpdateWithMemberIdNullButMemberEpochUpdated() { + makeFirstGroupMetadataUpdate(memberId, memberEpoch); + + time.sleep(2000); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + + assertEquals(1, result.unsentRequests.size()); + NetworkClientDelegate.UnsentRequest request = result.unsentRequests.get(0); + final int updatedMemberEpoch = 2; + ClientResponse responseWithMemberEpochUpdate = createHeartbeatResponseWithMemberIdNull( + request, + Errors.NONE, + updatedMemberEpoch + ); + request.handler().onComplete(responseWithMemberEpochUpdate); + assertEquals(1, backgroundEventQueue.size()); + final BackgroundEvent eventWithUpdatedMemberEpoch = backgroundEventQueue.poll(); + assertEquals(BackgroundEvent.Type.GROUP_METADATA_UPDATE, eventWithUpdatedMemberEpoch.type()); + final GroupMetadataUpdateEvent groupMetadataUpdateEvent = (GroupMetadataUpdateEvent) eventWithUpdatedMemberEpoch; + final GroupMetadataUpdateEvent expectedGroupMetadataUpdateEvent = new GroupMetadataUpdateEvent( + updatedMemberEpoch, + memberId + ); + assertEquals(expectedGroupMetadataUpdateEvent, groupMetadataUpdateEvent); + } + + @Test + public void testConsumerGroupMetadataUpdateWithMemberIdUpdatedAndMemberEpochSame() { + makeFirstGroupMetadataUpdate(memberId, memberEpoch); + + time.sleep(2000); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + + assertEquals(1, result.unsentRequests.size()); + NetworkClientDelegate.UnsentRequest request = result.unsentRequests.get(0); + final String updatedMemberId = "updatedMemberId"; + ClientResponse responseWithMemberIdUpdate = createHeartbeatResponse( + request, + Errors.NONE, + updatedMemberId, + memberEpoch + ); + request.handler().onComplete(responseWithMemberIdUpdate); + assertEquals(1, backgroundEventQueue.size()); + final BackgroundEvent eventWithUpdatedMemberEpoch = backgroundEventQueue.poll(); + assertEquals(BackgroundEvent.Type.GROUP_METADATA_UPDATE, eventWithUpdatedMemberEpoch.type()); + final GroupMetadataUpdateEvent groupMetadataUpdateEvent = (GroupMetadataUpdateEvent) eventWithUpdatedMemberEpoch; + final GroupMetadataUpdateEvent expectedGroupMetadataUpdateEvent = new GroupMetadataUpdateEvent( + memberEpoch, + updatedMemberId + ); + assertEquals(expectedGroupMetadataUpdateEvent, groupMetadataUpdateEvent); + } + + private GroupMetadataUpdateEvent makeFirstGroupMetadataUpdate(final String memberId, final int memberEpoch) { + resetWithZeroHeartbeatInterval(Optional.empty()); + mockStableMember(); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); + assertEquals(1, result.unsentRequests.size()); + NetworkClientDelegate.UnsentRequest request = result.unsentRequests.get(0); + ClientResponse firstResponse = createHeartbeatResponse(request, Errors.NONE, memberId, memberEpoch); + request.handler().onComplete(firstResponse); + assertEquals(1, backgroundEventQueue.size()); + final BackgroundEvent event = backgroundEventQueue.poll(); + assertEquals(BackgroundEvent.Type.GROUP_METADATA_UPDATE, event.type()); + return (GroupMetadataUpdateEvent) event; + } + @ParameterizedTest @MethodSource("errorProvider") public void testHeartbeatResponseOnErrorHandling(final Errors error, final boolean isFatal) { - // Sending first heartbeat w/o assignment to set the state to STABLE - ConsumerGroupHeartbeatResponse rs1 = new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() - .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) - .setMemberId(memberId) - .setMemberEpoch(memberEpoch)); - membershipManager.updateState(rs1.data()); - assertEquals(MemberState.STABLE, membershipManager.state()); + mockStableMember(); // Handling errors on the second heartbeat time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); @@ -255,6 +403,7 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole assertEquals(1, result.unsentRequests.size()); // Manually completing the response to test error handling + when(subscriptions.hasAutoAssignedPartitions()).thenReturn(true); ClientResponse response = createHeartbeatResponse( result.unsentRequests.get(0), error); @@ -263,8 +412,8 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole switch (error) { case NONE: - verify(backgroundEventHandler, never()).add(any()); - verify(membershipManager, times(2)).updateState(mockResponse.data()); + verify(backgroundEventHandler).add(any(GroupMetadataUpdateEvent.class)); + verify(membershipManager, times(2)).onHeartbeatResponseReceived(mockResponse.data()); assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); break; @@ -292,15 +441,92 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole } } + @Test + public void testHeartbeatState() { + // The initial ConsumerGroupHeartbeatRequest sets most fields to their initial empty values + ConsumerGroupHeartbeatRequestData data = heartbeatState.buildRequestData(); + assertEquals(ConsumerTestBuilder.DEFAULT_GROUP_ID, data.groupId()); + assertEquals("", data.memberId()); + assertEquals(0, data.memberEpoch()); + assertNull(data.instanceId()); + assertEquals(ConsumerTestBuilder.DEFAULT_MAX_POLL_INTERVAL_MS, data.rebalanceTimeoutMs()); + assertEquals(Collections.emptyList(), data.subscribedTopicNames()); + assertNull(data.subscribedTopicRegex()); + assertNull(data.serverAssignor()); + assertEquals(Collections.emptyList(), data.topicPartitions()); + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state()); + + // Mock a response from the group coordinator, that supplies the member ID and a new epoch + mockStableMember(); + data = heartbeatState.buildRequestData(); + assertEquals(ConsumerTestBuilder.DEFAULT_GROUP_ID, data.groupId()); + assertEquals(memberId, data.memberId()); + assertEquals(1, data.memberEpoch()); + assertNull(data.instanceId()); + assertEquals(-1, data.rebalanceTimeoutMs()); + assertNull(data.subscribedTopicNames()); + assertNull(data.subscribedTopicRegex()); + assertNull(data.serverAssignor()); + assertNull(data.topicPartitions()); + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.STABLE, membershipManager.state()); + + // Join the group and subscribe to a topic, but the response has not yet been received + String topic = "topic1"; + subscriptions.subscribe(Collections.singleton(topic), Optional.empty()); + membershipManager.onSubscriptionUpdated(); + membershipManager.transitionToFenced(); // And indirect way of moving to JOINING state + data = heartbeatState.buildRequestData(); + assertEquals(ConsumerTestBuilder.DEFAULT_GROUP_ID, data.groupId()); + assertEquals(memberId, data.memberId()); + assertEquals(0, data.memberEpoch()); + assertNull(data.instanceId()); + assertEquals(-1, data.rebalanceTimeoutMs()); + assertEquals(Collections.singletonList(topic), data.subscribedTopicNames()); + assertNull(data.subscribedTopicRegex()); + assertNull(data.serverAssignor()); + assertNull(data.topicPartitions()); + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.JOINING, membershipManager.state()); + + // Mock the response from the group coordinator which returns an assignment + ConsumerGroupHeartbeatResponseData.TopicPartitions tpTopic1 = + new ConsumerGroupHeartbeatResponseData.TopicPartitions(); + tpTopic1.setTopicId(Uuid.randomUuid()); + tpTopic1.setPartitions(Collections.singletonList(0)); + ConsumerGroupHeartbeatResponseData.Assignment assignmentTopic1 = + new ConsumerGroupHeartbeatResponseData.Assignment(); + assignmentTopic1.setTopicPartitions(Collections.singletonList(tpTopic1)); + ConsumerGroupHeartbeatResponse rs1 = new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() + .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) + .setMemberId(memberId) + .setMemberEpoch(1) + .setAssignment(assignmentTopic1)); + membershipManager.onHeartbeatResponseReceived(rs1.data()); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + } + + private void mockStableMember() { + membershipManager.onSubscriptionUpdated(); + // Heartbeat response without assignment to set the state to STABLE. + ConsumerGroupHeartbeatResponse rs1 = new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() + .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) + .setMemberId(memberId) + .setMemberEpoch(memberEpoch)); + membershipManager.onHeartbeatResponseReceived(rs1.data()); + assertEquals(MemberState.STABLE, membershipManager.state()); + } + private void ensureFatalError() { - verify(membershipManager).transitionToFailed(); + verify(membershipManager).transitionToFatal(); verify(backgroundEventHandler).add(any()); ensureHeartbeatStopped(); } private void ensureHeartbeatStopped() { time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); - assertEquals(MemberState.FAILED, membershipManager.state()); + assertEquals(MemberState.FATAL, membershipManager.state()); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(0, result.unsentRequests.size()); } @@ -322,18 +548,27 @@ private static Collection errorProvider() { Arguments.of(Errors.GROUP_MAX_SIZE_REACHED, true)); } - private static Collection stateProvider() { - return Arrays.asList( - Arguments.of(MemberState.UNJOINED), - Arguments.of(MemberState.RECONCILING), - Arguments.of(MemberState.FAILED), - Arguments.of(MemberState.STABLE), - Arguments.of(MemberState.FENCED)); + private ClientResponse createHeartbeatResponse( + final NetworkClientDelegate.UnsentRequest request, + final Errors error + ) { + return createHeartbeatResponse(request, error, memberId, memberEpoch); + } + + private ClientResponse createHeartbeatResponseWithMemberIdNull( + final NetworkClientDelegate.UnsentRequest request, + final Errors error, + final int memberEpoch + ) { + return createHeartbeatResponse(request, error, null, memberEpoch); } private ClientResponse createHeartbeatResponse( final NetworkClientDelegate.UnsentRequest request, - final Errors error) { + final Errors error, + final String memberId, + final int memberEpoch + ) { ConsumerGroupHeartbeatResponseData data = new ConsumerGroupHeartbeatResponseData() .setErrorCode(error.code()) .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java index d78bbf2ab63ee..8d3f61ea26ce2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java @@ -17,231 +17,1033 @@ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; -import org.apache.kafka.common.utils.LogContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class MembershipManagerImplTest { private static final String GROUP_ID = "test-group"; private static final String MEMBER_ID = "test-member-1"; private static final int MEMBER_EPOCH = 1; - private final LogContext logContext = new LogContext(); + + private SubscriptionState subscriptionState; + private ConsumerMetadata metadata; + + private CommitRequestManager commitRequestManager; + + private ConsumerTestBuilder testBuilder; + + @BeforeEach + public void setup() { + testBuilder = new ConsumerTestBuilder(ConsumerTestBuilder.createDefaultGroupInformation()); + metadata = testBuilder.metadata; + subscriptionState = testBuilder.subscriptions; + commitRequestManager = testBuilder.commitRequestManager.get(); + } + + @AfterEach + public void tearDown() { + if (testBuilder != null) { + testBuilder.close(); + } + } + + private MembershipManagerImpl createMembershipManagerJoiningGroup() { + MembershipManagerImpl manager = spy(new MembershipManagerImpl( + GROUP_ID, subscriptionState, commitRequestManager, + metadata, testBuilder.logContext)); + manager.transitionToJoining(); + return manager; + } + + private MembershipManagerImpl createMembershipManagerJoiningGroup(String groupInstanceId, + String serverAssignor) { + MembershipManagerImpl manager = new MembershipManagerImpl( + GROUP_ID, Optional.ofNullable(groupInstanceId), Optional.ofNullable(serverAssignor), + subscriptionState, commitRequestManager, metadata, testBuilder.logContext); + manager.transitionToJoining(); + return manager; + } @Test public void testMembershipManagerServerAssignor() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); assertEquals(Optional.empty(), membershipManager.serverAssignor()); - membershipManager = new MembershipManagerImpl(GROUP_ID, "instance1", "Uniform", logContext); + membershipManager = createMembershipManagerJoiningGroup("instance1", "Uniform"); assertEquals(Optional.of("Uniform"), membershipManager.serverAssignor()); } @Test public void testMembershipManagerInitSupportsEmptyGroupInstanceId() { - new MembershipManagerImpl(GROUP_ID, logContext); - new MembershipManagerImpl(GROUP_ID, null, null, logContext); + createMembershipManagerJoiningGroup(); + createMembershipManagerJoiningGroup(null, null); + } + + @Test + public void testMembershipManagerRegistersForClusterMetadataUpdatesOnFirstJoin() { + // First join should register to get metadata updates + MembershipManagerImpl manager = new MembershipManagerImpl( + GROUP_ID, subscriptionState, commitRequestManager, + metadata, testBuilder.logContext); + manager.transitionToJoining(); + verify(metadata).addClusterUpdateListener(manager); + clearInvocations(metadata); + + // Following joins should not register again. + receiveEmptyAssignment(manager); + mockLeaveGroup(); + manager.leaveGroup(); + assertEquals(MemberState.LEAVING, manager.state()); + manager.onHeartbeatRequestSent(); + assertEquals(MemberState.UNSUBSCRIBED, manager.state()); + manager.transitionToJoining(); + verify(metadata, never()).addClusterUpdateListener(manager); + } + + @Test + public void testReconcilingWhenReceivingAssignmentFoundInMetadata() { + MembershipManager membershipManager = mockJoinAndReceiveAssignment(true); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + + // When the ack is sent the member should go back to STABLE + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.STABLE, membershipManager.state()); } @Test public void testTransitionToReconcilingOnlyIfAssignmentReceived() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - assertEquals(MemberState.UNJOINED, membershipManager.state()); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + assertEquals(MemberState.JOINING, membershipManager.state()); ConsumerGroupHeartbeatResponse responseWithoutAssignment = createConsumerGroupHeartbeatResponse(null); - membershipManager.updateState(responseWithoutAssignment.data()); + membershipManager.onHeartbeatResponseReceived(responseWithoutAssignment.data()); assertNotEquals(MemberState.RECONCILING, membershipManager.state()); ConsumerGroupHeartbeatResponse responseWithAssignment = - createConsumerGroupHeartbeatResponse(createAssignment()); - membershipManager.updateState(responseWithAssignment.data()); + createConsumerGroupHeartbeatResponse(createAssignment(true)); + membershipManager.onHeartbeatResponseReceived(responseWithAssignment.data()); assertEquals(MemberState.RECONCILING, membershipManager.state()); } @Test public void testMemberIdAndEpochResetOnFencedMembers() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - ConsumerGroupHeartbeatResponse heartbeatResponse = - createConsumerGroupHeartbeatResponse(null); - membershipManager.updateState(heartbeatResponse.data()); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); assertEquals(MemberState.STABLE, membershipManager.state()); assertEquals(MEMBER_ID, membershipManager.memberId()); assertEquals(MEMBER_EPOCH, membershipManager.memberEpoch()); + mockMemberHasAutoAssignedPartition(); + membershipManager.transitionToFenced(); - assertFalse(membershipManager.memberId().isEmpty()); + assertEquals(MEMBER_ID, membershipManager.memberId()); assertEquals(0, membershipManager.memberEpoch()); } @Test - public void testTransitionToFailure() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); + public void testTransitionToFatal() { + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null); - membershipManager.updateState(heartbeatResponse.data()); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); assertEquals(MemberState.STABLE, membershipManager.state()); assertEquals(MEMBER_ID, membershipManager.memberId()); assertEquals(MEMBER_EPOCH, membershipManager.memberEpoch()); - membershipManager.transitionToFailed(); - assertEquals(MemberState.FAILED, membershipManager.state()); + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + membershipManager.transitionToFatal(); + assertEquals(MemberState.FATAL, membershipManager.state()); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + } + + @Test + public void testTransitionToFailedWhenTryingToJoin() { + MembershipManagerImpl membershipManager = new MembershipManagerImpl( + GROUP_ID, subscriptionState, commitRequestManager, metadata, + testBuilder.logContext); + assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state()); + membershipManager.transitionToJoining(); + + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + membershipManager.transitionToFatal(); + assertEquals(MemberState.FATAL, membershipManager.state()); } @Test public void testFencingWhenStateIsStable() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null); - membershipManager.updateState(heartbeatResponse.data()); + MembershipManager membershipManager = createMemberInStableState(); + testFencedMemberReleasesAssignmentAndTransitionsToJoining(membershipManager); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + } + + @Test + public void testFencingWhenStateIsReconciling() { + MembershipManager membershipManager = mockJoinAndReceiveAssignment(false); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + testFencedMemberReleasesAssignmentAndTransitionsToJoining(membershipManager); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + } + + /** + * This is the case where a member is stuck reconciling and transition out of the RECONCILING + * state (due to failure). When the reconciliation completes it should not be applied because + * it is not relevant anymore (it should not update the assignment on the member or send ack). + */ + @Test + public void testDelayedReconciliationResultDiscardedIfMemberNotInReconcilingStateAnymore() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + Uuid topicId1 = Uuid.randomUuid(); + String topic1 = "topic1"; + Set owned = Collections.singleton(new TopicPartition(topic1, 0)); + mockOwnedPartitionAndAssignmentReceived(topicId1, topic1, owned, true); + + // Reconciliation that does not complete stuck on revocation commit. + CompletableFuture commitResult = mockEmptyAssignmentAndRevocationStuckOnCommit(membershipManager); + + // Member received fatal error while reconciling + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + membershipManager.transitionToFatal(); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + clearInvocations(subscriptionState); + + // Complete commit request + commitResult.complete(null); + + // Member should not update the subscription or send ack when the delayed reconciliation + // completed. + verify(subscriptionState, never()).assignFromSubscribed(anySet()); + assertNotEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + } + + /** + * This is the case where a member is stuck reconciling an assignment A (waiting on + * metadata, commit or callbacks), and it rejoins (due to fence or unsubscribe/subscribe). If + * the reconciliation of A completes it should not be applied (it should not update the + * assignment on the member or send ack). + */ + @Test + public void testDelayedReconciliationResultDiscardedIfMemberRejoins() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + Uuid topicId1 = Uuid.randomUuid(); + String topic1 = "topic1"; + Set owned = Collections.singleton(new TopicPartition(topic1, 0)); + mockOwnedPartitionAndAssignmentReceived(topicId1, topic1, owned, true); + + // Reconciliation that does not complete stuck on revocation commit. + CompletableFuture commitResult = + mockNewAssignmentAndRevocationStuckOnCommit(membershipManager, topicId1, topic1, + Arrays.asList(1, 2), true); + Set assignment1 = new HashSet<>(); + assignment1.add(new TopicIdPartition(topicId1, new TopicPartition(topic1, 1))); + assignment1.add(new TopicIdPartition(topicId1, new TopicPartition(topic1, 2))); + assertEquals(assignment1, membershipManager.assignmentReadyToReconcile()); + + // Get fenced and rejoin while still reconciling. Get new assignment to reconcile after + // rejoining. + testFencedMemberReleasesAssignmentAndTransitionsToJoining(membershipManager); + clearInvocations(subscriptionState); + + // Get new assignment A2 after rejoining. This should not trigger a reconciliation just + // yet because there is another on in progress, but should keep the new assignment ready + // to be reconciled next. + Uuid topicId3 = Uuid.randomUuid(); + mockOwnedPartitionAndAssignmentReceived(topicId3, "topic3", owned, true); + receiveAssignmentAfterRejoin(topicId3, Collections.singletonList(5), membershipManager); + verifyReconciliationNotTriggered(membershipManager); + Set assignmentAfterRejoin = Collections.singleton( + new TopicIdPartition(topicId3, new TopicPartition("topic3", 5))); + assertEquals(assignmentAfterRejoin, membershipManager.assignmentReadyToReconcile()); + + // Reconciliation completes when the member has already re-joined the group. Should not + // update the subscription state or send ack. + commitResult.complete(null); + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); + assertNotEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + + // Assignment received after rejoining should be ready to reconcile on the next + // reconciliation loop. + assertEquals(MemberState.RECONCILING, membershipManager.state()); + assertEquals(assignmentAfterRejoin, membershipManager.assignmentReadyToReconcile()); + } + + /** + * This is the case where a member is stuck reconciling an assignment A (waiting on + * metadata, commit or callbacks), and the target assignment changes (due to new topics added + * to metadata, or new assignment received from broker). If the reconciliation of A completes + * it should be applied (should update the assignment on the member and send ack), and then + * the reconciliation of assignment B will be processed and applied in the next + * reconciliation loop. + */ + @Test + public void testDelayedReconciliationResultAppliedWhenTargetChangedWithMetadataUpdate() { + // Member receives and reconciles topic1-partition0 + Uuid topicId1 = Uuid.randomUuid(); + String topic1 = "topic1"; + MembershipManagerImpl membershipManager = + mockMemberSuccessfullyReceivesAndAcksAssignment(topicId1, topic1, Arrays.asList(0)); + membershipManager.onHeartbeatRequestSent(); assertEquals(MemberState.STABLE, membershipManager.state()); + clearInvocations(membershipManager, subscriptionState); + when(subscriptionState.assignedPartitions()).thenReturn(Collections.singleton(new TopicPartition(topic1, 0))); + + // New assignment revoking the partitions owned and adding a new one (not in metadata). + // Reconciliation triggered for topic 1 (stuck on revocation commit) and topic2 waiting + // for metadata. + Uuid topicId2 = Uuid.randomUuid(); + String topic2 = "topic2"; + CompletableFuture commitResult = + mockNewAssignmentAndRevocationStuckOnCommit(membershipManager, topicId2, topic2, + Arrays.asList(1, 2), false); + verify(metadata).requestUpdate(anyBoolean()); + assertEquals(Collections.singleton(topicId2), membershipManager.topicsWaitingForMetadata()); - testStateUpdateOnFenceError(membershipManager); + // Metadata discovered for topic2 while reconciliation in progress to revoke topic1. + // Should not trigger a new reconciliation because there is one already in progress. + mockTopicNameInMetadataCache(Collections.singletonMap(topicId2, topic2), true); + membershipManager.onUpdate(null); + assertEquals(Collections.emptySet(), membershipManager.topicsWaitingForMetadata()); + verifyReconciliationNotTriggered(membershipManager); + + // Reconciliation in progress completes. Should be applied revoking topic 1 only. Newly + // discovered topic2 will be reconciled in the next reconciliation loop. + commitResult.complete(null); + + // Member should update the subscription and send ack when the delayed reconciliation + // completes. + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + + // Pending assignment that was discovered in metadata should be ready to reconcile in the + // next reconciliation loop. + Set topic2Assignment = new HashSet<>(Arrays.asList( + new TopicIdPartition(topicId2, new TopicPartition(topic2, 1)), + new TopicIdPartition(topicId2, new TopicPartition(topic2, 2)))); + assertEquals(topic2Assignment, membershipManager.assignmentReadyToReconcile()); } @Test - public void testFencingWhenStateIsReconciling() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(createAssignment()); - membershipManager.updateState(heartbeatResponse.data()); + public void testLeaveGroupWhenStateIsStable() { + MembershipManager membershipManager = createMemberInStableState(); + testLeaveGroupReleasesAssignmentAndResetsEpochToSendLeaveGroup(membershipManager); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + } + + @Test + public void testLeaveGroupWhenStateIsReconciling() { + MembershipManager membershipManager = mockJoinAndReceiveAssignment(false); assertEquals(MemberState.RECONCILING, membershipManager.state()); - testStateUpdateOnFenceError(membershipManager); + testLeaveGroupReleasesAssignmentAndResetsEpochToSendLeaveGroup(membershipManager); + } + + @Test + public void testLeaveGroupWhenMemberAlreadyLeaving() { + MembershipManager membershipManager = createMemberInStableState(); + + // First leave attempt. Should trigger the callbacks and stay LEAVING until + // callbacks complete and the heartbeat is sent out. + mockLeaveGroup(); + CompletableFuture leaveResult1 = membershipManager.leaveGroup(); + assertFalse(leaveResult1.isDone()); + assertEquals(MemberState.LEAVING, membershipManager.state()); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + clearInvocations(subscriptionState); + + // Second leave attempt while the first one has not completed yet. Should not + // trigger any callbacks, and return a future that will complete when the ongoing first + // leave operation completes. + mockLeaveGroup(); + CompletableFuture leaveResult2 = membershipManager.leaveGroup(); + verify(subscriptionState, never()).rebalanceListener(); + assertFalse(leaveResult2.isDone()); + + // Complete first leave group operation. Should also complete the second leave group. + membershipManager.onHeartbeatRequestSent(); + assertTrue(leaveResult1.isDone()); + assertFalse(leaveResult1.isCompletedExceptionally()); + assertTrue(leaveResult2.isDone()); + assertFalse(leaveResult2.isCompletedExceptionally()); + + // Subscription should have been updated only once with the first leave group. + verify(subscriptionState, never()).assignFromSubscribed(Collections.emptySet()); + } + + @Test + public void testLeaveGroupWhenMemberAlreadyLeft() { + MembershipManager membershipManager = createMemberInStableState(); + + // Leave group triggered and completed + mockLeaveGroup(); + CompletableFuture leaveResult1 = membershipManager.leaveGroup(); + assertEquals(MemberState.LEAVING, membershipManager.state()); + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state()); + assertTrue(leaveResult1.isDone()); + assertFalse(leaveResult1.isCompletedExceptionally()); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + clearInvocations(subscriptionState); + + // Call to leave group again, when member already left. Should be no-op (no callbacks, + // no assignment updated) + mockLeaveGroup(); + CompletableFuture leaveResult2 = membershipManager.leaveGroup(); + assertTrue(leaveResult2.isDone()); + assertFalse(leaveResult2.isCompletedExceptionally()); + assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state()); + verify(subscriptionState, never()).rebalanceListener(); + verify(subscriptionState, never()).assignFromSubscribed(Collections.emptySet()); } @Test public void testFatalFailureWhenStateIsUnjoined() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - assertEquals(MemberState.UNJOINED, membershipManager.state()); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + assertEquals(MemberState.JOINING, membershipManager.state()); testStateUpdateOnFatalFailure(membershipManager); } @Test public void testFatalFailureWhenStateIsStable() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null); - membershipManager.updateState(heartbeatResponse.data()); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); assertEquals(MemberState.STABLE, membershipManager.state()); testStateUpdateOnFatalFailure(membershipManager); } - @Test - public void testFencingShouldNotHappenWhenStateIsUnjoined() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - assertEquals(MemberState.UNJOINED, membershipManager.state()); - - // Getting fenced when the member is not part of the group is not expected and should - // fail with invalid transition. - assertThrows(IllegalStateException.class, membershipManager::transitionToFenced); - } - @Test public void testUpdateStateFailsOnResponsesWithErrors() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); // Updating state with a heartbeat response containing errors cannot be performed and // should fail. ConsumerGroupHeartbeatResponse unknownMemberResponse = createConsumerGroupHeartbeatResponseWithError(Errors.UNKNOWN_MEMBER_ID); assertThrows(IllegalArgumentException.class, - () -> membershipManager.updateState(unknownMemberResponse.data())); + () -> membershipManager.onHeartbeatResponseReceived(unknownMemberResponse.data())); } + /** + * This test should be the case when an assignment is sent to the member, and it cannot find + * it in metadata (permanently, ex. topic deleted). The member will keep the assignment as + * waiting for metadata, but the following assignment received from the broker will not + * contain the deleted topic. The member will discard the assignment that was pending and + * proceed with the reconciliation of the new assignment. + */ @Test - public void testAssignmentUpdatedAsReceivedAndProcessed() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - ConsumerGroupHeartbeatResponseData.Assignment newAssignment = createAssignment(); - ConsumerGroupHeartbeatResponse heartbeatResponse = - createConsumerGroupHeartbeatResponse(newAssignment); - membershipManager.updateState(heartbeatResponse.data()); + public void testNewAssignmentReplacesPreviousOneWaitingOnMetadata() { + MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + // When the ack is sent the member should go back to RECONCILING + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0); - // Target assignment should be in the process of being reconciled - checkAssignments(membershipManager, null, newAssignment); - // Mark assignment processing completed - membershipManager.onTargetAssignmentProcessComplete(newAssignment); - // Target assignment should now be the current assignment - checkAssignments(membershipManager, newAssignment, null); + // New target assignment received while there is another one waiting to be resolved + // and reconciled. This assignment does not include the previous one that is waiting + // for metadata, so the member will discard the topics that were waiting for metadata, and + // reconcile the new assignment. + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName)); + receiveAssignment(topicId, Collections.singletonList(0), membershipManager); + Set expectedAssignment = Collections.singleton(new TopicPartition(topicName, 0)); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + verify(subscriptionState).assignFromSubscribed(expectedAssignment); + + // When ack for the reconciled assignment is sent, member should go back to STABLE + // because the first assignment that was not resolved should have been discarded + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.STABLE, membershipManager.state()); + assertTrue(membershipManager.topicsWaitingForMetadata().isEmpty()); } + /** + * This test should be the case when an assignment is sent to the member, and it cannot find + * it in metadata (temporarily). If the broker continues to send the assignment to the + * member, this one should keep it waiting for metadata and continue to request updates. + */ @Test - public void testMemberFailsIfAssignmentReceivedWhileAnotherOnBeingReconciled() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); - ConsumerGroupHeartbeatResponseData.Assignment newAssignment1 = createAssignment(); - membershipManager.updateState(createConsumerGroupHeartbeatResponse(newAssignment1).data()); + public void testMemberKeepsUnresolvedAssignmentWaitingForMetadataUntilResolved() { + // Assignment with 2 topics, only 1 found in metadata + Uuid topic1 = Uuid.randomUuid(); + String topic1Name = "topic1"; + Uuid topic2 = Uuid.randomUuid(); + ConsumerGroupHeartbeatResponseData.Assignment assignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(Arrays.asList( + new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topic1) + .setPartitions(Arrays.asList(0)), + new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topic2) + .setPartitions(Arrays.asList(1, 3)) + )); + when(metadata.topicNames()).thenReturn(Collections.singletonMap(topic1, topic1Name)); - // First target assignment received should be in the process of being reconciled - checkAssignments(membershipManager, null, newAssignment1); + // Receive assignment partly in metadata - reconcile+ack what's in metadata, keep the + // unresolved and request metadata update. + MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(true, assignment); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + verify(metadata).requestUpdate(anyBoolean()); + assertEquals(Collections.singleton(topic2), membershipManager.topicsWaitingForMetadata()); + + // When the ack is sent the member should go back to RECONCILING because it still has + // unresolved assignment to be reconciled. + membershipManager.onHeartbeatRequestSent(); + assertEquals(MemberState.RECONCILING, membershipManager.state()); - // Second target assignment received while there is another one being reconciled - ConsumerGroupHeartbeatResponseData.Assignment newAssignment2 = createAssignment(); - assertThrows(IllegalStateException.class, - () -> membershipManager.updateState(createConsumerGroupHeartbeatResponse(newAssignment2).data())); - assertEquals(MemberState.FAILED, membershipManager.state()); + // Target assignment received again with the same unresolved topic. Client should keep it + // as unresolved. + clearInvocations(subscriptionState); + membershipManager.onHeartbeatResponseReceived(createConsumerGroupHeartbeatResponse(assignment).data()); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + assertEquals(Collections.singleton(topic2), membershipManager.topicsWaitingForMetadata()); + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); } @Test - public void testAssignmentUpdatedFailsIfAssignmentReconciledDoesNotMatchTargetAssignment() { - MembershipManagerImpl membershipManager = new MembershipManagerImpl(GROUP_ID, logContext); + public void testReconcileNewPartitionsAssignedWhenNoPartitionOwned() { + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true); + + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager); + + Set assignedPartitions = new HashSet<>(Arrays.asList( + new TopicIdPartition(topicId, new TopicPartition(topicName, 0)), + new TopicIdPartition(topicId, new TopicPartition(topicName, 1)))); + verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions); + } + + @Test + public void testReconcileNewPartitionsAssignedWhenOtherPartitionsOwned() { + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + TopicPartition ownedPartition = new TopicPartition(topicName, 0); + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, + Collections.singleton(ownedPartition), true); + + MembershipManagerImpl membershipManager = createMemberInStableState(); + // New assignment received, adding partitions 1 and 2 to the previously owned partition 0. + receiveAssignment(topicId, Arrays.asList(0, 1, 2), membershipManager); + + Set assignedPartitions = new HashSet<>(Arrays.asList( + new TopicIdPartition(topicId, ownedPartition), + new TopicIdPartition(topicId, new TopicPartition("topic1", 1)), + new TopicIdPartition(topicId, new TopicPartition("topic1", 2)))); + verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions); + } + + @Test + public void testReconciliationSkippedWhenSameAssignmentReceived() { + // Member stable, no assignment + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + + // Receive assignment different from what the member owns - should reconcile + Set owned = Collections.emptySet(); + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, owned, true); + Set expectedAssignmentReconciled = new HashSet<>(); + expectedAssignmentReconciled.add(new TopicIdPartition(topicId, + new TopicPartition(topicName, 0))); + expectedAssignmentReconciled.add(new TopicIdPartition(topicId, + new TopicPartition(topicName, 1))); + receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager); + verifyReconciliationTriggeredAndCompleted(membershipManager, expectedAssignmentReconciled); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + clearInvocations(subscriptionState, membershipManager); + + // Receive same assignment again - should not trigger reconciliation + Set expectedTopicPartitionAssignment = buildTopicPartitions(expectedAssignmentReconciled); + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, expectedTopicPartitionAssignment, true); + receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager); + // Verify new reconciliation was not triggered + verify(membershipManager, never()).markReconciliationInProgress(); + verify(membershipManager, never()).markReconciliationCompleted(); + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); + } + + @Test + public void testReconcilePartitionsRevokedNoAutoCommitNoCallbacks() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + mockOwnedPartition("topic1", 0); + + mockRevocationNoCallbacks(false); + + receiveEmptyAssignment(membershipManager); + + testRevocationOfAllPartitionsCompleted(membershipManager); + } + + @Test + public void testReconcilePartitionsRevokedWithSuccessfulAutoCommitNoCallbacks() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + mockOwnedPartition("topic1", 0); + + CompletableFuture commitResult = mockRevocationNoCallbacks(true); + + receiveEmptyAssignment(membershipManager); + + // Member stays in RECONCILING while the commit request hasn't completed. + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + // Partitions should be still owned by the member + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); + + // Complete commit request + commitResult.complete(null); + + testRevocationOfAllPartitionsCompleted(membershipManager); + } + + @Test + public void testReconcilePartitionsRevokedWithFailedAutoCommitCompletesRevocationAnyway() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + mockOwnedPartition("topic1", 0); + + CompletableFuture commitResult = mockRevocationNoCallbacks(true); + + receiveEmptyAssignment(membershipManager); + + // Member stays in RECONCILING while the commit request hasn't completed. + assertEquals(MemberState.RECONCILING, membershipManager.state()); + // Partitions should be still owned by the member + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); + + // Complete commit request + commitResult.completeExceptionally(new KafkaException("Commit request failed with " + + "non-retriable error")); + + testRevocationOfAllPartitionsCompleted(membershipManager); + } + + @Test + public void testReconcileNewPartitionsAssignedAndRevoked() { + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + TopicPartition ownedPartition = new TopicPartition(topicName, 0); + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.singleton(ownedPartition), true); + + mockRevocationNoCallbacks(false); + + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + // New assignment received, revoking partition 0, and assigning new partitions 1 and 2. + receiveAssignment(topicId, Arrays.asList(1, 2), membershipManager); + + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + Set assignedPartitions = new HashSet<>(Arrays.asList( + new TopicIdPartition(topicId, new TopicPartition("topic1", 1)), + new TopicIdPartition(topicId, new TopicPartition("topic1", 2)))); + assertEquals(assignedPartitions, membershipManager.currentAssignment()); + assertFalse(membershipManager.reconciliationInProgress()); + + verify(subscriptionState).assignFromSubscribed(anyCollection()); + } + + @Test + public void testMetadataUpdatesReconcilesUnresolvedAssignments() { + Uuid topicId = Uuid.randomUuid(); + + // Assignment not in metadata ConsumerGroupHeartbeatResponseData.Assignment targetAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() .setTopicPartitions(Collections.singletonList( new ConsumerGroupHeartbeatResponseData.TopicPartitions() - .setTopicId(Uuid.randomUuid()) - .setPartitions(Arrays.asList(0, 1, 2)))); - ConsumerGroupHeartbeatResponse heartbeatResponse = - createConsumerGroupHeartbeatResponse(targetAssignment); - membershipManager.updateState(heartbeatResponse.data()); - - // Target assignment should be in the process of being reconciled - checkAssignments(membershipManager, null, targetAssignment); - // Mark assignment processing completed - ConsumerGroupHeartbeatResponseData.Assignment reconciled = - new ConsumerGroupHeartbeatResponseData.Assignment() - .setTopicPartitions(Collections.singletonList( - new ConsumerGroupHeartbeatResponseData.TopicPartitions() - .setTopicId(Uuid.randomUuid()) - .setPartitions(Collections.singletonList(0)))); - assertThrows(IllegalStateException.class, () -> membershipManager.onTargetAssignmentProcessComplete(reconciled)); - } - - private void checkAssignments( - MembershipManagerImpl membershipManager, - ConsumerGroupHeartbeatResponseData.Assignment expectedCurrentAssignment, - ConsumerGroupHeartbeatResponseData.Assignment expectedTargetAssignment) { + .setTopicId(topicId) + .setPartitions(Arrays.asList(0, 1)))); + MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false, targetAssignment); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + // Should not trigger reconciliation, and request a metadata update. + verifyReconciliationNotTriggered(membershipManager); + assertEquals(Collections.singleton(topicId), membershipManager.topicsWaitingForMetadata()); + verify(metadata).requestUpdate(anyBoolean()); + + String topicName = "topic1"; + mockTopicNameInMetadataCache(Collections.singletonMap(topicId, topicName), true); + + // When metadata is updated, the member should re-trigger reconciliation + membershipManager.onUpdate(null); + Set expectedAssignmentReconciled = new HashSet<>(Arrays.asList( + new TopicIdPartition(topicId, new TopicPartition(topicName, 0)), + new TopicIdPartition(topicId, new TopicPartition(topicName, 1)))); + verifyReconciliationTriggeredAndCompleted(membershipManager, expectedAssignmentReconciled); + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + assertTrue(membershipManager.topicsWaitingForMetadata().isEmpty()); + } + + @Test + public void testMetadataUpdatesRequestsAnotherUpdateIfNeeded() { + Uuid topicId = Uuid.randomUuid(); + + // Assignment not in metadata + ConsumerGroupHeartbeatResponseData.Assignment targetAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(Collections.singletonList( + new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topicId) + .setPartitions(Arrays.asList(0, 1)))); + MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false, targetAssignment); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + // Should not trigger reconciliation, and request a metadata update. + verifyReconciliationNotTriggered(membershipManager); + assertEquals(Collections.singleton(topicId), membershipManager.topicsWaitingForMetadata()); + verify(metadata).requestUpdate(anyBoolean()); + + // Metadata update received, but still without the unresolved topic in it. Should keep + // the unresolved and request update again. + when(metadata.topicNames()).thenReturn(Collections.emptyMap()); + membershipManager.onUpdate(null); + verifyReconciliationNotTriggered(membershipManager); + assertEquals(Collections.singleton(topicId), membershipManager.topicsWaitingForMetadata()); + verify(metadata, times(2)).requestUpdate(anyBoolean()); + } + + @Test + public void testRevokePartitionsUsesTopicNamesLocalCacheWhenMetadataNotAvailable() { + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic1"; + + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true); + + // Member received assignment to reconcile; + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager); + + // Member should complete reconciliation + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + Set assignedPartitions = new HashSet<>(Arrays.asList( + new TopicPartition(topicName, 0), + new TopicPartition(topicName, 1))); + Set assignedTopicIdPartitions = + assignedPartitions.stream().map(tp -> new TopicIdPartition(topicId, tp)).collect(Collectors.toSet()); + assertEquals(assignedTopicIdPartitions, membershipManager.currentAssignment()); + assertFalse(membershipManager.reconciliationInProgress()); + + mockAckSent(membershipManager); + when(subscriptionState.assignedPartitions()).thenReturn(assignedPartitions); + + // Revocation of topic not found in metadata cache + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + mockRevocationNoCallbacks(false); + mockTopicNameInMetadataCache(Collections.singletonMap(topicId, topicName), false); + + // Revoke one of the 2 partitions + receiveAssignment(topicId, Collections.singletonList(1), membershipManager); + + // Revocation should complete without requesting any metadata update given that the topic + // received in target assignment should exist in local topic name cache. + verify(metadata, never()).requestUpdate(anyBoolean()); + Set remainingAssignment = Collections.singleton( + new TopicIdPartition(topicId, new TopicPartition(topicName, 1))); + + testRevocationCompleted(membershipManager, remainingAssignment); + } + + @Test + public void testOnSubscriptionUpdatedTransitionsToJoiningOnlyIfNotInGroup() { + MembershipManagerImpl membershipManager = createMemberInStableState(); + verify(membershipManager).transitionToJoining(); + clearInvocations(membershipManager); + membershipManager.onSubscriptionUpdated(); + verify(membershipManager, never()).transitionToJoining(); + } + + private MembershipManagerImpl mockMemberSuccessfullyReceivesAndAcksAssignment( + Uuid topicId, String topicName, List partitions) { + mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true); + + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + receiveAssignment(topicId, partitions, membershipManager); + + Set assignedPartitions = new HashSet<>(); + partitions.forEach(tp -> assignedPartitions.add(new TopicIdPartition( + topicId, + new TopicPartition(topicName, tp)))); + verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions); + return membershipManager; + } + + private CompletableFuture mockEmptyAssignmentAndRevocationStuckOnCommit( + MembershipManagerImpl membershipManager) { + CompletableFuture commitResult = mockRevocationNoCallbacks(true); + receiveEmptyAssignment(membershipManager); + verifyReconciliationTriggered(membershipManager); + clearInvocations(membershipManager); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + return commitResult; + } + + private CompletableFuture mockNewAssignmentAndRevocationStuckOnCommit( + MembershipManagerImpl membershipManager, Uuid topicId, String topicName, + List partitions, boolean mockMetadata) { + CompletableFuture commitResult = mockRevocationNoCallbacks(true); + if (mockMetadata) { + when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName)); + } + receiveAssignment(topicId, partitions, membershipManager); + verifyReconciliationTriggered(membershipManager); + clearInvocations(membershipManager); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + + return commitResult; + } + + private void verifyReconciliationTriggered(MembershipManagerImpl membershipManager) { + verify(membershipManager).markReconciliationInProgress(); + assertEquals(MemberState.RECONCILING, membershipManager.state()); + } + + private void verifyReconciliationNotTriggered(MembershipManagerImpl membershipManager) { + verify(membershipManager, never()).markReconciliationInProgress(); + verify(membershipManager, never()).markReconciliationCompleted(); + } + + private void verifyReconciliationTriggeredAndCompleted(MembershipManagerImpl membershipManager, + Set expectedAssignment) { + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); + verify(membershipManager).markReconciliationInProgress(); + verify(membershipManager).markReconciliationCompleted(); + assertFalse(membershipManager.reconciliationInProgress()); + + // Assignment applied + Set expectedTopicPartitions = buildTopicPartitions(expectedAssignment); + verify(subscriptionState).assignFromSubscribed(expectedTopicPartitions); + // Set topicIdPartitions = buildTopicIdPartitions(expectedAssignment); + assertEquals(expectedAssignment, membershipManager.currentAssignment()); + + verify(commitRequestManager).resetAutoCommitTimer(); + } + + private Set buildTopicPartitions(Set topicIdPartitions) { + return topicIdPartitions.stream().map(TopicIdPartition::topicPartition).collect(Collectors.toSet()); + } + + private void mockAckSent(MembershipManagerImpl membershipManager) { + membershipManager.onHeartbeatRequestSent(); + } + + private void mockTopicNameInMetadataCache(Map topicNames, boolean isPresent) { + if (isPresent) { + when(metadata.topicNames()).thenReturn(topicNames); + } else { + when(metadata.topicNames()).thenReturn(Collections.emptyMap()); + } + } + + private CompletableFuture mockRevocationNoCallbacks(boolean withAutoCommit) { + doNothing().when(subscriptionState).markPendingRevocation(anySet()); + when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty()).thenReturn(Optional.empty()); + if (withAutoCommit) { + when(commitRequestManager.autoCommitEnabled()).thenReturn(true); + CompletableFuture commitResult = new CompletableFuture<>(); + when(commitRequestManager.maybeAutoCommitAllConsumed()).thenReturn(commitResult); + return commitResult; + } else { + return CompletableFuture.completedFuture(null); + } + } + + private void mockMemberHasAutoAssignedPartition() { + String topicName = "topic1"; + TopicPartition ownedPartition = new TopicPartition(topicName, 0); + when(subscriptionState.assignedPartitions()).thenReturn(Collections.singleton(ownedPartition)); + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty()).thenReturn(Optional.empty()); + } + + private void testRevocationOfAllPartitionsCompleted(MembershipManagerImpl membershipManager) { + testRevocationCompleted(membershipManager, Collections.emptySet()); + } + + private void testRevocationCompleted(MembershipManagerImpl membershipManager, + Set expectedCurrentAssignment) { + assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state()); assertEquals(expectedCurrentAssignment, membershipManager.currentAssignment()); - assertEquals(expectedTargetAssignment, membershipManager.targetAssignment().orElse(null)); + assertFalse(membershipManager.reconciliationInProgress()); + + verify(subscriptionState).markPendingRevocation(anySet()); + Set expectedTopicPartitionAssignment = + buildTopicPartitions(expectedCurrentAssignment); + verify(subscriptionState).assignFromSubscribed(expectedTopicPartitionAssignment); + } + + private void mockOwnedPartitionAndAssignmentReceived(Uuid topicId, + String topicName, + Set previouslyOwned, + boolean mockMetadata) { + when(subscriptionState.assignedPartitions()).thenReturn(previouslyOwned); + if (mockMetadata) { + when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName)); + } + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty()).thenReturn(Optional.empty()); + } + + private void mockOwnedPartition(String topic, int partition) { + TopicPartition previouslyOwned = new TopicPartition(topic, partition); + when(subscriptionState.assignedPartitions()).thenReturn(Collections.singleton(previouslyOwned)); + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + } + + private MembershipManagerImpl mockJoinAndReceiveAssignment(boolean expectSubscriptionUpdated) { + return mockJoinAndReceiveAssignment(expectSubscriptionUpdated, createAssignment(expectSubscriptionUpdated)); + } + + private MembershipManagerImpl mockJoinAndReceiveAssignment(boolean expectSubscriptionUpdated, + ConsumerGroupHeartbeatResponseData.Assignment assignment) { + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(assignment); + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty()).thenReturn(Optional.empty()); + + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); + if (expectSubscriptionUpdated) { + verify(subscriptionState).assignFromSubscribed(anyCollection()); + } else { + verify(subscriptionState, never()).assignFromSubscribed(anyCollection()); + } + + return membershipManager; + } + + private MembershipManagerImpl createMemberInStableState() { + MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(); + ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); + assertEquals(MemberState.STABLE, membershipManager.state()); + return membershipManager; + } + + private void receiveAssignment(Uuid topicId, List partitions, MembershipManager membershipManager) { + ConsumerGroupHeartbeatResponseData.Assignment targetAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(Collections.singletonList( + new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topicId) + .setPartitions(partitions))); + ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(targetAssignment); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); + } + + private void receiveAssignmentAfterRejoin(Uuid topicId, List partitions, MembershipManager membershipManager) { + ConsumerGroupHeartbeatResponseData.Assignment targetAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(Collections.singletonList( + new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topicId) + .setPartitions(partitions))); + ConsumerGroupHeartbeatResponse heartbeatResponse = + createConsumerGroupHeartbeatResponseWithBumpedEpoch(targetAssignment); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); + } + + private void receiveEmptyAssignment(MembershipManager membershipManager) { + // New empty assignment received, revoking owned partition. + ConsumerGroupHeartbeatResponseData.Assignment targetAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(Collections.emptyList()); + ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(targetAssignment); + membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data()); } - private void testStateUpdateOnFenceError(MembershipManager membershipManager) { + /** + * Fenced member should release assignment, reset epoch to 0, keep member ID, and transition + * to JOINING to rejoin the group. + */ + private void testFencedMemberReleasesAssignmentAndTransitionsToJoining(MembershipManager membershipManager) { + mockMemberHasAutoAssignedPartition(); + membershipManager.transitionToFenced(); - assertEquals(MemberState.FENCED, membershipManager.state()); - // Should reset member epoch and keep member id - assertFalse(membershipManager.memberId().isEmpty()); + + assertEquals(MEMBER_ID, membershipManager.memberId()); assertEquals(0, membershipManager.memberEpoch()); + assertEquals(MemberState.JOINING, membershipManager.state()); + } + + /** + * Member that intentionally leaves the group (via unsubscribe) should release assignment, + * reset epoch to -1, keep member ID, and transition to {@link MemberState#LEAVING} to send out a + * heartbeat with the leave epoch. Once the heartbeat request is sent out, the member should + * transition to {@link MemberState#UNSUBSCRIBED} + */ + private void testLeaveGroupReleasesAssignmentAndResetsEpochToSendLeaveGroup(MembershipManager membershipManager) { + mockLeaveGroup(); + + CompletableFuture leaveResult = membershipManager.leaveGroup(); + + assertEquals(MemberState.LEAVING, membershipManager.state()); + assertFalse(leaveResult.isDone(), "Leave group result should not complete until the " + + "heartbeat request to leave is sent out."); + + membershipManager.onHeartbeatRequestSent(); + + assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state()); + assertTrue(leaveResult.isDone()); + assertFalse(leaveResult.isCompletedExceptionally()); + assertEquals(MEMBER_ID, membershipManager.memberId()); + assertEquals(-1, membershipManager.memberEpoch()); + assertTrue(membershipManager.currentAssignment().isEmpty()); + verify(subscriptionState).assignFromSubscribed(Collections.emptySet()); + } + + private void mockLeaveGroup() { + mockMemberHasAutoAssignedPartition(); + doNothing().when(subscriptionState).markPendingRevocation(anySet()); } private void testStateUpdateOnFatalFailure(MembershipManager membershipManager) { - String initialMemberId = membershipManager.memberId(); - int initialMemberEpoch = membershipManager.memberEpoch(); - membershipManager.transitionToFailed(); - assertEquals(MemberState.FAILED, membershipManager.state()); - // Should not reset member id or epoch - assertEquals(initialMemberId, membershipManager.memberId()); - assertEquals(initialMemberEpoch, membershipManager.memberEpoch()); + String memberId = membershipManager.memberId(); + int lastEpoch = membershipManager.memberEpoch(); + when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true); + membershipManager.transitionToFatal(); + assertEquals(MemberState.FATAL, membershipManager.state()); + // Should keep its last member id and epoch + assertEquals(memberId, membershipManager.memberId()); + assertEquals(lastEpoch, membershipManager.memberEpoch()); } - private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponse(ConsumerGroupHeartbeatResponseData.Assignment assignment) { + private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponse( + ConsumerGroupHeartbeatResponseData.Assignment assignment) { return new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() .setErrorCode(Errors.NONE.code()) .setMemberId(MEMBER_ID) @@ -249,6 +1051,20 @@ private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponse(Cons .setAssignment(assignment)); } + /** + * Create heartbeat response with the given assignment and a bumped epoch (incrementing by 1 + * as default but could be any increment). This will be used to mock when a member + * receives a heartbeat response to the join request, and the response includes an assignment. + */ + private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponseWithBumpedEpoch( + ConsumerGroupHeartbeatResponseData.Assignment assignment) { + return new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() + .setErrorCode(Errors.NONE.code()) + .setMemberId(MEMBER_ID) + .setMemberEpoch(MEMBER_EPOCH + 1) + .setAssignment(assignment)); + } + private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponseWithError(Errors error) { return new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() .setErrorCode(error.code()) @@ -256,14 +1072,22 @@ private ConsumerGroupHeartbeatResponse createConsumerGroupHeartbeatResponseWithE .setMemberEpoch(5)); } - private ConsumerGroupHeartbeatResponseData.Assignment createAssignment() { + private ConsumerGroupHeartbeatResponseData.Assignment createAssignment(boolean mockMetadata) { + Uuid topic1 = Uuid.randomUuid(); + Uuid topic2 = Uuid.randomUuid(); + if (mockMetadata) { + Map topicNames = new HashMap<>(); + topicNames.put(topic1, "topic1"); + topicNames.put(topic2, "topic2"); + when(metadata.topicNames()).thenReturn(topicNames); + } return new ConsumerGroupHeartbeatResponseData.Assignment() .setTopicPartitions(Arrays.asList( new ConsumerGroupHeartbeatResponseData.TopicPartitions() - .setTopicId(Uuid.randomUuid()) + .setTopicId(topic1) .setPartitions(Arrays.asList(0, 1, 2)), new ConsumerGroupHeartbeatResponseData.TopicPartitions() - .setTopicId(Uuid.randomUuid()) + .setTopicId(topic2) .setPartitions(Arrays.asList(3, 4, 5)) )); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegateTest.java index aaa45ad7435b4..4fdcf917d6c35 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegateTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegateTest.java @@ -44,6 +44,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -66,7 +67,7 @@ public void testSuccessfulResponse() throws Exception { NetworkClientDelegate.UnsentRequest unsentRequest = newUnsentFindCoordinatorRequest(); prepareFindCoordinatorResponse(Errors.NONE); - ncd.send(unsentRequest); + ncd.add(unsentRequest); ncd.poll(0, time.milliseconds()); assertTrue(unsentRequest.future().isDone()); @@ -79,7 +80,7 @@ public void testTimeoutBeforeSend() throws Exception { try (NetworkClientDelegate ncd = newNetworkClientDelegate()) { client.setUnreachable(mockNode(), REQUEST_TIMEOUT_MS); NetworkClientDelegate.UnsentRequest unsentRequest = newUnsentFindCoordinatorRequest(); - ncd.send(unsentRequest); + ncd.add(unsentRequest); ncd.poll(0, time.milliseconds()); time.sleep(REQUEST_TIMEOUT_MS); ncd.poll(0, time.milliseconds()); @@ -92,7 +93,7 @@ public void testTimeoutBeforeSend() throws Exception { public void testTimeoutAfterSend() throws Exception { try (NetworkClientDelegate ncd = newNetworkClientDelegate()) { NetworkClientDelegate.UnsentRequest unsentRequest = newUnsentFindCoordinatorRequest(); - ncd.send(unsentRequest); + ncd.add(unsentRequest); ncd.poll(0, time.milliseconds()); time.sleep(REQUEST_TIMEOUT_MS); ncd.poll(0, time.milliseconds()); @@ -122,6 +123,23 @@ public void testEnsureCorrectCompletionTimeOnComplete() { assertEquals(timeMs, unsentRequest.handler().completionTimeMs()); } + @Test + public void testEnsureTimerSetOnAdd() { + NetworkClientDelegate ncd = newNetworkClientDelegate(); + NetworkClientDelegate.UnsentRequest findCoordRequest = newUnsentFindCoordinatorRequest(); + assertNull(findCoordRequest.timer()); + + // NetworkClientDelegate#add + ncd.add(findCoordRequest); + assertEquals(1, ncd.unsentRequests().size()); + assertEquals(REQUEST_TIMEOUT_MS, ncd.unsentRequests().poll().timer().timeoutMs()); + + // NetworkClientDelegate#addAll + ncd.addAll(Collections.singletonList(findCoordRequest)); + assertEquals(1, ncd.unsentRequests().size()); + assertEquals(REQUEST_TIMEOUT_MS, ncd.unsentRequests().poll().timer().timeoutMs()); + } + public NetworkClientDelegate newNetworkClientDelegate() { LogContext logContext = new LogContext(); Properties properties = new Properties(); @@ -141,7 +159,6 @@ public NetworkClientDelegate.UnsentRequest newUnsentFindCoordinatorRequest() { ), Optional.empty() ); - req.setTimer(this.time, REQUEST_TIMEOUT_MS); return req; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java deleted file mode 100644 index d67f509d905c9..0000000000000 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals; - -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.OffsetAndTimestamp; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; -import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; -import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; -import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.InvalidGroupIdException; -import org.apache.kafka.common.errors.TimeoutException; -import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.requests.ListOffsetsRequest; -import org.apache.kafka.common.utils.Timer; -import org.apache.kafka.test.TestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentMatchers; -import org.mockito.MockedConstruction; -import org.mockito.stubbing.Answer; - -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockConstruction; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class PrototypeAsyncConsumerTest { - - private PrototypeAsyncConsumer consumer; - private ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder testBuilder; - private ApplicationEventHandler applicationEventHandler; - - @BeforeEach - public void setup() { - // By default, the consumer is part of a group. - setup(ConsumerTestBuilder.createDefaultGroupInformation()); - } - - private void setup(Optional groupInfo) { - testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupInfo); - applicationEventHandler = testBuilder.applicationEventHandler; - consumer = testBuilder.consumer; - } - - @AfterEach - public void cleanup() { - if (testBuilder != null) { - testBuilder.close(); - } - } - - private void resetWithEmptyGroupId() { - // Create a consumer that is not configured as part of a group. - cleanup(); - setup(Optional.empty()); - } - - @Test - public void testSuccessfulStartupShutdown() { - assertDoesNotThrow(() -> consumer.close()); - } - - @Test - public void testInvalidGroupId() { - // Create consumer without group id - resetWithEmptyGroupId(); - assertThrows(InvalidGroupIdException.class, () -> consumer.committed(new HashSet<>())); - } - - @Test - public void testCommitAsync_NullCallback() throws InterruptedException { - CompletableFuture future = new CompletableFuture<>(); - Map offsets = new HashMap<>(); - offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); - offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - - doReturn(future).when(consumer).commit(offsets, false); - consumer.commitAsync(offsets, null); - future.complete(null); - TestUtils.waitForCondition(future::isDone, - 2000, - "commit future should complete"); - - assertFalse(future.isCompletedExceptionally()); - } - - @Test - public void testCommitAsync_UserSuppliedCallback() { - CompletableFuture future = new CompletableFuture<>(); - - Map offsets = new HashMap<>(); - offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); - offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - - doReturn(future).when(consumer).commit(offsets, false); - OffsetCommitCallback customCallback = mock(OffsetCommitCallback.class); - consumer.commitAsync(offsets, customCallback); - future.complete(null); - verify(customCallback).onComplete(offsets, null); - } - - @Test - public void testCommitted() { - Map offsets = mockTopicPartitionOffset(); - CompletableFuture> committedFuture = new CompletableFuture<>(); - committedFuture.complete(offsets); - - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - assertDoesNotThrow(() -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); - verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - } - } - - @Test - public void testCommitted_ExceptionThrown() { - Map offsets = mockTopicPartitionOffset(); - CompletableFuture> committedFuture = new CompletableFuture<>(); - committedFuture.completeExceptionally(new KafkaException("Test exception")); - - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - assertThrows(KafkaException.class, () -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); - verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - } - } - - /** - * This is a rather ugly bit of code. Not my choice :( - * - *

    - * - * Inside the {@link org.apache.kafka.clients.consumer.Consumer#committed(Set, Duration)} call we create an - * instance of {@link OffsetFetchApplicationEvent} that holds the partitions and internally holds a - * {@link CompletableFuture}. We want to test different behaviours of the {@link Future#get()}, such as - * returning normally, timing out, throwing an error, etc. By mocking the construction of the event object that - * is created, we can affect that behavior. - */ - private static MockedConstruction offsetFetchEventMocker(CompletableFuture> future) { - // This "answer" is where we pass the future to be invoked by the ConsumerUtils.getResult() method - Answer> getInvocationAnswer = invocation -> { - // This argument captures the actual argument value that was passed to the event's get() method, so we - // just "forward" that value to our mocked call - Timer timer = invocation.getArgument(0); - return ConsumerUtils.getResult(future, timer); - }; - - MockedConstruction.MockInitializer mockInitializer = (mock, ctx) -> { - // When the event's get() method is invoked, we call the "answer" method just above - when(mock.get(any())).thenAnswer(getInvocationAnswer); - - // When the event's type() method is invoked, we have to return the type as it will be null in the mock - when(mock.type()).thenReturn(ApplicationEvent.Type.FETCH_COMMITTED_OFFSET); - - // This is needed for the WakeupTrigger code that keeps track of the active task - when(mock.future()).thenReturn(future); - }; - - return mockConstruction(OffsetFetchApplicationEvent.class, mockInitializer); - } - - @Test - public void testAssign() { - final TopicPartition tp = new TopicPartition("foo", 3); - consumer.assign(singleton(tp)); - assertTrue(consumer.subscription().isEmpty()); - assertTrue(consumer.assignment().contains(tp)); - verify(applicationEventHandler).add(any(AssignmentChangeApplicationEvent.class)); - verify(applicationEventHandler).add(any(NewTopicsMetadataUpdateRequestEvent.class)); - } - - @Test - public void testAssignOnNullTopicPartition() { - assertThrows(IllegalArgumentException.class, () -> consumer.assign(null)); - } - - @Test - public void testAssignOnEmptyTopicPartition() { - consumer.assign(Collections.emptyList()); - assertTrue(consumer.subscription().isEmpty()); - assertTrue(consumer.assignment().isEmpty()); - } - - @Test - public void testAssignOnNullTopicInPartition() { - assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(null, 0)))); - } - - @Test - public void testAssignOnEmptyTopicInPartition() { - assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(" ", 0)))); - } - - @Test - public void testBeginningOffsetsFailsIfNullPartitions() { - assertThrows(NullPointerException.class, () -> consumer.beginningOffsets(null, - Duration.ofMillis(1))); - } - - @Test - public void testBeginningOffsets() { - Map expectedOffsetsAndTimestamp = - mockOffsetAndTimestamp(); - Set partitions = expectedOffsetsAndTimestamp.keySet(); - doReturn(expectedOffsetsAndTimestamp).when(applicationEventHandler).addAndGet(any(), any()); - Map result = - assertDoesNotThrow(() -> consumer.beginningOffsets(partitions, - Duration.ofMillis(1))); - Map expectedOffsets = expectedOffsetsAndTimestamp.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); - assertEquals(expectedOffsets, result); - verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), - ArgumentMatchers.isA(Timer.class)); - } - - @Test - public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailure() { - Set partitions = mockTopicPartitionOffset().keySet(); - Throwable eventProcessingFailure = new KafkaException("Unexpected failure " + - "processing List Offsets event"); - doThrow(eventProcessingFailure).when(applicationEventHandler).addAndGet(any(), any()); - Throwable consumerError = assertThrows(KafkaException.class, - () -> consumer.beginningOffsets(partitions, - Duration.ofMillis(1))); - assertEquals(eventProcessingFailure, consumerError); - verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); - } - - @Test - public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { - doThrow(new TimeoutException()).when(applicationEventHandler).addAndGet(any(), any()); - assertThrows(TimeoutException.class, - () -> consumer.beginningOffsets( - Collections.singletonList(new TopicPartition("t1", 0)), - Duration.ofMillis(1))); - verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), - ArgumentMatchers.isA(Timer.class)); - } - - @Test - public void testOffsetsForTimesOnNullPartitions() { - assertThrows(NullPointerException.class, () -> consumer.offsetsForTimes(null, - Duration.ofMillis(1))); - } - - @Test - public void testOffsetsForTimesFailsOnNegativeTargetTimes() { - assertThrows(IllegalArgumentException.class, - () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( - "topic1", 1), ListOffsetsRequest.EARLIEST_TIMESTAMP), - Duration.ofMillis(1))); - - assertThrows(IllegalArgumentException.class, - () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( - "topic1", 1), ListOffsetsRequest.LATEST_TIMESTAMP), - Duration.ofMillis(1))); - - assertThrows(IllegalArgumentException.class, - () -> consumer.offsetsForTimes(Collections.singletonMap(new TopicPartition( - "topic1", 1), ListOffsetsRequest.MAX_TIMESTAMP), - Duration.ofMillis(1))); - } - - @Test - public void testOffsetsForTimes() { - Map expectedResult = mockOffsetAndTimestamp(); - Map timestampToSearch = mockTimestampToSearch(); - - doReturn(expectedResult).when(applicationEventHandler).addAndGet(any(), any()); - Map result = - assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, Duration.ofMillis(1))); - assertEquals(expectedResult, result); - verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), - ArgumentMatchers.isA(Timer.class)); - } - - // This test ensures same behaviour as the current consumer when offsetsForTimes is called - // with 0 timeout. It should return map with all requested partitions as keys, with null - // OffsetAndTimestamp as value. - @Test - public void testOffsetsForTimesWithZeroTimeout() { - TopicPartition tp = new TopicPartition("topic1", 0); - Map expectedResult = - Collections.singletonMap(tp, null); - Map timestampToSearch = Collections.singletonMap(tp, 5L); - - Map result = - assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, - Duration.ofMillis(0))); - assertEquals(expectedResult, result); - verify(applicationEventHandler, never()).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), - ArgumentMatchers.isA(Timer.class)); - } - - @Test - public void testWakeup_committed() { - consumer.wakeup(); - assertThrows(WakeupException.class, () -> consumer.committed(mockTopicPartitionOffset().keySet())); - assertNoPendingWakeup(consumer.wakeupTrigger()); - } - - @Test - public void testRefreshCommittedOffsetsSuccess() { - TopicPartition partition = new TopicPartition("t1", 1); - Set partitions = Collections.singleton(partition); - Map committedOffsets = Collections.singletonMap(partition, new OffsetAndMetadata(10L)); - testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); - } - - @Test - public void testRefreshCommittedOffsetsSuccessButNoCommittedOffsetsFound() { - TopicPartition partition = new TopicPartition("t1", 1); - Set partitions = Collections.singleton(partition); - Map committedOffsets = Collections.emptyMap(); - testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); - } - - @Test - public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { - testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(true); - } - - @Test - public void testRefreshCommittedOffsetsNotCalledIfNoGroupId() { - // Create consumer without group id so committed offsets are not used for updating positions - resetWithEmptyGroupId(); - testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(false); - } - - private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean committedOffsetsEnabled) { - // Uncompleted future that will time out if used - CompletableFuture> committedFuture = new CompletableFuture<>(); - - - consumer.assign(singleton(new TopicPartition("t1", 1))); - - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - // Poll with 0 timeout to run a single iteration of the poll loop - consumer.poll(Duration.ofMillis(0)); - - verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); - - if (committedOffsetsEnabled) { - // Verify there was an OffsetFetch event and no ResetPositions event - verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(applicationEventHandler, - never()).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); - } else { - // Verify there was not any OffsetFetch event but there should be a ResetPositions - verify(applicationEventHandler, - never()).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); - } - } - } - - private void testRefreshCommittedOffsetsSuccess(Set partitions, - Map committedOffsets) { - CompletableFuture> committedFuture = new CompletableFuture<>(); - committedFuture.complete(committedOffsets); - consumer.assign(partitions); - - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - // Poll with 0 timeout to run a single iteration of the poll loop - consumer.poll(Duration.ofMillis(0)); - - verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); - verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); - } - } - - private void assertNoPendingWakeup(final WakeupTrigger wakeupTrigger) { - assertNull(wakeupTrigger.getPendingTask()); - } - - private HashMap mockTopicPartitionOffset() { - final TopicPartition t0 = new TopicPartition("t0", 2); - final TopicPartition t1 = new TopicPartition("t0", 3); - HashMap topicPartitionOffsets = new HashMap<>(); - topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L)); - topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L)); - return topicPartitionOffsets; - } - - private HashMap mockOffsetAndTimestamp() { - final TopicPartition t0 = new TopicPartition("t0", 2); - final TopicPartition t1 = new TopicPartition("t0", 3); - HashMap offsetAndTimestamp = new HashMap<>(); - offsetAndTimestamp.put(t0, new OffsetAndTimestamp(5L, 1L)); - offsetAndTimestamp.put(t1, new OffsetAndTimestamp(6L, 3L)); - return offsetAndTimestamp; - } - - private HashMap mockTimestampToSearch() { - final TopicPartition t0 = new TopicPartition("t0", 2); - final TopicPartition t1 = new TopicPartition("t0", 3); - HashMap timestampToSearch = new HashMap<>(); - timestampToSearch.put(t0, 1L); - timestampToSearch.put(t1, 2L); - return timestampToSearch; - } -} - diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManagerTest.java index 39428d1bd8efe..e72172e1ac29c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManagerTest.java @@ -112,8 +112,7 @@ public void testExceptionAndInflightRequests(final Errors error, final boolean s @MethodSource("topicsProvider") public void testSendingTheSameRequest(Optional topic) { CompletableFuture>> future = this.topicMetadataRequestManager.requestTopicMetadata(topic); - CompletableFuture>> future2 = - this.topicMetadataRequestManager.requestTopicMetadata(topic); + CompletableFuture>> future2 = this.topicMetadataRequestManager.requestTopicMetadata(topic); this.time.sleep(100); NetworkClientDelegate.PollResult res = this.topicMetadataRequestManager.poll(this.time.milliseconds()); assertEquals(1, res.unsentRequests.size()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/WakeupTriggerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/WakeupTriggerTest.java index bb3dfefdadcb1..235ec78168d5e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/WakeupTriggerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/WakeupTriggerTest.java @@ -19,17 +19,26 @@ import org.apache.kafka.common.errors.WakeupException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +@MockitoSettings(strictness = Strictness.STRICT_STUBS) public class WakeupTriggerTest { - private static long defaultTimeoutMs = 1000; + private final static long DEFAULT_TIMEOUT_MS = 1000; private WakeupTrigger wakeupTrigger; @BeforeEach @@ -59,14 +68,75 @@ public void testSettingActiveFutureAfterWakeupShouldThrow() { public void testUnsetActiveFuture() { CompletableFuture task = new CompletableFuture<>(); wakeupTrigger.setActiveTask(task); - wakeupTrigger.clearActiveTask(); + wakeupTrigger.clearTask(); assertNull(wakeupTrigger.getPendingTask()); } + @Test + public void testSettingFetchAction() { + try (final FetchBuffer fetchBuffer = mock(FetchBuffer.class)) { + wakeupTrigger.setFetchAction(fetchBuffer); + + final WakeupTrigger.Wakeupable wakeupable = wakeupTrigger.getPendingTask(); + assertInstanceOf(WakeupTrigger.FetchAction.class, wakeupable); + assertEquals(fetchBuffer, ((WakeupTrigger.FetchAction) wakeupable).fetchBuffer()); + } + } + + @Test + public void testUnsetFetchAction() { + try (final FetchBuffer fetchBuffer = mock(FetchBuffer.class)) { + wakeupTrigger.setFetchAction(fetchBuffer); + + wakeupTrigger.clearTask(); + + assertNull(wakeupTrigger.getPendingTask()); + } + } + + @Test + public void testWakeupFromFetchAction() { + try (final FetchBuffer fetchBuffer = mock(FetchBuffer.class)) { + wakeupTrigger.setFetchAction(fetchBuffer); + + wakeupTrigger.wakeup(); + + verify(fetchBuffer).wakeup(); + final WakeupTrigger.Wakeupable wakeupable = wakeupTrigger.getPendingTask(); + assertInstanceOf(WakeupTrigger.WakeupFuture.class, wakeupable); + } + } + + @Test + public void testManualTriggerWhenWakeupCalled() { + wakeupTrigger.wakeup(); + assertThrows(WakeupException.class, () -> wakeupTrigger.maybeTriggerWakeup()); + } + + @Test + public void testManualTriggerWhenWakeupNotCalled() { + assertDoesNotThrow(() -> wakeupTrigger.maybeTriggerWakeup()); + } + + @Test + public void testManualTriggerWhenWakeupCalledAndActiveTaskSet() { + final CompletableFuture future = new CompletableFuture<>(); + wakeupTrigger.setActiveTask(future); + assertDoesNotThrow(() -> wakeupTrigger.maybeTriggerWakeup()); + } + + @Test + public void testManualTriggerWhenWakeupCalledAndFetchActionSet() { + try (final FetchBuffer fetchBuffer = mock(FetchBuffer.class)) { + wakeupTrigger.setFetchAction(fetchBuffer); + assertDoesNotThrow(() -> wakeupTrigger.maybeTriggerWakeup()); + } + } + private void assertWakeupExceptionIsThrown(final CompletableFuture future) { assertTrue(future.isCompletedExceptionally()); try { - future.get(defaultTimeoutMs, TimeUnit.MILLISECONDS); + future.get(DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof WakeupException); return; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java deleted file mode 100644 index 0670b8bdb7c3f..0000000000000 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder; -import org.apache.kafka.common.KafkaException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.BlockingQueue; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -public class BackgroundEventHandlerTest { - - private ConsumerTestBuilder testBuilder; - private BlockingQueue backgroundEventQueue; - private BackgroundEventHandler backgroundEventHandler; - private BackgroundEventProcessor backgroundEventProcessor; - - @BeforeEach - public void setup() { - testBuilder = new ConsumerTestBuilder(); - backgroundEventQueue = testBuilder.backgroundEventQueue; - backgroundEventHandler = testBuilder.backgroundEventHandler; - backgroundEventProcessor = testBuilder.backgroundEventProcessor; - } - - @AfterEach - public void tearDown() { - if (testBuilder != null) - testBuilder.close(); - } - - @Test - public void testNoEvents() { - assertTrue(backgroundEventQueue.isEmpty()); - backgroundEventProcessor.process((event, error) -> { }); - assertTrue(backgroundEventQueue.isEmpty()); - } - - @Test - public void testSingleEvent() { - BackgroundEvent event = new ErrorBackgroundEvent(new RuntimeException("A")); - backgroundEventQueue.add(event); - assertPeeked(event); - backgroundEventProcessor.process((e, error) -> { }); - assertTrue(backgroundEventQueue.isEmpty()); - } - - @Test - public void testSingleErrorEvent() { - KafkaException error = new KafkaException("error"); - BackgroundEvent event = new ErrorBackgroundEvent(error); - backgroundEventHandler.add(new ErrorBackgroundEvent(error)); - assertPeeked(event); - assertProcessThrows(error); - } - - @Test - public void testMultipleEvents() { - BackgroundEvent event1 = new ErrorBackgroundEvent(new RuntimeException("A")); - backgroundEventQueue.add(event1); - backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("B"))); - backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); - - assertPeeked(event1); - backgroundEventProcessor.process((event, error) -> { }); - assertTrue(backgroundEventQueue.isEmpty()); - } - - @Test - public void testMultipleErrorEvents() { - Throwable error1 = new Throwable("error1"); - KafkaException error2 = new KafkaException("error2"); - KafkaException error3 = new KafkaException("error3"); - - backgroundEventHandler.add(new ErrorBackgroundEvent(error1)); - backgroundEventHandler.add(new ErrorBackgroundEvent(error2)); - backgroundEventHandler.add(new ErrorBackgroundEvent(error3)); - - assertProcessThrows(new KafkaException(error1)); - } - - @Test - public void testMixedEventsWithErrorEvents() { - Throwable error1 = new Throwable("error1"); - KafkaException error2 = new KafkaException("error2"); - KafkaException error3 = new KafkaException("error3"); - - RuntimeException errorToCheck = new RuntimeException("A"); - backgroundEventQueue.add(new ErrorBackgroundEvent(errorToCheck)); - backgroundEventHandler.add(new ErrorBackgroundEvent(error1)); - backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("B"))); - backgroundEventHandler.add(new ErrorBackgroundEvent(error2)); - backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); - backgroundEventHandler.add(new ErrorBackgroundEvent(error3)); - backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("D"))); - - assertProcessThrows(new KafkaException(errorToCheck)); - } - - private void assertPeeked(BackgroundEvent event) { - BackgroundEvent peekEvent = backgroundEventQueue.peek(); - assertNotNull(peekEvent); - assertEquals(event, peekEvent); - } - - private void assertProcessThrows(Throwable error) { - assertFalse(backgroundEventQueue.isEmpty()); - - try { - backgroundEventProcessor.process(); - fail("Should have thrown error: " + error); - } catch (Throwable t) { - assertEquals(error.getClass(), t.getClass()); - assertEquals(error.getMessage(), t.getMessage()); - } - - assertTrue(backgroundEventQueue.isEmpty()); - } -} diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java index df34088427a9a..19c4b98ede207 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java @@ -37,6 +37,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.errors.ClusterAuthorizationException; @@ -50,6 +51,7 @@ import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.message.TxnOffsetCommitRequestData; +import org.apache.kafka.common.metrics.JmxReporter; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; import org.apache.kafka.common.metrics.stats.Avg; @@ -72,6 +74,8 @@ import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.telemetry.internals.ClientTelemetryReporter; +import org.apache.kafka.common.telemetry.internals.ClientTelemetrySender; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; @@ -86,9 +90,9 @@ import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; +import org.mockito.internal.stubbing.answers.CallsRealMethods; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.lang.management.ManagementFactory; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -114,6 +118,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import javax.management.MBeanServer; +import javax.management.ObjectName; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -130,9 +136,11 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.notNull; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -454,31 +462,48 @@ public void testMetricsReporterAutoGeneratedClientId() { KafkaProducer producer = new KafkaProducer<>( props, new StringSerializer(), new StringSerializer()); - MockMetricsReporter mockMetricsReporter = (MockMetricsReporter) producer.metrics.reporters().get(0); + assertEquals(3, producer.metrics.reporters().size()); + MockMetricsReporter mockMetricsReporter = (MockMetricsReporter) producer.metrics.reporters().stream() + .filter(reporter -> reporter instanceof MockMetricsReporter).findFirst().get(); assertEquals(producer.getClientId(), mockMetricsReporter.clientId); - assertEquals(2, producer.metrics.reporters().size()); + producer.close(); } @Test @SuppressWarnings("deprecation") - public void testDisableJmxReporter() { + public void testDisableJmxAndClientTelemetryReporter() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ProducerConfig.AUTO_INCLUDE_JMX_REPORTER_CONFIG, "false"); + props.setProperty(ProducerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); assertTrue(producer.metrics.reporters().isEmpty()); producer.close(); } @Test - public void testExplicitlyEnableJmxReporter() { + public void testExplicitlyOnlyEnableJmxReporter() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, "org.apache.kafka.common.metrics.JmxReporter"); + props.setProperty(ProducerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); assertEquals(1, producer.metrics.reporters().size()); + assertTrue(producer.metrics.reporters().get(0) instanceof JmxReporter); + producer.close(); + } + + @Test + @SuppressWarnings("deprecation") + public void testExplicitlyOnlyEnableClientTelemetryReporter() { + Properties props = new Properties(); + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.AUTO_INCLUDE_JMX_REPORTER_CONFIG, "false"); + KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); + assertEquals(1, producer.metrics.reporters().size()); + assertTrue(producer.metrics.reporters().get(0) instanceof ClientTelemetryReporter); producer.close(); } @@ -1656,6 +1681,55 @@ public void testInvalidGenerationIdAndMemberIdCombinedInSendOffsets() { verifyInvalidGroupMetadata(new ConsumerGroupMetadata("group", 2, JoinGroupRequest.UNKNOWN_MEMBER_ID, Optional.empty())); } + @Test + public void testClientInstanceId() { + Properties props = new Properties(); + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + + ClientTelemetryReporter clientTelemetryReporter = mock(ClientTelemetryReporter.class); + clientTelemetryReporter.configure(any()); + + MockedStatic mockedCommonClientConfigs = mockStatic(CommonClientConfigs.class, new CallsRealMethods()); + mockedCommonClientConfigs.when(() -> CommonClientConfigs.telemetryReporter(anyString(), any())).thenReturn(Optional.of(clientTelemetryReporter)); + + ClientTelemetrySender clientTelemetrySender = mock(ClientTelemetrySender.class); + Uuid expectedUuid = Uuid.randomUuid(); + when(clientTelemetryReporter.telemetrySender()).thenReturn(clientTelemetrySender); + when(clientTelemetrySender.clientInstanceId(any())).thenReturn(Optional.of(expectedUuid)); + + KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); + Uuid uuid = producer.clientInstanceId(Duration.ofMillis(0)); + assertEquals(expectedUuid, uuid); + + mockedCommonClientConfigs.close(); + producer.close(); + } + + @Test + public void testClientInstanceIdInvalidTimeout() { + Properties props = new Properties(); + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + + KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); + Exception exception = assertThrows(IllegalArgumentException.class, () -> producer.clientInstanceId(Duration.ofMillis(-1))); + assertEquals("The timeout cannot be negative.", exception.getMessage()); + + producer.close(); + } + + @Test + public void testClientInstanceIdNoTelemetryReporterRegistered() { + Properties props = new Properties(); + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); + props.setProperty(ProducerConfig.ENABLE_METRICS_PUSH_CONFIG, "false"); + + KafkaProducer producer = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer()); + Exception exception = assertThrows(IllegalStateException.class, () -> producer.clientInstanceId(Duration.ofMillis(0))); + assertEquals("Telemetry is not enabled. Set config `enable.metrics.push` to `true`.", exception.getMessage()); + + producer.close(); + } + private void verifyInvalidGroupMetadata(ConsumerGroupMetadata groupMetadata) { Map configs = new HashMap<>(); configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "some.id"); @@ -2403,7 +2477,8 @@ public KafkaProducer newKafkaProducer() { interceptors, partitioner, time, - ioThread + ioThread, + Optional.empty() ); } } @@ -2424,7 +2499,7 @@ void testDeliveryTimeoutAndLingerMsConfig() { configs.put(ProducerConfig.LINGER_MS_CONFIG, 999); configs.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 1); - assertDoesNotThrow(() -> new KafkaProducer<>(configs, new StringSerializer(), new StringSerializer())); + assertDoesNotThrow(() -> new KafkaProducer<>(configs, new StringSerializer(), new StringSerializer()).close()); } } diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index 4d15d21759093..366681126099c 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -107,30 +107,39 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco int failedAuthenticationDelayMs, Time time, DelegationTokenCache tokenCache) throws Exception { super("echoserver"); setDaemon(true); - serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.configureBlocking(false); - serverSocketChannel.socket().bind(new InetSocketAddress(serverHost, 0)); - this.port = serverSocketChannel.socket().getLocalPort(); - this.socketChannels = Collections.synchronizedList(new ArrayList<>()); - this.newChannels = Collections.synchronizedList(new ArrayList<>()); - this.credentialCache = credentialCache; - this.tokenCache = tokenCache; - if (securityProtocol == SecurityProtocol.SASL_PLAINTEXT || securityProtocol == SecurityProtocol.SASL_SSL) { - for (String mechanism : ScramMechanism.mechanismNames()) { - if (credentialCache.cache(mechanism, ScramCredential.class) == null) - credentialCache.createCache(mechanism, ScramCredential.class); + ServerSocketChannel serverSocketChannel = null; + try { + serverSocketChannel = ServerSocketChannel.open(); + this.serverSocketChannel = serverSocketChannel; + serverSocketChannel.configureBlocking(false); + serverSocketChannel.socket().bind(new InetSocketAddress(serverHost, 0)); + this.port = serverSocketChannel.socket().getLocalPort(); + this.socketChannels = Collections.synchronizedList(new ArrayList<>()); + this.newChannels = Collections.synchronizedList(new ArrayList<>()); + this.credentialCache = credentialCache; + this.tokenCache = tokenCache; + if (securityProtocol == SecurityProtocol.SASL_PLAINTEXT || securityProtocol == SecurityProtocol.SASL_SSL) { + for (String mechanism : ScramMechanism.mechanismNames()) { + if (credentialCache.cache(mechanism, ScramCredential.class) == null) + credentialCache.createCache(mechanism, ScramCredential.class); + } + } + LogContext logContext = new LogContext(); + if (channelBuilder == null) + channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, + securityProtocol, config, credentialCache, tokenCache, time, logContext, + () -> TestUtils.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); + this.metrics = new Metrics(); + this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, + "MetricGroup", channelBuilder, logContext); + acceptorThread = new AcceptorThread(); + this.time = time; + } catch (Exception e) { + if (serverSocketChannel != null) { + serverSocketChannel.close(); } + throw e; } - LogContext logContext = new LogContext(); - if (channelBuilder == null) - channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, - securityProtocol, config, credentialCache, tokenCache, time, logContext, - () -> TestUtils.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); - this.metrics = new Metrics(); - this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, - "MetricGroup", channelBuilder, logContext); - acceptorThread = new AcceptorThread(); - this.time = time; } public int port() { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 988aac59803d1..8c8ab97b8634b 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -408,8 +408,8 @@ KafkaChannel buildChannel(String id, TransportLayer transportLayer, Supplier newKeystoreConfigs = newServerCertStores.keyStoreProps(); @@ -1087,6 +1088,8 @@ false, securityProtocol, config, null, null, time, new LogContext(), // Verify that new connections continue to work with the server with previously configured keystore after failed reconfiguration newClientSelector.connect("3", addr, BUFFER_SIZE, BUFFER_SIZE); NetworkTestUtils.checkClientConnection(newClientSelector, "3", 100, 10); + // manually stop the oldClientSelector because the test harness doesn't manage it. + oldClientSelector.close(); } @ParameterizedTest @@ -1172,8 +1175,10 @@ false, securityProtocol, config, null, null, time, new LogContext(), // Verify that client with matching keystore can authenticate, send and receive String oldNode = "0"; Selector oldClientSelector = createSelector(args.sslClientConfigs); + // take responsibility for closing oldClientSelector, so that we can keep it alive concurrent with the new one. + this.selector = null; oldClientSelector.connect(oldNode, addr, BUFFER_SIZE, BUFFER_SIZE); - NetworkTestUtils.checkClientConnection(selector, oldNode, 100, 10); + NetworkTestUtils.checkClientConnection(oldClientSelector, oldNode, 100, 10); CertStores newClientCertStores = certBuilder(true, "client", args.useInlinePem).addHostName("localhost").build(); args.sslClientConfigs = args.getTrustingConfig(newClientCertStores, args.serverCertStores); @@ -1209,6 +1214,8 @@ false, securityProtocol, config, null, null, time, new LogContext(), // Verify that new connections continue to work with the server with previously configured keystore after failed reconfiguration newClientSelector.connect("3", addr, BUFFER_SIZE, BUFFER_SIZE); NetworkTestUtils.checkClientConnection(newClientSelector, "3", 100, 10); + // manually stop the oldClientSelector because the test harness doesn't manage it. + oldClientSelector.close(); } /** @@ -1267,6 +1274,9 @@ private Selector createSelector(Map sslClientConfigs, final Inte TestSslChannelBuilder channelBuilder = new TestSslChannelBuilder(Mode.CLIENT); channelBuilder.configureBufferSizes(netReadBufSize, netWriteBufSize, appBufSize); channelBuilder.configure(sslClientConfigs); + if (this.selector != null) { + this.selector.close(); + } this.selector = new Selector(100 * 5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext()); return selector; } diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportTls12Tls13Test.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportTls12Tls13Test.java index d11ca48d9b42c..4f6e4b3aced70 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportTls12Tls13Test.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportTls12Tls13Test.java @@ -160,6 +160,9 @@ private void createSelector(Map sslClientConfigs) { SslTransportLayerTest.TestSslChannelBuilder channelBuilder = new SslTransportLayerTest.TestSslChannelBuilder(Mode.CLIENT); channelBuilder.configureBufferSizes(null, null, null); channelBuilder.configure(sslClientConfigs); + if (this.selector != null) { + this.selector.close(); + } this.selector = new Selector(100 * 5000, new Metrics(), TIME, "MetricGroup", channelBuilder, new LogContext()); } } diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslVersionsTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslVersionsTransportLayerTest.java index 584d48fa3ece6..5e7073ae61837 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslVersionsTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslVersionsTransportLayerTest.java @@ -117,6 +117,8 @@ public void testTlsDefaults(List serverProtocols, List clientPro NetworkTestUtils.waitForChannelClose(selector, node, ChannelState.State.AUTHENTICATION_FAILED); server.verifyAuthenticationMetrics(0, 1); } + server.close(); + selector.close(); } /** diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java index 4f3f03c3f2d21..5616fb23f7dbc 100644 --- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java @@ -801,7 +801,7 @@ public Stream provideArguments(ExtensionContext context) { } } - private void verifyRecordsProcessingStats(CompressionType compressionType, RecordConversionStats processingStats, + private void verifyRecordsProcessingStats(CompressionType compressionType, RecordValidationStats processingStats, int numRecords, int numRecordsConverted, long finalBytes, long preConvertedBytes) { assertNotNull(processingStats, "Records processing info is null"); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java index 19d3c46818659..cf832050f2a5d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java @@ -103,7 +103,8 @@ public void shouldHaveCommonlyAgreedApiVersionResponseWithControllerOnForwardabl ApiMessageType.ListenerType.ZK_BROKER, RecordVersion.current(), activeControllerApiVersions, - true + true, + false ); verifyVersions(forwardableAPIKey.id, minVersion, maxVersion, commonResponse); @@ -123,7 +124,8 @@ public void shouldCreateApiResponseOnlyWithKeysSupportedByMagicValue() { null, ListenerType.ZK_BROKER, true, - false + false, + true ); verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1); assertEquals(10, response.throttleTimeMs()); @@ -144,7 +146,8 @@ public void shouldReturnFeatureKeysWhenMagicIsCurrentValueAndThrottleMsIsDefault null, ListenerType.ZK_BROKER, true, - false + false, + true ); verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1); @@ -174,7 +177,8 @@ public void shouldReturnAllKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThro null, listenerType, true, - false + false, + true ); assertEquals(new HashSet<>(ApiKeys.apisForListener(listenerType)), apiKeysInResponse(response)); assertEquals(AbstractResponse.DEFAULT_THROTTLE_TIME, response.throttleTimeMs()); @@ -183,6 +187,40 @@ public void shouldReturnAllKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThro assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, response.data().finalizedFeaturesEpoch()); } + @Test + public void shouldCreateApiResponseWithTelemetryWhenEnabled() { + ApiVersionsResponse response = ApiVersionsResponse.createApiVersionsResponse( + 10, + RecordVersion.V1, + Features.emptySupportedFeatures(), + Collections.emptyMap(), + ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, + null, + ListenerType.BROKER, + true, + false, + true + ); + verifyApiKeysForTelemetry(response, 2); + } + + @Test + public void shouldNotCreateApiResponseWithTelemetryWhenDisabled() { + ApiVersionsResponse response = ApiVersionsResponse.createApiVersionsResponse( + 10, + RecordVersion.V1, + Features.emptySupportedFeatures(), + Collections.emptyMap(), + ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, + null, + ListenerType.BROKER, + true, + false, + false + ); + verifyApiKeysForTelemetry(response, 0); + } + @Test public void testMetadataQuorumApisAreDisabled() { ApiVersionsResponse response = ApiVersionsResponse.createApiVersionsResponse( @@ -194,7 +232,8 @@ public void testMetadataQuorumApisAreDisabled() { null, ListenerType.ZK_BROKER, true, - false + false, + true ); // Ensure that APIs needed for the KRaft mode are not exposed through ApiVersions until we are ready for them @@ -254,6 +293,16 @@ private void verifyApiKeysForMagic(ApiVersionsResponse response, Byte maxMagic) } } + private void verifyApiKeysForTelemetry(ApiVersionsResponse response, int expectedCount) { + int count = 0; + for (ApiVersion version : response.data().apiKeys()) { + if (version.apiKey() == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS.id || version.apiKey() == ApiKeys.PUSH_TELEMETRY.id) { + count++; + } + } + assertEquals(expectedCount, count); + } + private HashSet apiKeysInResponse(ApiVersionsResponse apiVersions) { HashSet apiKeys = new HashSet<>(); for (ApiVersion version : apiVersions.data().apiKeys()) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/BrokerRegistrationRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/BrokerRegistrationRequestTest.java new file mode 100644 index 0000000000000..0ea4eb4aac27c --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/requests/BrokerRegistrationRequestTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.BrokerRegistrationRequestData; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.ByteBuffer; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class BrokerRegistrationRequestTest { + private static Stream BrokerRegistrationRequestVersions() { + return IntStream.range(BrokerRegistrationRequestData.LOWEST_SUPPORTED_VERSION, + BrokerRegistrationRequestData.HIGHEST_SUPPORTED_VERSION + 1).mapToObj(version -> Arguments.of((short) version)); + } + + @ParameterizedTest + @MethodSource("BrokerRegistrationRequestVersions") + public void testBasicBuild(short version) { + Uuid incarnationId = Uuid.randomUuid(); + BrokerRegistrationRequestData data = new BrokerRegistrationRequestData(); + data.setBrokerId(0) + .setIsMigratingZkBroker(false) + .setClusterId("test") + .setFeatures(new BrokerRegistrationRequestData.FeatureCollection()) + .setIncarnationId(incarnationId) + .setListeners(new BrokerRegistrationRequestData.ListenerCollection()) + .setRack("a") + .setPreviousBrokerEpoch(1L); + BrokerRegistrationRequest.Builder builder = new BrokerRegistrationRequest.Builder(data); + BrokerRegistrationRequest request = builder.build(version); + + ObjectSerializationCache cache = new ObjectSerializationCache(); + int size = request.data().size(cache, version); + ByteBuffer buf = ByteBuffer.allocate(size); + ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf); + request.data().write(byteBufferAccessor, cache, version); + + BrokerRegistrationRequestData data2 = new BrokerRegistrationRequestData(); + buf.flip(); + data2.read(byteBufferAccessor, version); + + assertEquals(0, data2.brokerId(), "Unepxected broker ID in " + data2); + assertEquals("test", data2.clusterId(), "Unexpected cluster ID in " + data2); + assertEquals(incarnationId, data2.incarnationId(), "Unepxected incarnation ID in " + data2); + assertEquals("a", data2.rack(), "Unepxected rack in " + data2); + if (version >= 3) { + assertEquals(1, data2.previousBrokerEpoch(), "Unexpected previousBrokerEpoch in " + data2); + } else { + assertEquals(-1, data2.previousBrokerEpoch(), "Unexpected previousBrokerEpoch in " + data2); + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequestTest.java index 81255da2b113c..23ea03c173062 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ConsumerGroupDescribeRequestTest.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.List; +import static org.apache.kafka.common.requests.ConsumerGroupDescribeRequest.getErrorDescribedGroupList; import static org.junit.jupiter.api.Assertions.assertEquals; public class ConsumerGroupDescribeRequestTest { @@ -47,4 +48,26 @@ void testGetErrorResponse() { assertEquals(Errors.forException(e).code(), group.errorCode()); } } + + @Test + public void testGetErrorDescribedGroupList() { + List expectedDescribedGroupList = Arrays.asList( + new ConsumerGroupDescribeResponseData.DescribedGroup() + .setGroupId("group-id-1") + .setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code()), + new ConsumerGroupDescribeResponseData.DescribedGroup() + .setGroupId("group-id-2") + .setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code()), + new ConsumerGroupDescribeResponseData.DescribedGroup() + .setGroupId("group-id-3") + .setErrorCode(Errors.COORDINATOR_LOAD_IN_PROGRESS.code()) + ); + + List describedGroupList = getErrorDescribedGroupList( + Arrays.asList("group-id-1", "group-id-2", "group-id-3"), + Errors.COORDINATOR_LOAD_IN_PROGRESS + ); + + assertEquals(expectedDescribedGroupList, describedGroupList); + } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/JoinGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/JoinGroupResponseTest.java index 038b51c1d6916..1b7e5d2a19281 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/JoinGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/JoinGroupResponseTest.java @@ -39,4 +39,19 @@ public void testProtocolNameBackwardCompatibility(short version) { assertNull(joinGroupResponse.data().protocolName()); } } + + @ParameterizedTest + @ApiKeyVersionsSource(apiKey = ApiKeys.JOIN_GROUP) + public void testProtocolNameComplianceWithVersion7AndAbove(short version) { + JoinGroupResponseData data = new JoinGroupResponseData() + .setProtocolName(""); + + JoinGroupResponse joinGroupResponse = new JoinGroupResponse(data, version); + + if (version < 7) { + assertEquals("", joinGroupResponse.data().protocolName()); + } else { + assertNull(joinGroupResponse.data().protocolName()); + } + } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index d5132182ceee2..4bb5c72281952 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -16,13 +16,16 @@ */ package org.apache.kafka.common.requests; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.MessageUtil; +import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; import java.nio.ByteBuffer; import java.util.Arrays; @@ -34,6 +37,7 @@ import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class LeaveGroupResponseTest { @@ -165,4 +169,57 @@ public void testEqualityWithMemberResponses() { assertEquals(primaryResponse.hashCode(), reversedResponse.hashCode()); } } + + @ParameterizedTest + @ApiKeyVersionsSource(apiKey = ApiKeys.LEAVE_GROUP) + public void testNoErrorNoMembersResponses(short version) { + LeaveGroupResponseData data = new LeaveGroupResponseData() + .setErrorCode(Errors.NONE.code()) + .setMembers(Collections.emptyList()); + + if (version < 3) { + assertThrows(UnsupportedVersionException.class, + () -> new LeaveGroupResponse(data, version)); + } else { + LeaveGroupResponse response = new LeaveGroupResponse(data, version); + assertEquals(Errors.NONE, response.topLevelError()); + assertEquals(Collections.emptyList(), response.memberResponses()); + } + } + + @ParameterizedTest + @ApiKeyVersionsSource(apiKey = ApiKeys.LEAVE_GROUP) + public void testNoErrorMultipleMembersResponses(short version) { + LeaveGroupResponseData data = new LeaveGroupResponseData() + .setErrorCode(Errors.NONE.code()) + .setMembers(memberResponses); + + if (version < 3) { + assertThrows(UnsupportedVersionException.class, + () -> new LeaveGroupResponse(data, version)); + } else { + LeaveGroupResponse response = new LeaveGroupResponse(data, version); + assertEquals(Errors.NONE, response.topLevelError()); + assertEquals(memberResponses, response.memberResponses()); + } + } + + @ParameterizedTest + @ApiKeyVersionsSource(apiKey = ApiKeys.LEAVE_GROUP) + public void testErrorResponses(short version) { + LeaveGroupResponseData dataNoMembers = new LeaveGroupResponseData() + .setErrorCode(Errors.GROUP_ID_NOT_FOUND.code()) + .setMembers(Collections.emptyList()); + + LeaveGroupResponse responseNoMembers = new LeaveGroupResponse(dataNoMembers, version); + assertEquals(Errors.GROUP_ID_NOT_FOUND, responseNoMembers.topLevelError()); + + LeaveGroupResponseData dataMembers = new LeaveGroupResponseData() + .setErrorCode(Errors.GROUP_ID_NOT_FOUND.code()) + .setMembers(memberResponses); + + LeaveGroupResponse responseMembers = new LeaveGroupResponse(dataMembers, version); + assertEquals(Errors.GROUP_ID_NOT_FOUND, responseMembers.topLevelError()); + } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index a802ee3e58035..b1ecb6aa4b41d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -167,6 +167,8 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; +import org.apache.kafka.common.message.ListClientMetricsResourcesRequestData; +import org.apache.kafka.common.message.ListClientMetricsResourcesResponseData; import org.apache.kafka.common.message.ListGroupsRequestData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; @@ -1074,6 +1076,7 @@ private AbstractRequest getRequest(ApiKeys apikey, short version) { case GET_TELEMETRY_SUBSCRIPTIONS: return createGetTelemetrySubscriptionsRequest(version); case PUSH_TELEMETRY: return createPushTelemetryRequest(version); case ASSIGN_REPLICAS_TO_DIRS: return createAssignReplicasToDirsRequest(version); + case LIST_CLIENT_METRICS_RESOURCES: return createListClientMetricsResourcesRequest(version); default: throw new IllegalArgumentException("Unknown API key " + apikey); } } @@ -1154,6 +1157,7 @@ private AbstractResponse getResponse(ApiKeys apikey, short version) { case GET_TELEMETRY_SUBSCRIPTIONS: return createGetTelemetrySubscriptionsResponse(); case PUSH_TELEMETRY: return createPushTelemetryResponse(); case ASSIGN_REPLICAS_TO_DIRS: return createAssignReplicasToDirsResponse(); + case LIST_CLIENT_METRICS_RESOURCES: return createListClientMetricsResourcesResponse(); default: throw new IllegalArgumentException("Unknown API key " + apikey); } } @@ -3616,6 +3620,17 @@ private PushTelemetryResponse createPushTelemetryResponse() { return new PushTelemetryResponse(response); } + private ListClientMetricsResourcesRequest createListClientMetricsResourcesRequest(short version) { + return new ListClientMetricsResourcesRequest.Builder(new ListClientMetricsResourcesRequestData()).build(version); + } + + private ListClientMetricsResourcesResponse createListClientMetricsResourcesResponse() { + ListClientMetricsResourcesResponseData response = new ListClientMetricsResourcesResponseData(); + response.setErrorCode(Errors.NONE.code()); + response.setThrottleTimeMs(10); + return new ListClientMetricsResourcesResponse(response); + } + @Test public void testInvalidSaslHandShakeRequest() { AbstractRequest request = new SaslHandshakeRequest.Builder( diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index b85dcd8d51ed7..98b66827e5605 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -1133,6 +1133,7 @@ public void testAuthenticateCallbackHandlerMechanisms() throws Exception { TestServerCallbackHandler.class); saslServerConfigs.put(listener.saslMechanismConfigPrefix("digest-md5") + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, DigestServerCallbackHandler.class); + server.close(); server = createEchoServer(securityProtocol); // Verify that DIGEST-MD5 (currently configured for client) works with `DigestServerCallbackHandler` diff --git a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/secured/RefreshingHttpsJwksTest.java b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/secured/RefreshingHttpsJwksTest.java index 705a539d9067b..cc19c74e66f70 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/secured/RefreshingHttpsJwksTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/oauthbearer/internals/secured/RefreshingHttpsJwksTest.java @@ -41,7 +41,6 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.Time; import org.jose4j.http.SimpleResponse; import org.jose4j.jwk.HttpsJwks; import org.junit.jupiter.api.Test; @@ -62,14 +61,16 @@ public class RefreshingHttpsJwksTest extends OAuthBearerTest { @Test public void testBasicScheduleRefresh() throws Exception { String keyId = "abc123"; - Time time = new MockTime(); + MockTime time = new MockTime(); HttpsJwks httpsJwks = spyHttpsJwks(); + // we use mocktime here to ensure that scheduled refresh _doesn't_ run and update the invocation count + // we expect httpsJwks.refresh() to be invoked twice, once from init() and maybeExpediteRefresh() each try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks)) { refreshingHttpsJwks.init(); verify(httpsJwks, times(1)).refresh(); assertTrue(refreshingHttpsJwks.maybeExpediteRefresh(keyId)); - verify(httpsJwks, times(1)).refresh(); + verify(httpsJwks, times(2)).refresh(); } } @@ -81,7 +82,7 @@ public void testBasicScheduleRefresh() throws Exception { @Test public void testMaybeExpediteRefreshNoDelay() throws Exception { String keyId = "abc123"; - Time time = new MockTime(); + MockTime time = new MockTime(); HttpsJwks httpsJwks = spyHttpsJwks(); try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks)) { @@ -113,7 +114,7 @@ public void testLongKey() throws Exception { Arrays.fill(keyIdChars, '0'); String keyId = new String(keyIdChars); - Time time = new MockTime(); + MockTime time = new MockTime(); HttpsJwks httpsJwks = spyHttpsJwks(); try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks)) { @@ -134,6 +135,20 @@ public void testSecondaryRefreshAfterElapsedDelay() throws Exception { String keyId = "abc123"; MockTime time = new MockTime(); HttpsJwks httpsJwks = spyHttpsJwks(); + + try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks)) { + refreshingHttpsJwks.init(); + // We refresh once at the initialization time from getJsonWebKeys. + verify(httpsJwks, times(1)).refresh(); + assertTrue(refreshingHttpsJwks.maybeExpediteRefresh(keyId)); + verify(httpsJwks, times(2)).refresh(); + time.sleep(REFRESH_MS + 1); + verify(httpsJwks, times(3)).refresh(); + assertFalse(refreshingHttpsJwks.maybeExpediteRefresh(keyId)); + } + } + + private ScheduledExecutorService mockExecutorService(MockTime time) { MockExecutorService mockExecutorService = new MockExecutorService(time); ScheduledExecutorService executorService = Mockito.mock(ScheduledExecutorService.class); Mockito.doAnswer(invocation -> { @@ -155,22 +170,12 @@ public void testSecondaryRefreshAfterElapsedDelay() throws Exception { return null; }, unit.toMillis(initialDelay), period); }).when(executorService).scheduleAtFixedRate(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(TimeUnit.class)); - - try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks, executorService)) { - refreshingHttpsJwks.init(); - // We refresh once at the initialization time from getJsonWebKeys. - verify(httpsJwks, times(1)).refresh(); - assertTrue(refreshingHttpsJwks.maybeExpediteRefresh(keyId)); - verify(httpsJwks, times(2)).refresh(); - time.sleep(REFRESH_MS + 1); - verify(httpsJwks, times(3)).refresh(); - assertFalse(refreshingHttpsJwks.maybeExpediteRefresh(keyId)); - } + return executorService; } private void assertMaybeExpediteRefreshWithDelay(long sleepDelay, boolean shouldBeScheduled) throws Exception { String keyId = "abc123"; - Time time = new MockTime(); + MockTime time = new MockTime(); HttpsJwks httpsJwks = spyHttpsJwks(); try (RefreshingHttpsJwks refreshingHttpsJwks = getRefreshingHttpsJwks(time, httpsJwks)) { @@ -181,12 +186,8 @@ private void assertMaybeExpediteRefreshWithDelay(long sleepDelay, boolean should } } - private RefreshingHttpsJwks getRefreshingHttpsJwks(final Time time, final HttpsJwks httpsJwks) { - return new RefreshingHttpsJwks(time, httpsJwks, REFRESH_MS, RETRY_BACKOFF_MS, RETRY_BACKOFF_MAX_MS); - } - - private RefreshingHttpsJwks getRefreshingHttpsJwks(final Time time, final HttpsJwks httpsJwks, final ScheduledExecutorService executorService) { - return new RefreshingHttpsJwks(time, httpsJwks, REFRESH_MS, RETRY_BACKOFF_MS, RETRY_BACKOFF_MAX_MS, executorService); + private RefreshingHttpsJwks getRefreshingHttpsJwks(final MockTime time, final HttpsJwks httpsJwks) { + return new RefreshingHttpsJwks(time, httpsJwks, REFRESH_MS, RETRY_BACKOFF_MS, RETRY_BACKOFF_MAX_MS, mockExecutorService(time)); } /** diff --git a/clients/src/test/java/org/apache/kafka/common/security/ssl/SslFactoryTest.java b/clients/src/test/java/org/apache/kafka/common/security/ssl/SslFactoryTest.java index 922166020ff82..9f02e847335ff 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/ssl/SslFactoryTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/ssl/SslFactoryTest.java @@ -543,26 +543,33 @@ public void testUsedConfigs() throws IOException, GeneralSecurityException { public void testDynamicUpdateCompatibility() throws Exception { KeyPair keyPair = TestSslUtils.generateKeyPair("RSA"); KeyStore ks = createKeyStore(keyPair, "*.example.com", "Kafka", true, "localhost", "*.example.com"); - ensureCompatible(ks, ks); - ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", true, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, " *.example.com", " Kafka ", true, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.example.COM", "Kafka", true, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "KAFKA", true, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "localhost")); - - ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", false, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.example.COM", "Kafka", false, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "KAFKA", false, "localhost", "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", false, "*.example.com")); - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", false, "localhost")); + ensureCompatible(ks, ks, false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", true, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, " *.example.com", " Kafka ", true, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.example.COM", "Kafka", true, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "KAFKA", true, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "localhost"), false, false); + + ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", false, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.example.COM", "Kafka", false, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "KAFKA", false, "localhost", "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", false, "*.example.com"), false, false); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", false, "localhost"), false, false); assertThrows(ConfigException.class, () -> - ensureCompatible(ks, createKeyStore(keyPair, " *.example.com", " Kafka ", false, "localhost", "*.example.com"))); + ensureCompatible(ks, createKeyStore(keyPair, " *.example.com", " Kafka ", false, "localhost", "*.example.com"), false, false)); assertThrows(ConfigException.class, () -> - ensureCompatible(ks, createKeyStore(keyPair, "*.another.example.com", "Kafka", true, "*.example.com"))); + ensureCompatible(ks, createKeyStore(keyPair, "*.another.example.com", "Kafka", true, "*.example.com"), false, false)); assertThrows(ConfigException.class, () -> - ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "*.another.example.com"))); + ensureCompatible(ks, createKeyStore(keyPair, "*.EXAMPLE.COM", "Kafka", true, "*.another.example.com"), false, false)); + + // Test disabling of validation + ensureCompatible(ks, createKeyStore(keyPair, " *.another.example.com", "Kafka ", true, "localhost", "*.another.example.com"), true, true); + ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", true, "localhost", "*.another.example.com"), false, true); + assertThrows(ConfigException.class, () -> ensureCompatible(ks, createKeyStore(keyPair, "*.example.com", "Kafka", true, "localhost", "*.another.example.com"), true, false)); + ensureCompatible(ks, createKeyStore(keyPair, "*.another.example.com", "Kafka", true, "localhost", "*.example.com"), true, false); + assertThrows(ConfigException.class, () -> ensureCompatible(ks, createKeyStore(keyPair, "*.another.example.com", "Kafka", true, "localhost", "*.example.com"), false, true)); } private KeyStore createKeyStore(KeyPair keyPair, String commonName, String org, boolean utf8, String... dnsNames) throws Exception { diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitterTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitterTest.java new file mode 100644 index 0000000000000..32fd2a5ad930f --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryEmitterTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ClientTelemetryEmitterTest { + + private MetricKey metricKey; + private Instant now; + + @BeforeEach + public void setUp() { + metricKey = new MetricKey("name", Collections.emptyMap()); + now = Instant.now(); + } + + @Test + public void testShouldEmitMetric() { + Predicate selector = ClientTelemetryUtils.getSelectorFromRequestedMetrics( + Collections.singletonList("io.test.metric")); + ClientTelemetryEmitter emitter = new ClientTelemetryEmitter(selector, true); + + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric1"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric.producer.bytes"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("io.test"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("org.io.test.metric"))); + assertTrue(emitter.shouldEmitDeltaMetrics()); + } + + @Test + public void testShouldEmitMetricSelectorAll() { + ClientTelemetryEmitter emitter = new ClientTelemetryEmitter(ClientTelemetryUtils.SELECTOR_ALL_METRICS, true); + + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric1"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test.metric.producer.bytes"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("io.test"))); + assertTrue(emitter.shouldEmitMetric(new MetricKey("org.io.test.metric"))); + assertTrue(emitter.shouldEmitDeltaMetrics()); + } + + @Test + public void testShouldEmitMetricSelectorNone() { + ClientTelemetryEmitter emitter = new ClientTelemetryEmitter(ClientTelemetryUtils.SELECTOR_NO_METRICS, true); + + assertFalse(emitter.shouldEmitMetric(new MetricKey("io.test.metric"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("io.test.metric1"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("io.test.metric.producer.bytes"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("io.test"))); + assertFalse(emitter.shouldEmitMetric(new MetricKey("org.io.test.metric"))); + assertTrue(emitter.shouldEmitDeltaMetrics()); + } + + @Test + public void testShouldEmitDeltaMetricsFalse() { + ClientTelemetryEmitter emitter = new ClientTelemetryEmitter(ClientTelemetryUtils.SELECTOR_ALL_METRICS, false); + assertFalse(emitter.shouldEmitDeltaMetrics()); + } + + @Test + public void testEmitMetric() { + Predicate selector = ClientTelemetryUtils.getSelectorFromRequestedMetrics( + Collections.singletonList("name")); + ClientTelemetryEmitter emitter = new ClientTelemetryEmitter(selector, true); + + SinglePointMetric gauge = SinglePointMetric.gauge(metricKey, Long.valueOf(1), now); + SinglePointMetric sum = SinglePointMetric.sum(metricKey, 1.0, true, now); + assertTrue(emitter.emitMetric(gauge)); + assertTrue(emitter.emitMetric(sum)); + + MetricKey anotherKey = new MetricKey("io.name", Collections.emptyMap()); + assertFalse(emitter.emitMetric(SinglePointMetric.gauge(anotherKey, Long.valueOf(1), now))); + + assertEquals(2, emitter.emittedMetrics().size()); + assertEquals(Arrays.asList(gauge, sum), emitter.emittedMetrics()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporterTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporterTest.java new file mode 100644 index 0000000000000..64d8a3af33fb0 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryReporterTest.java @@ -0,0 +1,619 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.common.v1.KeyValue; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsRequestData; +import org.apache.kafka.common.message.GetTelemetrySubscriptionsResponseData; +import org.apache.kafka.common.message.PushTelemetryRequestData; +import org.apache.kafka.common.message.PushTelemetryResponseData; +import org.apache.kafka.common.metrics.KafkaMetricsContext; +import org.apache.kafka.common.metrics.MetricsContext; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsRequest; +import org.apache.kafka.common.requests.GetTelemetrySubscriptionsResponse; +import org.apache.kafka.common.requests.PushTelemetryRequest; +import org.apache.kafka.common.requests.PushTelemetryResponse; +import org.apache.kafka.common.telemetry.ClientTelemetryState; +import org.apache.kafka.common.utils.MockTime; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ClientTelemetryReporterTest { + + private MockTime time; + private ClientTelemetryReporter clientTelemetryReporter; + private Map configs; + private MetricsContext metricsContext; + private Uuid uuid; + private ClientTelemetryReporter.ClientTelemetrySubscription subscription; + + @BeforeEach + public void setUp() { + time = new MockTime(); + clientTelemetryReporter = new ClientTelemetryReporter(time); + configs = new HashMap<>(); + metricsContext = new KafkaMetricsContext("test"); + uuid = Uuid.randomUuid(); + subscription = new ClientTelemetryReporter.ClientTelemetrySubscription(uuid, 1234, 20000, + Collections.emptyList(), true, null); + } + + @Test + public void testInitTelemetryReporter() { + configs.put(CommonClientConfigs.CLIENT_ID_CONFIG, "test-client"); + configs.put(CommonClientConfigs.CLIENT_RACK_CONFIG, "rack"); + + clientTelemetryReporter.configure(configs); + clientTelemetryReporter.contextChange(metricsContext); + assertNotNull(clientTelemetryReporter.metricsCollector()); + assertNotNull(clientTelemetryReporter.telemetryProvider().resource()); + assertEquals(1, clientTelemetryReporter.telemetryProvider().resource().getAttributesCount()); + assertEquals( + ClientTelemetryProvider.CLIENT_RACK, clientTelemetryReporter.telemetryProvider().resource().getAttributes(0).getKey()); + assertEquals("rack", clientTelemetryReporter.telemetryProvider().resource().getAttributes(0).getValue().getStringValue()); + } + + @Test + public void testInitTelemetryReporterNoCollector() { + // Remove namespace config which skips the collector initialization. + MetricsContext metricsContext = Collections::emptyMap; + + clientTelemetryReporter.configure(configs); + clientTelemetryReporter.contextChange(metricsContext); + assertNull(clientTelemetryReporter.metricsCollector()); + } + + @Test + public void testProducerLabels() { + configs.put(CommonClientConfigs.CLIENT_ID_CONFIG, "test-client"); + configs.put(ConsumerConfig.GROUP_ID_CONFIG, "group-id"); + configs.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "group-instance-id"); + configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "transaction-id"); + configs.put(CommonClientConfigs.CLIENT_RACK_CONFIG, "rack"); + + clientTelemetryReporter.configure(configs); + clientTelemetryReporter.contextChange(new KafkaMetricsContext("kafka.producer")); + assertNotNull(clientTelemetryReporter.metricsCollector()); + assertNotNull(clientTelemetryReporter.telemetryProvider().resource()); + + List attributes = clientTelemetryReporter.telemetryProvider().resource().getAttributesList(); + assertEquals(2, attributes.size()); + attributes.forEach(attribute -> { + if (attribute.getKey().equals(ClientTelemetryProvider.CLIENT_RACK)) { + assertEquals("rack", attribute.getValue().getStringValue()); + } else if (attribute.getKey().equals(ClientTelemetryProvider.TRANSACTIONAL_ID)) { + assertEquals("transaction-id", attribute.getValue().getStringValue()); + } + }); + } + + @Test + public void testConsumerLabels() { + configs.put(CommonClientConfigs.CLIENT_ID_CONFIG, "test-client"); + configs.put(ConsumerConfig.GROUP_ID_CONFIG, "group-id"); + configs.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "group-instance-id"); + configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "transaction-id"); + configs.put(CommonClientConfigs.CLIENT_RACK_CONFIG, "rack"); + + clientTelemetryReporter.configure(configs); + clientTelemetryReporter.contextChange(new KafkaMetricsContext("kafka.consumer")); + assertNotNull(clientTelemetryReporter.metricsCollector()); + assertNotNull(clientTelemetryReporter.telemetryProvider().resource()); + + List attributes = clientTelemetryReporter.telemetryProvider().resource().getAttributesList(); + assertEquals(3, attributes.size()); + attributes.forEach(attribute -> { + if (attribute.getKey().equals(ClientTelemetryProvider.CLIENT_RACK)) { + assertEquals("rack", attribute.getValue().getStringValue()); + } else if (attribute.getKey().equals(ClientTelemetryProvider.GROUP_ID)) { + assertEquals("group-id", attribute.getValue().getStringValue()); + } else if (attribute.getKey().equals(ClientTelemetryProvider.GROUP_INSTANCE_ID)) { + assertEquals("group-instance-id", attribute.getValue().getStringValue()); + } + }); + } + + @Test + public void testTelemetryReporterClose() { + clientTelemetryReporter.close(); + assertEquals(ClientTelemetryState.TERMINATED, ((ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter + .telemetrySender()).state()); + } + + @Test + public void testTelemetryReporterCloseMultipleTimesNoException() { + clientTelemetryReporter.close(); + clientTelemetryReporter.close(); + assertEquals(ClientTelemetryState.TERMINATED, ((ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter + .telemetrySender()).state()); + } + + @Test + public void testTelemetrySenderTimeToNextUpdate() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(0, telemetrySender.timeToNextUpdate(100)); + + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + assertEquals(20000, telemetrySender.timeToNextUpdate(100), 200); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertEquals(100, telemetrySender.timeToNextUpdate(100)); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + long time = telemetrySender.timeToNextUpdate(100); + assertTrue(time > 0 && time >= 0.5 * time && time <= 1.5 * time); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + assertEquals(100, telemetrySender.timeToNextUpdate(100)); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATING_PUSH_NEEDED)); + assertEquals(0, telemetrySender.timeToNextUpdate(100)); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS)); + assertEquals(Long.MAX_VALUE, telemetrySender.timeToNextUpdate(100)); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATED)); + assertThrows(IllegalStateException.class, () -> telemetrySender.timeToNextUpdate(100)); + } + + @Test + public void testCreateRequestSubscriptionNeeded() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + + Optional> requestOptional = telemetrySender.createRequest(); + assertNotNull(requestOptional); + assertTrue(requestOptional.isPresent()); + assertTrue(requestOptional.get().build() instanceof GetTelemetrySubscriptionsRequest); + GetTelemetrySubscriptionsRequest request = (GetTelemetrySubscriptionsRequest) requestOptional.get().build(); + + GetTelemetrySubscriptionsRequest expectedResult = new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData().setClientInstanceId(Uuid.ZERO_UUID), true).build(); + + assertEquals(expectedResult.data(), request.data()); + assertEquals(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS, telemetrySender.state()); + } + + @Test + public void testCreateRequestSubscriptionNeededAfterExistingSubscription() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + + Optional> requestOptional = telemetrySender.createRequest(); + assertNotNull(requestOptional); + assertTrue(requestOptional.isPresent()); + assertTrue(requestOptional.get().build() instanceof GetTelemetrySubscriptionsRequest); + GetTelemetrySubscriptionsRequest request = (GetTelemetrySubscriptionsRequest) requestOptional.get().build(); + + GetTelemetrySubscriptionsRequest expectedResult = new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData().setClientInstanceId(subscription.clientInstanceId()), true).build(); + + assertEquals(expectedResult.data(), request.data()); + assertEquals(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS, telemetrySender.state()); + } + + @Test + public void testCreateRequestPushNeeded() { + clientTelemetryReporter.configure(configs); + clientTelemetryReporter.contextChange(metricsContext); + + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + // create request to move state to SUBSCRIPTION_IN_PROGRESS + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + telemetrySender.createRequest(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + + Optional> requestOptional = telemetrySender.createRequest(); + assertNotNull(requestOptional); + assertTrue(requestOptional.isPresent()); + assertTrue(requestOptional.get().build() instanceof PushTelemetryRequest); + PushTelemetryRequest request = (PushTelemetryRequest) requestOptional.get().build(); + + PushTelemetryRequest expectedResult = new PushTelemetryRequest.Builder( + new PushTelemetryRequestData().setClientInstanceId(subscription.clientInstanceId()) + .setSubscriptionId(subscription.subscriptionId()), true).build(); + + assertEquals(expectedResult.data(), request.data()); + assertEquals(ClientTelemetryState.PUSH_IN_PROGRESS, telemetrySender.state()); + } + + @Test + public void testCreateRequestPushNeededWithoutSubscription() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + // create request to move state to SUBSCRIPTION_IN_PROGRESS + telemetrySender.createRequest(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + + Optional> requestOptional = telemetrySender.createRequest(); + assertNotNull(requestOptional); + assertFalse(requestOptional.isPresent()); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + } + + @Test + public void testCreateRequestInvalidState() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertFalse(telemetrySender.createRequest().isPresent()); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + assertFalse(telemetrySender.createRequest().isPresent()); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATING_PUSH_NEEDED)); + assertFalse(telemetrySender.createRequest().isPresent()); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATING_PUSH_IN_PROGRESS)); + assertFalse(telemetrySender.createRequest().isPresent()); + + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.TERMINATED)); + assertFalse(telemetrySender.createRequest().isPresent()); + } + + @Test + public void testCreateRequestPushNoCollector() { + final long now = time.milliseconds(); + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + // create request to move state to SUBSCRIPTION_IN_PROGRESS + telemetrySender.createRequest(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + + telemetrySender.updateSubscriptionResult(subscription, now); + long interval = telemetrySender.timeToNextUpdate(100); + assertTrue(interval > 0 && interval != 2000 && interval >= 0.5 * interval && interval <= 1.5 * interval); + + time.sleep(1000); + Optional> requestOptional = telemetrySender.createRequest(); + assertFalse(requestOptional.isPresent()); + + assertEquals(20000, telemetrySender.timeToNextUpdate(100)); + assertEquals(now + 1000, telemetrySender.lastRequestMs()); + } + + @Test + public void testHandleResponseGetSubscriptions() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + Uuid clientInstanceId = Uuid.randomUuid(); + GetTelemetrySubscriptionsResponse response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData() + .setClientInstanceId(clientInstanceId) + .setSubscriptionId(5678) + .setAcceptedCompressionTypes(Collections.singletonList(CompressionType.GZIP.id)) + .setPushIntervalMs(20000) + .setRequestedMetrics(Collections.singletonList("*"))); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.PUSH_NEEDED, telemetrySender.state()); + + ClientTelemetryReporter.ClientTelemetrySubscription subscription = telemetrySender.subscription(); + assertNotNull(subscription); + assertEquals(clientInstanceId, subscription.clientInstanceId()); + assertEquals(5678, subscription.subscriptionId()); + assertEquals(Collections.singletonList(CompressionType.GZIP), subscription.acceptedCompressionTypes()); + assertEquals(20000, subscription.pushIntervalMs()); + assertEquals(ClientTelemetryUtils.SELECTOR_ALL_METRICS, subscription.selector()); + } + + @Test + public void testHandleResponseGetSubscriptionsWithoutMetrics() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + Uuid clientInstanceId = Uuid.randomUuid(); + GetTelemetrySubscriptionsResponse response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData() + .setClientInstanceId(clientInstanceId) + .setSubscriptionId(5678) + .setAcceptedCompressionTypes(Collections.singletonList(CompressionType.GZIP.id)) + .setPushIntervalMs(20000)); + + telemetrySender.handleResponse(response); + // Again subscription should be required. + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + + ClientTelemetryReporter.ClientTelemetrySubscription subscription = telemetrySender.subscription(); + assertNotNull(subscription); + assertEquals(clientInstanceId, subscription.clientInstanceId()); + assertEquals(5678, subscription.subscriptionId()); + assertEquals(Collections.singletonList(CompressionType.GZIP), subscription.acceptedCompressionTypes()); + assertEquals(20000, subscription.pushIntervalMs()); + assertEquals(ClientTelemetryUtils.SELECTOR_NO_METRICS, subscription.selector()); + } + + @Test + public void testHandleResponseGetTelemetryErrorResponse() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + // throttling quota exceeded + GetTelemetrySubscriptionsResponse response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData().setErrorCode(Errors.THROTTLING_QUOTA_EXCEEDED.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(300000, telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + // invalid request error + response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData().setErrorCode(Errors.INVALID_REQUEST.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + // unsupported version error + telemetrySender.enabled(true); + response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData().setErrorCode(Errors.UNSUPPORTED_VERSION.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + // unknown error + telemetrySender.enabled(true); + response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData().setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + } + + @Test + public void testHandleResponseSubscriptionChange() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + KafkaMetricsCollector kafkaMetricsCollector = Mockito.mock(KafkaMetricsCollector.class); + clientTelemetryReporter.metricsCollector(kafkaMetricsCollector); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + Uuid clientInstanceId = Uuid.randomUuid(); + GetTelemetrySubscriptionsResponse response = new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData() + .setClientInstanceId(clientInstanceId) + .setSubscriptionId(15678) + .setAcceptedCompressionTypes(Collections.singletonList(CompressionType.ZSTD.id)) + .setPushIntervalMs(10000) + .setDeltaTemporality(false) // Change delta temporality as well + .setRequestedMetrics(Collections.singletonList("org.apache.kafka.producer"))); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.PUSH_NEEDED, telemetrySender.state()); + + ClientTelemetryReporter.ClientTelemetrySubscription responseSubscription = telemetrySender.subscription(); + assertNotNull(responseSubscription); + assertEquals(clientInstanceId, responseSubscription.clientInstanceId()); + assertEquals(15678, responseSubscription.subscriptionId()); + assertEquals(Collections.singletonList(CompressionType.ZSTD), responseSubscription.acceptedCompressionTypes()); + assertEquals(10000, responseSubscription.pushIntervalMs()); + assertFalse(responseSubscription.deltaTemporality()); + assertTrue(responseSubscription.selector().test(new MetricKey("org.apache.kafka.producer"))); + assertTrue(responseSubscription.selector().test(new MetricKey("org.apache.kafka.producerabc"))); + assertTrue(responseSubscription.selector().test(new MetricKey("org.apache.kafka.producer.abc"))); + assertFalse(responseSubscription.selector().test(new MetricKey("org.apache.kafka.produce"))); + + Mockito.verify(kafkaMetricsCollector, Mockito.times(1)).metricsReset(); + } + + @Test + public void testHandleResponsePushTelemetry() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + PushTelemetryResponse response = new PushTelemetryResponse(new PushTelemetryResponseData()); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.PUSH_NEEDED, telemetrySender.state()); + assertEquals(subscription.pushIntervalMs(), telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + } + + @Test + public void testHandleResponsePushTelemetryErrorResponse() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // unknown subscription id + PushTelemetryResponse response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.UNKNOWN_SUBSCRIPTION_ID.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(0, telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // unsupported compression type + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.UNSUPPORTED_COMPRESSION_TYPE.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(0, telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // telemetry too large + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.TELEMETRY_TOO_LARGE.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(20000, telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // throttling quota exceeded + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.THROTTLING_QUOTA_EXCEEDED.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(20000, telemetrySender.intervalMs()); + assertTrue(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // invalid request error + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.INVALID_REQUEST.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // unsupported version error + telemetrySender.enabled(true); + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.UNSUPPORTED_VERSION.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // invalid record + telemetrySender.enabled(true); + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.INVALID_RECORD.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_NEEDED)); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.PUSH_IN_PROGRESS)); + + // unknown error + telemetrySender.enabled(true); + response = new PushTelemetryResponse( + new PushTelemetryResponseData().setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code())); + + telemetrySender.handleResponse(response); + assertEquals(ClientTelemetryState.SUBSCRIPTION_NEEDED, telemetrySender.state()); + assertEquals(Integer.MAX_VALUE, telemetrySender.intervalMs()); + assertFalse(telemetrySender.enabled()); + } + + @Test + public void testClientInstanceId() throws InterruptedException { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertTrue(telemetrySender.maybeSetState(ClientTelemetryState.SUBSCRIPTION_IN_PROGRESS)); + + CountDownLatch lock = new CountDownLatch(2); + + AtomicReference> clientInstanceId = new AtomicReference<>(); + new Thread(() -> { + try { + clientInstanceId.set(telemetrySender.clientInstanceId(Duration.ofMillis(10000))); + } finally { + lock.countDown(); + } + }).start(); + + new Thread(() -> { + try { + telemetrySender.updateSubscriptionResult(subscription, time.milliseconds()); + } finally { + lock.countDown(); + } + }).start(); + + assertTrue(lock.await(2000, TimeUnit.MILLISECONDS)); + assertNotNull(clientInstanceId.get()); + assertTrue(clientInstanceId.get().isPresent()); + assertEquals(uuid, clientInstanceId.get().get()); + } + + @Test + public void testComputeStaggeredIntervalMs() { + ClientTelemetryReporter.DefaultClientTelemetrySender telemetrySender = (ClientTelemetryReporter.DefaultClientTelemetrySender) clientTelemetryReporter.telemetrySender(); + assertEquals(0, telemetrySender.computeStaggeredIntervalMs(0, 0.5, 1.5)); + assertEquals(1, telemetrySender.computeStaggeredIntervalMs(1, 0.99, 1)); + long timeMs = telemetrySender.computeStaggeredIntervalMs(1000, 0.5, 1.5); + assertTrue(timeMs >= 500 && timeMs <= 1500); + } + + @AfterEach + public void tearDown() { + clientTelemetryReporter.close(); + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtilsTest.java new file mode 100644 index 0000000000000..9897e10229516 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/ClientTelemetryUtilsTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ClientTelemetryUtilsTest { + + @Test + public void testMaybeFetchErrorIntervalMs() { + assertEquals(Optional.empty(), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.NONE.code(), -1)); + assertEquals(Optional.of(Integer.MAX_VALUE), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.INVALID_REQUEST.code(), -1)); + assertEquals(Optional.of(Integer.MAX_VALUE), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.INVALID_RECORD.code(), -1)); + assertEquals(Optional.of(0), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.UNKNOWN_SUBSCRIPTION_ID.code(), -1)); + assertEquals(Optional.of(0), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.UNSUPPORTED_COMPRESSION_TYPE.code(), -1)); + assertEquals(Optional.of(ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.TELEMETRY_TOO_LARGE.code(), -1)); + assertEquals(Optional.of(20000), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.TELEMETRY_TOO_LARGE.code(), 20000)); + assertEquals(Optional.of(ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.THROTTLING_QUOTA_EXCEEDED.code(), -1)); + assertEquals(Optional.of(20000), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.THROTTLING_QUOTA_EXCEEDED.code(), 20000)); + assertEquals(Optional.of(Integer.MAX_VALUE), ClientTelemetryUtils.maybeFetchErrorIntervalMs(Errors.UNKNOWN_SERVER_ERROR.code(), -1)); + } + + @Test + public void testGetSelectorFromRequestedMetrics() { + // no metrics selector + assertEquals(ClientTelemetryUtils.SELECTOR_NO_METRICS, ClientTelemetryUtils.getSelectorFromRequestedMetrics(Collections.emptyList())); + assertEquals(ClientTelemetryUtils.SELECTOR_NO_METRICS, ClientTelemetryUtils.getSelectorFromRequestedMetrics(null)); + // all metrics selector + assertEquals(ClientTelemetryUtils.SELECTOR_ALL_METRICS, ClientTelemetryUtils.getSelectorFromRequestedMetrics(Collections.singletonList("*"))); + // specific metrics selector + Predicate selector = ClientTelemetryUtils.getSelectorFromRequestedMetrics(Arrays.asList("metric1", "metric2")); + assertNotEquals(ClientTelemetryUtils.SELECTOR_NO_METRICS, selector); + assertNotEquals(ClientTelemetryUtils.SELECTOR_ALL_METRICS, selector); + assertTrue(selector.test(new MetricKey("metric1.test"))); + assertTrue(selector.test(new MetricKey("metric2.test"))); + assertFalse(selector.test(new MetricKey("test.metric1"))); + assertFalse(selector.test(new MetricKey("test.metric2"))); + } + + @Test + public void testGetCompressionTypesFromAcceptedList() { + assertEquals(0, ClientTelemetryUtils.getCompressionTypesFromAcceptedList(null).size()); + assertEquals(0, ClientTelemetryUtils.getCompressionTypesFromAcceptedList(Collections.emptyList()).size()); + + List compressionTypes = new ArrayList<>(); + compressionTypes.add(CompressionType.GZIP.id); + compressionTypes.add(CompressionType.LZ4.id); + compressionTypes.add(CompressionType.SNAPPY.id); + compressionTypes.add(CompressionType.ZSTD.id); + compressionTypes.add(CompressionType.NONE.id); + compressionTypes.add((byte) -1); + + // should take the first compression type + assertEquals(5, ClientTelemetryUtils.getCompressionTypesFromAcceptedList(compressionTypes).size()); + } + + @Test + public void testValidateClientInstanceId() { + assertThrows(IllegalArgumentException.class, () -> ClientTelemetryUtils.validateClientInstanceId(null)); + assertThrows(IllegalArgumentException.class, () -> ClientTelemetryUtils.validateClientInstanceId(Uuid.ZERO_UUID)); + + Uuid uuid = Uuid.randomUuid(); + assertEquals(uuid, ClientTelemetryUtils.validateClientInstanceId(uuid)); + } + + @ParameterizedTest + @ValueSource(ints = {300_000, Integer.MAX_VALUE - 1, Integer.MAX_VALUE}) + public void testValidateIntervalMsValid(int pushIntervalMs) { + assertEquals(pushIntervalMs, ClientTelemetryUtils.validateIntervalMs(pushIntervalMs)); + } + + @ParameterizedTest + @ValueSource(ints = {-1, 0}) + public void testValidateIntervalMsInvalid(int pushIntervalMs) { + assertEquals(ClientTelemetryReporter.DEFAULT_PUSH_INTERVAL_MS, ClientTelemetryUtils.validateIntervalMs(pushIntervalMs)); + } + + @Test + public void testPreferredCompressionType() { + assertEquals(CompressionType.NONE, ClientTelemetryUtils.preferredCompressionType(Collections.emptyList())); + assertEquals(CompressionType.NONE, ClientTelemetryUtils.preferredCompressionType(null)); + } +} \ No newline at end of file diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollectorTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollectorTest.java new file mode 100644 index 0000000000000..bb4e0cdc2d3b6 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/KafkaMetricsCollectorTest.java @@ -0,0 +1,605 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.AggregationTemporality; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; + +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.metrics.Gauge; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.metrics.MetricsReporter; +import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.CumulativeSum; +import org.apache.kafka.common.metrics.stats.WindowedCount; +import org.apache.kafka.common.utils.MockTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class KafkaMetricsCollectorTest { + + private static final String DOMAIN = "test.domain"; + + private MetricName metricName; + private Map tags; + private Metrics metrics; + private MetricNamingStrategy metricNamingStrategy; + private KafkaMetricsCollector collector; + + private TestEmitter testEmitter; + private MockTime time; + + @BeforeEach + public void setUp() { + metrics = new Metrics(); + tags = Collections.singletonMap("tag", "value"); + metricName = metrics.metricName("name1", "group1", tags); + time = new MockTime(0, 1000L, TimeUnit.MILLISECONDS.toNanos(1000L)); + testEmitter = new TestEmitter(); + + // Define metric naming strategy. + metricNamingStrategy = TelemetryMetricNamingConvention.getClientTelemetryMetricNamingStrategy(DOMAIN); + + // Define collector to test. + collector = new KafkaMetricsCollector( + metricNamingStrategy, + time + ); + + // Add reporter to metrics. + metrics.addReporter(getTestMetricsReporter()); + } + + @Test + public void testMeasurableCounter() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new WindowedCount()); + + sensor.record(); + sensor.record(); + + time.sleep(60 * 1000L); + + // Collect metrics. + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 2 Kafka measurables since Metrics always includes a count measurable. + assertEquals(2, result.size()); + + Metric counter = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + assertTrue(counter.hasSum()); + assertEquals(tags, getTags(counter.getSum().getDataPoints(0).getAttributesList())); + + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, counter.getSum().getAggregationTemporality()); + assertTrue(counter.getSum().getIsMonotonic()); + NumberDataPoint point = counter.getSum().getDataPoints(0); + assertEquals(2d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(61L).getEpochSecond()) + + Instant.ofEpochSecond(61L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(1L).getEpochSecond()) + + Instant.ofEpochSecond(1L).getNano(), point.getStartTimeUnixNano()); + } + + @Test + public void testMeasurableCounterDeltaMetrics() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new WindowedCount()); + + sensor.record(); + sensor.record(); + + time.sleep(60 * 1000L); + + // Collect delta metrics. + testEmitter.onlyDeltaMetrics(true); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 2 Kafka measurables since Metrics always includes a count measurable. + assertEquals(2, result.size()); + + Metric counter = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + assertTrue(counter.hasSum()); + assertEquals(tags, getTags(counter.getSum().getDataPoints(0).getAttributesList())); + + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, counter.getSum().getAggregationTemporality()); + assertTrue(counter.getSum().getIsMonotonic()); + NumberDataPoint point = counter.getSum().getDataPoints(0); + assertEquals(2d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(61L).getEpochSecond()) + + Instant.ofEpochSecond(61L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(1L).getEpochSecond()) + + Instant.ofEpochSecond(1L).getNano(), point.getStartTimeUnixNano()); + } + + @Test + public void testMeasurableTotal() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new CumulativeSum()); + + sensor.record(10L); + sensor.record(5L); + + // Collect metrics. + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 2 Kafka measurables since Metrics always includes a count measurable. + assertEquals(2, result.size()); + + Metric counter = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + assertTrue(counter.hasSum()); + assertEquals(tags, getTags(counter.getSum().getDataPoints(0).getAttributesList())); + assertEquals(15, counter.getSum().getDataPoints(0).getAsDouble(), 0.0); + } + + @Test + public void testMeasurableTotalDeltaMetrics() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new CumulativeSum()); + + sensor.record(10L); + sensor.record(5L); + + // Collect metrics. + testEmitter.onlyDeltaMetrics(true); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 2 Kafka measurables since Metrics always includes a count measurable. + assertEquals(2, result.size()); + + Metric counter = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + assertTrue(counter.hasSum()); + assertEquals(tags, getTags(counter.getSum().getDataPoints(0).getAttributesList())); + assertEquals(15, counter.getSum().getDataPoints(0).getAsDouble(), 0.0); + } + + @Test + public void testMeasurableGauge() { + metrics.addMetric(metricName, (config, now) -> 100.0); + + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 2 Kafka measurables since Metrics always includes a count measurable. + assertEquals(2, result.size()); + + Metric counter = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + assertTrue(counter.hasGauge()); + assertEquals(tags, getTags(counter.getGauge().getDataPoints(0).getAttributesList())); + assertEquals(100L, counter.getGauge().getDataPoints(0).getAsDouble(), 0.0); + } + + @Test + public void testNonMeasurable() { + metrics.addMetric(metrics.metricName("float", "group1", tags), (Gauge) (config, now) -> 99f); + metrics.addMetric(metrics.metricName("double", "group1", tags), (Gauge) (config, now) -> 99d); + metrics.addMetric(metrics.metricName("int", "group1", tags), (Gauge) (config, now) -> 100); + metrics.addMetric(metrics.metricName("long", "group1", tags), (Gauge) (config, now) -> 100L); + + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 5 Kafka measurables since Metrics always includes a count measurable. + assertEquals(5, result.size()); + + result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.(float|double)")).forEach( + doubleGauge -> { + assertTrue(doubleGauge.hasGauge()); + assertEquals(tags, getTags(doubleGauge.getGauge().getDataPoints(0).getAttributesList())); + assertEquals(99d, doubleGauge.getGauge().getDataPoints(0).getAsDouble(), 0.0); + }); + + result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.(int|long)")).forEach( + intGauge -> { + assertTrue(intGauge.hasGauge()); + assertEquals(tags, getTags(intGauge.getGauge().getDataPoints(0).getAttributesList())); + assertEquals(100, intGauge.getGauge().getDataPoints(0).getAsDouble(), 0.0); + }); + } + + @Test + public void testMeasurableWithException() { + metrics.addMetric(metricName, null, (config, now) -> { + throw new RuntimeException(); + }); + + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + //Verify only the global count of metrics exist + assertEquals(1, result.size()); + // Group is registered as kafka-metrics-count + assertEquals("test.domain.kafka.count.count", result.get(0).builder().build().getName()); + //Verify metrics with measure() method throw exception is not returned + assertFalse(result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .anyMatch(metric -> metric.getName().equals("test.domain.group1.name1"))); + } + + @Test + public void testMetricRemoval() { + metrics.addMetric(metricName, (config, now) -> 100.0); + + collector.collect(testEmitter); + assertEquals(2, testEmitter.emittedMetrics().size()); + + metrics.removeMetric(metricName); + assertFalse(collector.getTrackedMetrics().contains(metricNamingStrategy.metricKey(metricName))); + + // verify that the metric was removed. + testEmitter.reset(); + collector.collect(testEmitter); + List collected = testEmitter.emittedMetrics(); + assertEquals(1, collected.size()); + assertEquals("test.domain.kafka.count.count", collected.get(0).builder().build().getName()); + } + + @Test + public void testSecondCollectCumulative() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new CumulativeSum()); + + sensor.record(); + sensor.record(); + time.sleep(60 * 1000L); + + collector.collect(testEmitter); + + // Update it again by 5 and advance time by another 60 seconds. + sensor.record(); + sensor.record(); + sensor.record(); + sensor.record(); + sensor.record(); + time.sleep(60 * 1000L); + + testEmitter.reset(); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + assertEquals(2, result.size()); + + + Metric cumulative = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + NumberDataPoint point = cumulative.getSum().getDataPoints(0); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, cumulative.getSum().getAggregationTemporality()); + assertTrue(cumulative.getSum().getIsMonotonic()); + assertEquals(7d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(121L).getEpochSecond()) + + Instant.ofEpochSecond(121L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(1L).getEpochSecond()) + + Instant.ofEpochSecond(1L).getNano(), point.getStartTimeUnixNano()); + } + + @Test + public void testSecondDeltaCollectDouble() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new CumulativeSum()); + + sensor.record(); + sensor.record(); + time.sleep(60 * 1000L); + + testEmitter.onlyDeltaMetrics(true); + collector.collect(testEmitter); + + // Update it again by 5 and advance time by another 60 seconds. + sensor.record(); + sensor.record(); + sensor.record(); + sensor.record(); + sensor.record(); + time.sleep(60 * 1000L); + + testEmitter.reset(); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + assertEquals(2, result.size()); + + Metric cumulative = result.stream() + .flatMap(metrics -> Stream.of(metrics.builder().build())) + .filter(metric -> metric.getName().equals("test.domain.group1.name1")).findFirst().get(); + + NumberDataPoint point = cumulative.getSum().getDataPoints(0); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, cumulative.getSum().getAggregationTemporality()); + assertTrue(cumulative.getSum().getIsMonotonic()); + assertEquals(5d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(121L).getEpochSecond()) + + Instant.ofEpochSecond(121L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(61L).getEpochSecond()) + + Instant.ofEpochSecond(61L).getNano(), point.getStartTimeUnixNano()); + } + + @Test + public void testCollectFilter() { + metrics.addMetric(metricName, (config, now) -> 100.0); + + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".count")); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // Should get exactly 1 Kafka measurables because we excluded the count measurable + assertEquals(1, result.size()); + + Metric counter = result.get(0).builder().build(); + + assertTrue(counter.hasGauge()); + assertEquals(100L, counter.getGauge().getDataPoints(0).getAsDouble(), 0.0); + } + + @Test + public void testCollectFilterWithCumulativeTemporality() { + MetricName name1 = metrics.metricName("nonMeasurable", "group1", tags); + MetricName name2 = metrics.metricName("windowed", "group1", tags); + MetricName name3 = metrics.metricName("cumulative", "group1", tags); + + metrics.addMetric(name1, (Gauge) (config, now) -> 99d); + + Sensor sensor = metrics.sensor("test"); + sensor.add(name2, new WindowedCount()); + sensor.add(name3, new CumulativeSum()); + + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // no-filter shall result in all 4 data metrics. + assertEquals(4, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".count")); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Drop metrics for Count type. + assertEquals(3, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".nonmeasurable")); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Drop non-measurable metric. + assertEquals(3, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(key -> true); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Again no filter. + assertEquals(4, result.size()); + } + + @Test + public void testCollectFilterWithDeltaTemporality() { + MetricName name1 = metrics.metricName("nonMeasurable", "group1", tags); + MetricName name2 = metrics.metricName("windowed", "group1", tags); + MetricName name3 = metrics.metricName("cumulative", "group1", tags); + + metrics.addMetric(name1, (Gauge) (config, now) -> 99d); + + Sensor sensor = metrics.sensor("test"); + sensor.add(name2, new WindowedCount()); + sensor.add(name3, new CumulativeSum()); + + testEmitter.onlyDeltaMetrics(true); + collector.collect(testEmitter); + List result = testEmitter.emittedMetrics(); + + // no-filter shall result in all 4 data metrics. + assertEquals(4, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".count")); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Drop metrics for Count type. + assertEquals(3, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".nonmeasurable")); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Drop non-measurable metric. + assertEquals(3, result.size()); + + testEmitter.reset(); + testEmitter.reconfigurePredicate(key -> true); + collector.collect(testEmitter); + result = testEmitter.emittedMetrics(); + + // Again no filter. + assertEquals(4, result.size()); + } + + @Test + public void testCollectMetricsWithTemporalityChange() { + Sensor sensor = metrics.sensor("test"); + sensor.add(metricName, new WindowedCount()); + testEmitter.reconfigurePredicate(k -> !k.key().name().endsWith(".count")); + + // Emit metrics as cumulative, verify the current time is 60 seconds ahead of start time. + sensor.record(); + time.sleep(60 * 1000L); + collector.collect(testEmitter); + + List result = testEmitter.emittedMetrics(); + Metric counter = result.get(0).builder().build(); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, counter.getSum().getAggregationTemporality()); + NumberDataPoint point = counter.getSum().getDataPoints(0); + assertEquals(1d, point.getAsDouble()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(61L).getEpochSecond()) + + Instant.ofEpochSecond(61L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(1L).getEpochSecond()) + + Instant.ofEpochSecond(1L).getNano(), point.getStartTimeUnixNano()); + + // Again emit metrics as cumulative, verify the start time is unchanged and current time is + // advanced by 60 seconds again. + time.sleep(60 * 1000L); + sensor.record(); + testEmitter.reset(); + collector.collect(testEmitter); + + result = testEmitter.emittedMetrics(); + counter = result.get(0).builder().build(); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, counter.getSum().getAggregationTemporality()); + point = counter.getSum().getDataPoints(0); + assertEquals(2d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(121L).getEpochSecond()) + + Instant.ofEpochSecond(121L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(1L).getEpochSecond()) + + Instant.ofEpochSecond(1L).getNano(), point.getStartTimeUnixNano()); + + + // Change Temporality. Emit metrics as delta, verify the temporality changes to delta and start time is reset to + // current time. + time.sleep(60 * 1000L); + sensor.record(); + testEmitter.reset(); + testEmitter.onlyDeltaMetrics(true); + collector.metricsReset(); + collector.collect(testEmitter); + + result = testEmitter.emittedMetrics(); + counter = result.get(0).builder().build(); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, counter.getSum().getAggregationTemporality()); + point = counter.getSum().getDataPoints(0); + assertEquals(3d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(181L).getEpochSecond()) + + Instant.ofEpochSecond(181L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(181L).getEpochSecond()) + + Instant.ofEpochSecond(181L).getNano(), point.getStartTimeUnixNano()); + + // Again emit metrics as delta, verify the start time is tracked properly and only delta value + // is present on response. + time.sleep(60 * 1000L); + sensor.record(); + testEmitter.reset(); + collector.collect(testEmitter); + + result = testEmitter.emittedMetrics(); + counter = result.get(0).builder().build(); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, counter.getSum().getAggregationTemporality()); + point = counter.getSum().getDataPoints(0); + assertEquals(1d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(241L).getEpochSecond()) + + Instant.ofEpochSecond(241L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(181L).getEpochSecond()) + + Instant.ofEpochSecond(181L).getNano(), point.getStartTimeUnixNano()); + + // Change Temporality. Emit metrics as cumulative, verify the temporality changes to cumulative + // and start time is reset to current time. + time.sleep(60 * 1000L); + sensor.record(); + testEmitter.reset(); + testEmitter.onlyDeltaMetrics(false); + collector.metricsReset(); + collector.collect(testEmitter); + + result = testEmitter.emittedMetrics(); + counter = result.get(0).builder().build(); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, counter.getSum().getAggregationTemporality()); + point = counter.getSum().getDataPoints(0); + assertEquals(5d, point.getAsDouble(), 0.0); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(301L).getEpochSecond()) + + Instant.ofEpochSecond(301L).getNano(), point.getTimeUnixNano()); + assertEquals(TimeUnit.SECONDS.toNanos(Instant.ofEpochSecond(301L).getEpochSecond()) + + Instant.ofEpochSecond(301L).getNano(), point.getStartTimeUnixNano()); + } + + private MetricsReporter getTestMetricsReporter() { + // Inline implementation of MetricsReporter for testing. + return new MetricsReporter() { + @Override + public void init(List metrics) { + collector.init(metrics); + } + + @Override + public void metricChange(KafkaMetric metric) { + collector.metricChange(metric); + } + + @Override + public void metricRemoval(KafkaMetric metric) { + collector.metricRemoval(metric); + } + + @Override + public void close() { + // do nothing + } + + @Override + public void configure(Map configs) { + // do nothing + } + }; + } + + public static Map getTags(List attributes) { + return attributes.stream() + .filter(attr -> attr.getValue().hasStringValue()) + .collect(Collectors.toMap( + KeyValue::getKey, + attr -> attr.getValue().getStringValue() + )); + } +} \ No newline at end of file diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/LastValueTrackerTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/LastValueTrackerTest.java new file mode 100644 index 0000000000000..a5947ae58ad21 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/LastValueTrackerTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import org.apache.kafka.common.telemetry.internals.LastValueTracker.InstantAndValue; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Collections; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class LastValueTrackerTest { + + private static final MetricKey METRIC_NAME = new MetricKey("test-metric", Collections.emptyMap()); + + private final Instant instant1 = Instant.now(); + private final Instant instant2 = instant1.plusMillis(1); + + @Test + public void testGetAndSetDouble() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + Optional> result = lastValueTracker.getAndSet(METRIC_NAME, instant1, 1d); + assertFalse(result.isPresent()); + } + + @Test + public void testGetAndSetDoubleWithTrackedValue() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + lastValueTracker.getAndSet(METRIC_NAME, instant1, 1d); + + Optional> result = lastValueTracker + .getAndSet(METRIC_NAME, instant2, 1000d); + + assertTrue(result.isPresent()); + assertEquals(instant1, result.get().getIntervalStart()); + assertEquals(1d, result.get().getValue(), 1e-6); + } + + @Test + public void testGetAndSetLong() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + Optional> result = lastValueTracker.getAndSet(METRIC_NAME, instant1, 1L); + assertFalse(result.isPresent()); + } + + @Test + public void testGetAndSetLongWithTrackedValue() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + lastValueTracker.getAndSet(METRIC_NAME, instant1, 2L); + Optional> result = lastValueTracker + .getAndSet(METRIC_NAME, instant2, 10000L); + + assertTrue(result.isPresent()); + assertEquals(instant1, result.get().getIntervalStart()); + assertEquals(2L, result.get().getValue().longValue()); + } + + @Test + public void testRemove() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + lastValueTracker.getAndSet(METRIC_NAME, instant1, 1d); + + assertTrue(lastValueTracker.contains(METRIC_NAME)); + + InstantAndValue result = lastValueTracker.remove(METRIC_NAME); + assertNotNull(result); + assertEquals(instant1, result.getIntervalStart()); + assertEquals(1d, result.getValue().longValue()); + } + + @Test + public void testRemoveWithNullKey() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + assertThrows(NullPointerException.class, () -> lastValueTracker.remove(null)); + } + + @Test + public void testRemoveWithInvalidKey() { + LastValueTracker lastValueTracker = new LastValueTracker<>(); + assertFalse(lastValueTracker.contains(METRIC_NAME)); + + InstantAndValue result = lastValueTracker.remove(METRIC_NAME); + assertNull(result); + } +} \ No newline at end of file diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/SinglePointMetricTest.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/SinglePointMetricTest.java new file mode 100644 index 0000000000000..97801b1e06904 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/SinglePointMetricTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import io.opentelemetry.proto.metrics.v1.AggregationTemporality; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SinglePointMetricTest { + + private MetricKey metricKey; + private Instant now; + + /* + Test compares the metric representation from returned builder to ensure that the metric is + constructed correctly. + + For example: Gauge metric with name "name" and double value 1.0 at certain time is represented as: + + name: "name" + gauge { + data_points { + time_unix_nano: 1698063981021420000 + as_double: 1.0 + } + } + */ + + @BeforeEach + public void setUp() { + metricKey = new MetricKey("name", Collections.emptyMap()); + now = Instant.now(); + } + + @Test + public void testGaugeWithNumberValue() { + SinglePointMetric gaugeNumber = SinglePointMetric.gauge(metricKey, Long.valueOf(1), now); + MetricKey metricKey = gaugeNumber.key(); + assertEquals("name", metricKey.name()); + + Metric metric = gaugeNumber.builder().build(); + assertEquals(1, metric.getGauge().getDataPointsCount()); + + NumberDataPoint point = metric.getGauge().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(0, point.getStartTimeUnixNano()); + assertEquals(1, point.getAsInt()); + assertEquals(0, point.getAttributesCount()); + } + + @Test + public void testGaugeWithDoubleValue() { + SinglePointMetric gaugeNumber = SinglePointMetric.gauge(metricKey, 1.0, now); + MetricKey metricKey = gaugeNumber.key(); + assertEquals("name", metricKey.name()); + + Metric metric = gaugeNumber.builder().build(); + assertEquals(1, metric.getGauge().getDataPointsCount()); + + NumberDataPoint point = metric.getGauge().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(0, point.getStartTimeUnixNano()); + assertEquals(1.0, point.getAsDouble()); + assertEquals(0, point.getAttributesCount()); + } + + @Test + public void testGaugeWithMetricTags() { + MetricKey metricKey = new MetricKey("name", Collections.singletonMap("tag", "value")); + SinglePointMetric gaugeNumber = SinglePointMetric.gauge(metricKey, 1.0, now); + + MetricKey key = gaugeNumber.key(); + assertEquals("name", key.name()); + + Metric metric = gaugeNumber.builder().build(); + assertEquals(1, metric.getGauge().getDataPointsCount()); + + NumberDataPoint point = metric.getGauge().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(0, point.getStartTimeUnixNano()); + assertEquals(1.0, point.getAsDouble()); + assertEquals(1, point.getAttributesCount()); + assertEquals("tag", point.getAttributes(0).getKey()); + assertEquals("value", point.getAttributes(0).getValue().getStringValue()); + } + + @Test + public void testSum() { + SinglePointMetric sum = SinglePointMetric.sum(metricKey, 1.0, false, now); + + MetricKey key = sum.key(); + assertEquals("name", key.name()); + + Metric metric = sum.builder().build(); + assertFalse(metric.getSum().getIsMonotonic()); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, metric.getSum().getAggregationTemporality()); + assertEquals(1, metric.getSum().getDataPointsCount()); + + NumberDataPoint point = metric.getSum().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(0, point.getStartTimeUnixNano()); + assertEquals(1.0, point.getAsDouble()); + assertEquals(0, point.getAttributesCount()); + } + + @Test + public void testSumWithStartTimeAndTags() { + MetricKey metricKey = new MetricKey("name", Collections.singletonMap("tag", "value")); + SinglePointMetric sum = SinglePointMetric.sum(metricKey, 1.0, true, now, now); + + MetricKey key = sum.key(); + assertEquals("name", key.name()); + + Metric metric = sum.builder().build(); + assertTrue(metric.getSum().getIsMonotonic()); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE, metric.getSum().getAggregationTemporality()); + assertEquals(1, metric.getSum().getDataPointsCount()); + + NumberDataPoint point = metric.getSum().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getStartTimeUnixNano()); + assertEquals(1.0, point.getAsDouble()); + assertEquals(1, point.getAttributesCount()); + assertEquals("tag", point.getAttributes(0).getKey()); + assertEquals("value", point.getAttributes(0).getValue().getStringValue()); + } + + @Test + public void testDeltaSum() { + SinglePointMetric sum = SinglePointMetric.deltaSum(metricKey, 1.0, true, now, now); + + MetricKey key = sum.key(); + assertEquals("name", key.name()); + + Metric metric = sum.builder().build(); + assertTrue(metric.getSum().getIsMonotonic()); + assertEquals(AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA, metric.getSum().getAggregationTemporality()); + assertEquals(1, metric.getSum().getDataPointsCount()); + + NumberDataPoint point = metric.getSum().getDataPoints(0); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getTimeUnixNano()); + assertEquals(now.getEpochSecond() * Math.pow(10, 9) + now.getNano(), point.getStartTimeUnixNano()); + assertEquals(1.0, point.getAsDouble()); + assertEquals(0, point.getAttributesCount()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/telemetry/internals/TestEmitter.java b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/TestEmitter.java new file mode 100644 index 0000000000000..bcc5a86855847 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/telemetry/internals/TestEmitter.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.telemetry.internals; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; + +public class TestEmitter implements MetricsEmitter { + + private final List emittedMetrics; + private Predicate metricsPredicate = metricKeyable -> true; + private boolean onlyDeltaMetrics; + + public TestEmitter() { + this(false); + } + + public TestEmitter(boolean onlyDeltaMetrics) { + this.emittedMetrics = new ArrayList<>(); + this.onlyDeltaMetrics = onlyDeltaMetrics; + } + + @Override + public boolean shouldEmitMetric(MetricKeyable metricKeyable) { + return metricsPredicate.test(metricKeyable); + } + + @Override + public boolean shouldEmitDeltaMetrics() { + return onlyDeltaMetrics; + } + + @Override + public boolean emitMetric(SinglePointMetric metric) { + return emittedMetrics.add(metric); + } + + @Override + public List emittedMetrics() { + return Collections.unmodifiableList(emittedMetrics); + } + + public void reset() { + this.emittedMetrics.clear(); + } + + public void onlyDeltaMetrics(boolean onlyDeltaMetrics) { + this.onlyDeltaMetrics = onlyDeltaMetrics; + } + + public void reconfigurePredicate(Predicate metricsPredicate) { + this.metricsPredicate = metricsPredicate; + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/utils/AbstractIteratorTest.java b/clients/src/test/java/org/apache/kafka/common/utils/AbstractIteratorTest.java index 939b2f6daa205..5f3ca4ddf0d17 100644 --- a/clients/src/test/java/org/apache/kafka/common/utils/AbstractIteratorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/utils/AbstractIteratorTest.java @@ -61,7 +61,7 @@ public ListIterator(List l) { this.list = l; } - public T makeNext() { + protected T makeNext() { if (position < list.size()) return list.get(position++); else diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ConfigUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ConfigUtilsTest.java index d760330eb2bc7..540229bd65597 100644 --- a/clients/src/test/java/org/apache/kafka/common/utils/ConfigUtilsTest.java +++ b/clients/src/test/java/org/apache/kafka/common/utils/ConfigUtilsTest.java @@ -168,4 +168,30 @@ public void testConfigMapToRedactedStringWithSecrets() { assertEquals("{myInt=123, myPassword=(redacted), myString=\"whatever\", myString2=null, myUnknown=(redacted)}", ConfigUtils.configMapToRedactedString(testMap1, CONFIG)); } + + @Test + public void testGetBoolean() { + String key = "test.key"; + Boolean defaultValue = true; + + Map config = new HashMap<>(); + config.put("some.other.key", false); + assertEquals(defaultValue, ConfigUtils.getBoolean(config, key, defaultValue)); + + config = new HashMap<>(); + config.put(key, false); + assertEquals(false, ConfigUtils.getBoolean(config, key, defaultValue)); + + config = new HashMap<>(); + config.put(key, "false"); + assertEquals(false, ConfigUtils.getBoolean(config, key, defaultValue)); + + config = new HashMap<>(); + config.put(key, "not-a-boolean"); + assertEquals(false, ConfigUtils.getBoolean(config, key, defaultValue)); + + config = new HashMap<>(); + config.put(key, 5); + assertEquals(defaultValue, ConfigUtils.getBoolean(config, key, defaultValue)); + } } diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index 3fc51f23a2bad..c8a6db6f6ca32 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -600,7 +600,7 @@ public static ApiVersionsResponse defaultApiVersionsResponse( ) { return createApiVersionsResponse( throttleTimeMs, - ApiVersionsResponse.filterApis(RecordVersion.current(), listenerType, true), + ApiVersionsResponse.filterApis(RecordVersion.current(), listenerType, true, true), Features.emptySupportedFeatures(), false ); @@ -613,7 +613,7 @@ public static ApiVersionsResponse defaultApiVersionsResponse( ) { return createApiVersionsResponse( throttleTimeMs, - ApiVersionsResponse.filterApis(RecordVersion.current(), listenerType, enableUnstableLastVersion), + ApiVersionsResponse.filterApis(RecordVersion.current(), listenerType, enableUnstableLastVersion, true), Features.emptySupportedFeatures(), false ); diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java index 84e393edb36dd..cad57d4ad02f0 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java @@ -103,12 +103,8 @@ public void start(Map props) { consumer = MirrorUtils.newConsumer(config.sourceConsumerConfig("replication-consumer")); offsetProducer = MirrorUtils.newProducer(config.offsetSyncsTopicProducerConfig()); Set taskTopicPartitions = config.taskTopicPartitions(); - Map topicPartitionOffsets = loadOffsets(taskTopicPartitions); - consumer.assign(topicPartitionOffsets.keySet()); - log.info("Starting with {} previously uncommitted partitions.", topicPartitionOffsets.entrySet().stream() - .filter(x -> x.getValue() == 0L).count()); - log.trace("Seeking offsets: {}", topicPartitionOffsets); - topicPartitionOffsets.forEach(consumer::seek); + initializeConsumer(taskTopicPartitions); + log.info("{} replicating {} topic-partitions {}->{}: {}.", Thread.currentThread().getName(), taskTopicPartitions.size(), sourceClusterAlias, config.targetClusterAlias(), taskTopicPartitions); } @@ -266,7 +262,26 @@ private Map loadOffsets(Set topicPartition private Long loadOffset(TopicPartition topicPartition) { Map wrappedPartition = MirrorUtils.wrapPartition(topicPartition, sourceClusterAlias); Map wrappedOffset = context.offsetStorageReader().offset(wrappedPartition); - return MirrorUtils.unwrapOffset(wrappedOffset) + 1; + return MirrorUtils.unwrapOffset(wrappedOffset); + } + + // visible for testing + void initializeConsumer(Set taskTopicPartitions) { + Map topicPartitionOffsets = loadOffsets(taskTopicPartitions); + consumer.assign(topicPartitionOffsets.keySet()); + log.info("Starting with {} previously uncommitted partitions.", topicPartitionOffsets.values().stream() + .filter(this::isUncommitted).count()); + + topicPartitionOffsets.forEach((topicPartition, offset) -> { + // Do not call seek on partitions that don't have an existing offset committed. + if (isUncommitted(offset)) { + log.trace("Skipping seeking offset for topicPartition: {}", topicPartition); + return; + } + long nextOffsetToCommittedOffset = offset + 1L; + log.trace("Seeking to offset {} for topicPartition: {}", nextOffsetToCommittedOffset, topicPartition); + consumer.seek(topicPartition, nextOffsetToCommittedOffset); + }); } // visible for testing @@ -302,6 +317,10 @@ private static int byteSize(byte[] bytes) { } } + private boolean isUncommitted(Long offset) { + return offset == null || offset < 0; + } + static class PartitionState { long previousUpstreamOffset = -1L; long previousDownstreamOffset = -1L; diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java index 0c566eb596bc8..647935eb35662 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceTaskTest.java @@ -31,25 +31,33 @@ import org.apache.kafka.connect.mirror.MirrorSourceTask.PartitionState; import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTaskContext; +import org.apache.kafka.connect.storage.OffsetStorageReader; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Semaphore; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static org.mockito.Mockito.verifyNoInteractions; @@ -214,6 +222,61 @@ public void testPoll() { } } + @Test + public void testSeekBehaviorDuringStart() { + // Setting up mock behavior. + @SuppressWarnings("unchecked") + KafkaConsumer mockConsumer = mock(KafkaConsumer.class); + + SourceTaskContext mockSourceTaskContext = mock(SourceTaskContext.class); + OffsetStorageReader mockOffsetStorageReader = mock(OffsetStorageReader.class); + when(mockSourceTaskContext.offsetStorageReader()).thenReturn(mockOffsetStorageReader); + + Set topicPartitions = new HashSet<>(Arrays.asList( + new TopicPartition("previouslyReplicatedTopic", 8), + new TopicPartition("previouslyReplicatedTopic1", 0), + new TopicPartition("previouslyReplicatedTopic", 1), + new TopicPartition("newTopicToReplicate1", 1), + new TopicPartition("newTopicToReplicate1", 4), + new TopicPartition("newTopicToReplicate2", 0) + )); + + long arbitraryCommittedOffset = 4L; + long offsetToSeek = arbitraryCommittedOffset + 1L; + when(mockOffsetStorageReader.offset(anyMap())).thenAnswer(testInvocation -> { + Map topicPartitionOffsetMap = testInvocation.getArgument(0); + String topicName = topicPartitionOffsetMap.get("topic").toString(); + + // Only return the offset for previously replicated topics. + // For others, there is no value set. + if (topicName.startsWith("previouslyReplicatedTopic")) { + topicPartitionOffsetMap.put("offset", arbitraryCommittedOffset); + } + return topicPartitionOffsetMap; + }); + + MirrorSourceTask mirrorSourceTask = new MirrorSourceTask(mockConsumer, null, null, + new DefaultReplicationPolicy(), 50, null, null, null, null); + mirrorSourceTask.initialize(mockSourceTaskContext); + + // Call test subject + mirrorSourceTask.initializeConsumer(topicPartitions); + + // Verifications + // Ensure all the topic partitions are assigned to consumer + verify(mockConsumer, times(1)).assign(topicPartitions); + + // Ensure seek is only called for previously committed topic partitions. + verify(mockConsumer, times(1)) + .seek(new TopicPartition("previouslyReplicatedTopic", 8), offsetToSeek); + verify(mockConsumer, times(1)) + .seek(new TopicPartition("previouslyReplicatedTopic", 1), offsetToSeek); + verify(mockConsumer, times(1)) + .seek(new TopicPartition("previouslyReplicatedTopic1", 0), offsetToSeek); + + verifyNoMoreInteractions(mockConsumer); + } + @Test public void testCommitRecordWithNullMetadata() { // Create a consumer mock diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java index 0f3a189b05e31..3595eebe1d4ee 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java @@ -81,6 +81,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -874,6 +875,34 @@ public void testReplicateTargetDefault() throws Exception { }, 30000, "Topic configurations were not synced"); } + @Test + public void testReplicateFromLatest() throws Exception { + // populate topic with records that should not be replicated + String topic = "test-topic-1"; + produceMessages(primaryProducer, topic, NUM_PARTITIONS); + + // consume from the ends of topics when no committed offsets are found + mm2Props.put(PRIMARY_CLUSTER_ALIAS + ".consumer." + AUTO_OFFSET_RESET_CONFIG, "latest"); + // one way replication from primary to backup + mm2Props.put(BACKUP_CLUSTER_ALIAS + "->" + PRIMARY_CLUSTER_ALIAS + ".enabled", "false"); + mm2Config = new MirrorMakerConfig(mm2Props); + waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS); + + // produce some more messages to the topic, now that MM2 is running and replication should be taking place + produceMessages(primaryProducer, topic, NUM_PARTITIONS); + + String backupTopic = remoteTopicName(topic, PRIMARY_CLUSTER_ALIAS); + // wait for at least the expected number of records to be replicated to the backup cluster + backup.kafka().consume(NUM_PARTITIONS * NUM_RECORDS_PER_PARTITION, RECORD_TRANSFER_DURATION_MS, backupTopic); + // consume all records from backup cluster + ConsumerRecords replicatedRecords = backup.kafka().consumeAll(RECORD_TRANSFER_DURATION_MS, backupTopic); + // ensure that we only replicated the records produced after startup + replicatedRecords.partitions().forEach(topicPartition -> { + int replicatedCount = replicatedRecords.records(topicPartition).size(); + assertEquals(NUM_RECORDS_PER_PARTITION, replicatedCount); + }); + } + private TopicPartition remoteTopicPartition(TopicPartition tp, String alias) { return new TopicPartition(remoteTopicName(tp.topic(), alias), tp.partition()); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java index c1977de3df9a4..7068377a90085 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java @@ -16,20 +16,26 @@ */ package org.apache.kafka.connect.cli; +import com.fasterxml.jackson.core.exc.StreamReadException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DatabindException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.kafka.common.utils.Exit; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy; +import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.runtime.Connect; -import org.apache.kafka.connect.runtime.ConnectorConfig; import org.apache.kafka.connect.runtime.Herder; import org.apache.kafka.connect.runtime.Worker; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.rest.RestClient; import org.apache.kafka.connect.runtime.rest.RestServer; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; +import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest; import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; import org.apache.kafka.connect.runtime.standalone.StandaloneHerder; import org.apache.kafka.connect.storage.FileOffsetBackingStore; @@ -38,9 +44,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; import java.util.Collections; import java.util.Map; +import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG; + /** *

    * Command line utility that runs Kafka Connect as a standalone process. In this mode, work (connectors and tasks) is not @@ -61,23 +72,24 @@ public ConnectStandalone(String... args) { @Override protected String usage() { - return "ConnectStandalone worker.properties [connector1.properties connector2.properties ...]"; + return "ConnectStandalone worker.properties [connector1.properties connector2.json ...]"; } @Override protected void processExtraArgs(Herder herder, Connect connect, String[] extraArgs) { try { - for (final String connectorPropsFile : extraArgs) { - Map connectorProps = Utils.propsToStringMap(Utils.loadProps(connectorPropsFile)); + for (final String connectorConfigFile : extraArgs) { + CreateConnectorRequest createConnectorRequest = parseConnectorConfigurationFile(connectorConfigFile); FutureCallback> cb = new FutureCallback<>((error, info) -> { if (error != null) - log.error("Failed to create connector for {}", connectorPropsFile); + log.error("Failed to create connector for {}", connectorConfigFile); else log.info("Created connector {}", info.result().name()); }); herder.putConnectorConfig( - connectorProps.get(ConnectorConfig.NAME_CONFIG), - connectorProps, false, cb); + createConnectorRequest.name(), createConnectorRequest.config(), + createConnectorRequest.initialTargetState(), + false, cb); cb.get(); } } catch (Throwable t) { @@ -87,6 +99,64 @@ protected void processExtraArgs(Herder herder, Connect connect, String[] extraAr } } + /** + * Parse a connector configuration file into a {@link CreateConnectorRequest}. The file can have any one of the following formats (note that + * we attempt to parse the file in this order): + *

      + *
    1. A JSON file containing an Object with only String keys and values that represent the connector configuration.
      2. + *
    2. A JSON file containing an Object that can be parsed directly into a {@link CreateConnectorRequest}
      3. + *
    3. A valid Java Properties file (i.e. containing String key/value pairs representing the connector configuration)
      4. + *
    + *

    + * Visible for testing. + * + * @param filePath the path of the connector configuration file + * @return the parsed connector configuration in the form of a {@link CreateConnectorRequest} + */ + CreateConnectorRequest parseConnectorConfigurationFile(String filePath) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + + File connectorConfigurationFile = Paths.get(filePath).toFile(); + try { + Map connectorConfigs = objectMapper.readValue( + connectorConfigurationFile, + new TypeReference>() { }); + + if (!connectorConfigs.containsKey(NAME_CONFIG)) { + throw new ConnectException("Connector configuration at '" + filePath + "' is missing the mandatory '" + NAME_CONFIG + "' " + + "configuration"); + } + return new CreateConnectorRequest(connectorConfigs.get(NAME_CONFIG), connectorConfigs, null); + } catch (StreamReadException | DatabindException e) { + log.debug("Could not parse connector configuration file '{}' into a Map with String keys and values", filePath); + } + + try { + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + CreateConnectorRequest createConnectorRequest = objectMapper.readValue(connectorConfigurationFile, + new TypeReference() { }); + if (createConnectorRequest.config().containsKey(NAME_CONFIG)) { + if (!createConnectorRequest.config().get(NAME_CONFIG).equals(createConnectorRequest.name())) { + throw new ConnectException("Connector name configuration in 'config' doesn't match the one specified in 'name' at '" + filePath + + "'"); + } + } else { + createConnectorRequest.config().put(NAME_CONFIG, createConnectorRequest.name()); + } + return createConnectorRequest; + } catch (StreamReadException | DatabindException e) { + log.debug("Could not parse connector configuration file '{}' into an object of type {}", + filePath, CreateConnectorRequest.class.getSimpleName()); + } + + Map connectorConfigs = Utils.propsToStringMap(Utils.loadProps(filePath)); + if (!connectorConfigs.containsKey(NAME_CONFIG)) { + throw new ConnectException("Connector configuration at '" + filePath + "' is missing the mandatory '" + NAME_CONFIG + "' " + + "configuration"); + } + return new CreateConnectorRequest(connectorConfigs.get(NAME_CONFIG), connectorConfigs, null); + } + @Override protected Herder createHerder(StandaloneConfig config, String workerId, Plugins plugins, ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy, diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java index 566a5c4c096ae..a8a6e7858d802 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/Herder.java @@ -108,6 +108,19 @@ public interface Herder { */ void putConnectorConfig(String connName, Map config, boolean allowReplace, Callback> callback); + /** + * Set the configuration for a connector, along with a target state optionally. This supports creation and updating. + * @param connName name of the connector + * @param config the connector's configuration + * @param targetState the desired target state for the connector; may be {@code null} if no target state change is desired. Note that the default + * target state is {@link TargetState#STARTED} if no target state exists previously + * @param allowReplace if true, allow overwriting previous configs; if false, throw {@link AlreadyExistsException} + * if a connector with the same name already exists + * @param callback callback to invoke when the configuration has been written + */ + void putConnectorConfig(String connName, Map config, TargetState targetState, boolean allowReplace, + Callback> callback); + /** * Delete a connector and its configuration. * @param connName name of the connector diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java index 48b5fad3423ce..92f513a9f7453 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java @@ -51,14 +51,13 @@ import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.TaskStatus; import org.apache.kafka.connect.runtime.Worker; -import org.apache.kafka.connect.runtime.rest.entities.ConnectorOffsets; -import org.apache.kafka.connect.runtime.rest.entities.Message; -import org.apache.kafka.connect.storage.PrivilegedWriteException; import org.apache.kafka.connect.runtime.rest.InternalRequestSignature; import org.apache.kafka.connect.runtime.rest.RestClient; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; +import org.apache.kafka.connect.runtime.rest.entities.ConnectorOffsets; import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; import org.apache.kafka.connect.runtime.rest.entities.ConnectorType; +import org.apache.kafka.connect.runtime.rest.entities.Message; import org.apache.kafka.connect.runtime.rest.entities.TaskInfo; import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; @@ -69,6 +68,7 @@ import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.storage.ClusterConfigState; import org.apache.kafka.connect.storage.ConfigBackingStore; +import org.apache.kafka.connect.storage.PrivilegedWriteException; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectUtils; @@ -100,12 +100,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -147,7 +146,6 @@ *

    */ public class DistributedHerder extends AbstractHerder implements Runnable { - private static final AtomicInteger CONNECT_CLIENT_ID_SEQUENCE = new AtomicInteger(1); private final Logger log; private static final long FORWARD_REQUEST_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(10); @@ -296,7 +294,7 @@ public DistributedHerder(DistributedConfig config, this.uponShutdown = Arrays.asList(uponShutdown); String clientIdConfig = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); - String clientId = clientIdConfig.length() <= 0 ? "connect-" + CONNECT_CLIENT_ID_SEQUENCE.getAndIncrement() : clientIdConfig; + String clientId = clientIdConfig.isEmpty() ? "connect-" + workerId : clientIdConfig; // Thread factory uses String.format and '%' is handled as a placeholder // need to escape if the client.id contains an actual % character String escapedClientIdForThreadNameFormat = clientId.replace("%", "%%"); @@ -1051,6 +1049,12 @@ private boolean connectorUsesSeparateOffsetsTopicClients(org.apache.kafka.connec @Override public void putConnectorConfig(final String connName, final Map config, final boolean allowReplace, final Callback> callback) { + putConnectorConfig(connName, config, null, allowReplace, callback); + } + + @Override + public void putConnectorConfig(final String connName, final Map config, final TargetState targetState, + final boolean allowReplace, final Callback> callback) { log.trace("Submitting connector config write request {}", connName); addRequest( () -> { @@ -1081,7 +1085,7 @@ public void putConnectorConfig(final String connName, final Map } log.trace("Submitting connector config {} {} {}", connName, allowReplace, configState.connectors()); - writeToConfigTopicAsLeader(() -> configBackingStore.putConnectorConfig(connName, config)); + writeToConfigTopicAsLeader(() -> configBackingStore.putConnectorConfig(connName, config, targetState)); // Note that we use the updated connector config despite the fact that we don't have an updated // snapshot yet. The existing task info should still be accurate. @@ -1638,7 +1642,7 @@ private String leaderUrl() { * exclusively by the leader. For example, {@link ConfigBackingStore#putTargetState(String, TargetState)} does not require this * method, as it can be invoked by any worker in the cluster. * @param write the action that writes to the config topic, such as {@link ConfigBackingStore#putSessionKey(SessionKey)} or - * {@link ConfigBackingStore#putConnectorConfig(String, Map)}. + * {@link ConfigBackingStore#putConnectorConfig(String, Map, TargetState)}. */ private void writeToConfigTopicAsLeader(Runnable write) { try { diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequest.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequest.java index 1c52d8db99706..da8e235e42411 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequest.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequest.java @@ -18,18 +18,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kafka.connect.runtime.TargetState; +import java.util.Locale; import java.util.Map; import java.util.Objects; public class CreateConnectorRequest { private final String name; private final Map config; + private final InitialState initialState; @JsonCreator - public CreateConnectorRequest(@JsonProperty("name") String name, @JsonProperty("config") Map config) { + public CreateConnectorRequest(@JsonProperty("name") String name, @JsonProperty("config") Map config, + @JsonProperty("initial_state") InitialState initialState) { this.name = name; this.config = config; + this.initialState = initialState; } @JsonProperty @@ -42,17 +47,55 @@ public Map config() { return config; } + @JsonProperty + public InitialState initialState() { + return initialState; + } + + public TargetState initialTargetState() { + if (initialState != null) { + return initialState.toTargetState(); + } else { + return null; + } + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CreateConnectorRequest that = (CreateConnectorRequest) o; return Objects.equals(name, that.name) && - Objects.equals(config, that.config); + Objects.equals(config, that.config) && + Objects.equals(initialState, that.initialState); } @Override public int hashCode() { - return Objects.hash(name, config); + return Objects.hash(name, config, initialState); + } + + public enum InitialState { + RUNNING, + PAUSED, + STOPPED; + + @JsonCreator + public static InitialState forValue(String value) { + return InitialState.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public TargetState toTargetState() { + switch (this) { + case RUNNING: + return TargetState.STARTED; + case PAUSED: + return TargetState.PAUSED; + case STOPPED: + return TargetState.STOPPED; + default: + throw new IllegalArgumentException("Unknown initial state: " + this); + } + } } } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java index 4878b8df9e1fb..75e510ef9ad5a 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java @@ -145,7 +145,7 @@ public Response createConnector(final @Parameter(hidden = true) @QueryParam("for checkAndPutConnectorConfigName(name, configs); FutureCallback> cb = new FutureCallback<>(); - herder.putConnectorConfig(name, configs, false, cb); + herder.putConnectorConfig(name, configs, createRequest.initialTargetState(), false, cb); Herder.Created info = requestHandler.completeOrForwardRequest(cb, "/connectors", "POST", headers, createRequest, new TypeReference() { }, new CreatedConnectorInfoTranslator(), forward); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java index 0da89b2f668ce..40e19da19c79c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java @@ -183,6 +183,12 @@ public synchronized void putConnectorConfig(String connName, final Map config, boolean allowReplace, final Callback> callback) { + putConnectorConfig(connName, config, null, allowReplace, callback); + } + + @Override + public void putConnectorConfig(final String connName, final Map config, final TargetState targetState, + final boolean allowReplace, final Callback> callback) { try { validateConnectorConfig(config, (error, configInfos) -> { if (error != null) { @@ -191,7 +197,7 @@ public synchronized void putConnectorConfig(String connName, } requestExecutorService.submit( - () -> putConnectorConfig(connName, config, allowReplace, callback, configInfos) + () -> putConnectorConfig(connName, config, targetState, allowReplace, callback, configInfos) ); }); } catch (Throwable t) { @@ -201,6 +207,7 @@ public synchronized void putConnectorConfig(String connName, private synchronized void putConnectorConfig(String connName, final Map config, + TargetState targetState, boolean allowReplace, final Callback> callback, ConfigInfos configInfos) { @@ -221,7 +228,7 @@ private synchronized void putConnectorConfig(String connName, created = true; } - configBackingStore.putConnectorConfig(connName, config); + configBackingStore.putConnectorConfig(connName, config, targetState); startConnector(connName, (error, result) -> { if (error != null) { diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java index c869c545f809b..9f12ab0d3cf39 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/ConfigBackingStore.java @@ -56,8 +56,10 @@ public interface ConfigBackingStore { * Update the configuration for a connector. * @param connector name of the connector * @param properties the connector configuration + * @param targetState the desired target state for the connector; may be {@code null} if no target state change is desired. Note that the default + * target state is {@link TargetState#STARTED} if no target state exists previously */ - void putConnectorConfig(String connector, Map properties); + void putConnectorConfig(String connector, Map properties, TargetState targetState); /** * Remove configuration for a connector diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java index a95c82499429c..35d43ea3ccaed 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/KafkaConfigBackingStore.java @@ -497,26 +497,34 @@ public boolean contains(String connector) { } /** - * Write this connector configuration to persistent storage and wait until it has been acknowledged and read back by - * tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling + * Write this connector configuration (and optionally a target state) to persistent storage and wait until it has been acknowledged and read + * back by tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling * this method if the worker is configured to use a fencable producer for writes to the config topic. * * @param connector name of the connector to write data for * @param properties the configuration to write + * @param targetState the desired target state for the connector; may be {@code null} if no target state change is desired. Note that the default + * target state is {@link TargetState#STARTED} if no target state exists previously * @throws IllegalStateException if {@link #claimWritePrivileges()} is required, but was not successfully invoked before * this method was called * @throws PrivilegedWriteException if the worker is configured to use a fencable producer for writes to the config topic * and the write fails */ @Override - public void putConnectorConfig(String connector, Map properties) { + public void putConnectorConfig(String connector, Map properties, TargetState targetState) { log.debug("Writing connector configuration for connector '{}'", connector); Struct connectConfig = new Struct(CONNECTOR_CONFIGURATION_V0); connectConfig.put("properties", properties); byte[] serializedConfig = converter.fromConnectData(topic, CONNECTOR_CONFIGURATION_V0, connectConfig); try { Timer timer = time.timer(READ_WRITE_TOTAL_TIMEOUT_MS); - sendPrivileged(CONNECTOR_KEY(connector), serializedConfig, timer); + List keyValues = new ArrayList<>(); + if (targetState != null) { + log.debug("Writing target state {} for connector {}", targetState, connector); + keyValues.add(new ProducerKeyValue(TARGET_STATE_KEY(connector), serializeTargetState(targetState))); + } + keyValues.add(new ProducerKeyValue(CONNECTOR_KEY(connector), serializedConfig)); + sendPrivileged(keyValues, timer); configLog.readToEnd().get(timer.remainingMs(), TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to write connector configuration to Kafka: ", e); @@ -647,20 +655,24 @@ public void refresh(long timeout, TimeUnit unit) throws TimeoutException { */ @Override public void putTargetState(String connector, TargetState state) { - Struct connectTargetState = new Struct(TARGET_STATE_V1); - // Older workers don't support the STOPPED state; fall back on PAUSED - connectTargetState.put("state", state == STOPPED ? PAUSED.name() : state.name()); - connectTargetState.put("state.v2", state.name()); - byte[] serializedTargetState = converter.fromConnectData(topic, TARGET_STATE_V1, connectTargetState); log.debug("Writing target state {} for connector {}", state, connector); try { - configLog.sendWithReceipt(TARGET_STATE_KEY(connector), serializedTargetState).get(READ_WRITE_TOTAL_TIMEOUT_MS, TimeUnit.MILLISECONDS); + configLog.sendWithReceipt(TARGET_STATE_KEY(connector), serializeTargetState(state)) + .get(READ_WRITE_TOTAL_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to write target state to Kafka", e); throw new ConnectException("Error writing target state to Kafka", e); } } + private byte[] serializeTargetState(TargetState state) { + Struct connectTargetState = new Struct(TARGET_STATE_V1); + // Older workers don't support the STOPPED state; fall back on PAUSED + connectTargetState.put("state", state == STOPPED ? PAUSED.name() : state.name()); + connectTargetState.put("state.v2", state.name()); + return converter.fromConnectData(topic, TARGET_STATE_V1, connectTargetState); + } + /** * Write a task count record for a connector to persistent storage and wait until it has been acknowledged and read back by * tailing the Kafka log with a consumer. {@link #claimWritePrivileges()} must be successfully invoked before calling this method @@ -985,7 +997,9 @@ private void processTargetStateRecord(String connectorName, SchemaAndValue value // Note that we do not notify the update listener if the target state has been removed. // Instead we depend on the removal callback of the connector config itself to notify the worker. - if (started && !removed && stateChanged) + // We also don't notify the update listener if the connector doesn't exist yet - a scenario that can + // occur if an explicit initial target state is specified in the connector creation request. + if (started && !removed && stateChanged && connectorConfigs.containsKey(connectorName)) updateListener.onConnectorTargetStateChange(connectorName); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java index 52c360c3b33ba..3b9ba966ca20a 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/MemoryConfigBackingStore.java @@ -92,12 +92,16 @@ public synchronized boolean contains(String connector) { } @Override - public synchronized void putConnectorConfig(String connector, Map properties) { + public synchronized void putConnectorConfig(String connector, Map properties, TargetState targetState) { ConnectorState state = connectors.get(connector); if (state == null) - connectors.put(connector, new ConnectorState(properties)); - else + connectors.put(connector, new ConnectorState(properties, targetState)); + else { state.connConfig = properties; + if (targetState != null) { + state.targetState = targetState; + } + } if (updateListener != null) updateListener.onConnectorConfigUpdate(connector); @@ -184,8 +188,13 @@ private static class ConnectorState { private Map connConfig; private Map> taskConfigs; - public ConnectorState(Map connConfig) { - this.targetState = TargetState.STARTED; + /** + * @param connConfig the connector's configuration + * @param targetState the connector's initial {@link TargetState}; may be {@code null} in which case the default initial target state + * {@link TargetState#STARTED} will be used + */ + public ConnectorState(Map connConfig, TargetState targetState) { + this.targetState = targetState == null ? TargetState.STARTED : targetState; this.connConfig = connConfig; this.taskConfigs = new HashMap<>(); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/cli/ConnectStandaloneTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/cli/ConnectStandaloneTest.java new file mode 100644 index 0000000000000..9a762f1a394f3 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/cli/ConnectStandaloneTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.cli; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest; +import org.apache.kafka.test.TestUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.kafka.connect.runtime.ConnectorConfig.NAME_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ConnectStandaloneTest { + + private static final String CONNECTOR_NAME = "test-connector"; + private static final Map CONNECTOR_CONFIG = new HashMap<>(); + static { + CONNECTOR_CONFIG.put(NAME_CONFIG, CONNECTOR_NAME); + CONNECTOR_CONFIG.put("key1", "val1"); + CONNECTOR_CONFIG.put("key2", "val2"); + } + + private final ConnectStandalone connectStandalone = new ConnectStandalone(); + private File connectorConfigurationFile; + + @Before + public void setUp() throws IOException { + connectorConfigurationFile = TestUtils.tempFile(); + } + + @Test + public void testParseJavaPropertiesFile() throws Exception { + Properties properties = new Properties(); + CONNECTOR_CONFIG.forEach(properties::setProperty); + + try (FileWriter writer = new FileWriter(connectorConfigurationFile)) { + properties.store(writer, null); + } + + CreateConnectorRequest request = connectStandalone.parseConnectorConfigurationFile(connectorConfigurationFile.getAbsolutePath()); + assertEquals(CONNECTOR_NAME, request.name()); + assertEquals(CONNECTOR_CONFIG, request.config()); + assertNull(request.initialState()); + } + + @Test + public void testParseJsonFileWithConnectorConfiguration() throws Exception { + try (FileWriter writer = new FileWriter(connectorConfigurationFile)) { + writer.write(new ObjectMapper().writeValueAsString(CONNECTOR_CONFIG)); + } + + CreateConnectorRequest request = connectStandalone.parseConnectorConfigurationFile(connectorConfigurationFile.getAbsolutePath()); + assertEquals(CONNECTOR_NAME, request.name()); + assertEquals(CONNECTOR_CONFIG, request.config()); + assertNull(request.initialState()); + } + + @Test + public void testParseJsonFileWithCreateConnectorRequest() throws Exception { + CreateConnectorRequest requestToWrite = new CreateConnectorRequest( + CONNECTOR_NAME, + CONNECTOR_CONFIG, + CreateConnectorRequest.InitialState.STOPPED + ); + + try (FileWriter writer = new FileWriter(connectorConfigurationFile)) { + writer.write(new ObjectMapper().writeValueAsString(requestToWrite)); + } + + CreateConnectorRequest parsedRequest = connectStandalone.parseConnectorConfigurationFile(connectorConfigurationFile.getAbsolutePath()); + assertEquals(requestToWrite, parsedRequest); + } + + @Test + public void testParseJsonFileWithCreateConnectorRequestWithoutInitialState() throws Exception { + Map requestToWrite = new HashMap<>(); + requestToWrite.put("name", CONNECTOR_NAME); + requestToWrite.put("config", CONNECTOR_CONFIG); + + try (FileWriter writer = new FileWriter(connectorConfigurationFile)) { + writer.write(new ObjectMapper().writeValueAsString(requestToWrite)); + } + + CreateConnectorRequest parsedRequest = connectStandalone.parseConnectorConfigurationFile(connectorConfigurationFile.getAbsolutePath()); + CreateConnectorRequest expectedRequest = new CreateConnectorRequest(CONNECTOR_NAME, CONNECTOR_CONFIG, null); + assertEquals(expectedRequest, parsedRequest); + } + + @Test + public void testParseJsonFileWithCreateConnectorRequestWithUnknownField() throws Exception { + Map requestToWrite = new HashMap<>(); + requestToWrite.put("name", CONNECTOR_NAME); + requestToWrite.put("config", CONNECTOR_CONFIG); + requestToWrite.put("unknown-field", "random-value"); + + try (FileWriter writer = new FileWriter(connectorConfigurationFile)) { + writer.write(new ObjectMapper().writeValueAsString(requestToWrite)); + } + + CreateConnectorRequest parsedRequest = connectStandalone.parseConnectorConfigurationFile(connectorConfigurationFile.getAbsolutePath()); + CreateConnectorRequest expectedRequest = new CreateConnectorRequest(CONNECTOR_NAME, CONNECTOR_CONFIG, null); + assertEquals(expectedRequest, parsedRequest); + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java index cd9b9c0517190..8a271603eefc2 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/ConnectWorkerIntegrationTest.java @@ -16,8 +16,11 @@ */ package org.apache.kafka.connect.integration; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.LogCaptureAppender; +import org.apache.kafka.connect.errors.ConnectException; import org.apache.kafka.connect.runtime.distributed.DistributedConfig; +import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest; import org.apache.kafka.connect.runtime.rest.resources.ConnectorsResource; import org.apache.kafka.connect.storage.StringConverter; import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; @@ -49,14 +52,15 @@ import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; import static org.apache.kafka.connect.runtime.WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG; import static org.apache.kafka.connect.runtime.WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.connect.util.clusters.ConnectAssertions.CONNECTOR_SETUP_DURATION_MS; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; -import static org.apache.kafka.connect.util.clusters.ConnectAssertions.CONNECTOR_SETUP_DURATION_MS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -73,7 +77,7 @@ public class ConnectWorkerIntegrationTest { private static final int NUM_WORKERS = 3; private static final int NUM_TASKS = 4; private static final int MESSAGES_PER_POLL = 10; - private static final String CONNECTOR_NAME = "simple-source"; + private static final String CONNECTOR_NAME = "simple-connector"; private static final String TOPIC_NAME = "test-topic"; private EmbeddedConnectCluster.Builder connectBuilder; @@ -529,8 +533,10 @@ public void testTasksConfigDeprecation() throws Exception { // start the clusters connect.start(); - connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS, - "Initial group of workers did not start in time."); + connect.assertions().assertAtLeastNumWorkersAreUp( + NUM_WORKERS, + "Initial group of workers did not start in time." + ); connect.configureConnector(CONNECTOR_NAME, defaultSourceConnectorProps(TOPIC_NAME)); connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( @@ -546,10 +552,227 @@ public void testTasksConfigDeprecation() throws Exception { assertEquals(Level.WARN.toString(), logEvents.get(0).getLevel()); assertThat(logEvents.get(0).getMessage(), containsString("deprecated")); } + + } + + @Test + public void testCreateConnectorWithPausedInitialState() throws Exception { + connect = connectBuilder.build(); + // start the clusters + connect.start(); + + connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS, + "Initial group of workers did not start in time."); + + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + defaultSourceConnectorProps(TOPIC_NAME), + CreateConnectorRequest.InitialState.PAUSED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is PAUSED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorAndExactlyNumTasksArePaused( + CONNECTOR_NAME, + 0, + "Connector was not created in a paused state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that a connector created in the PAUSED state can be resumed successfully + connect.resumeConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( + CONNECTOR_NAME, + NUM_TASKS, + "Connector or tasks did not start running healthily in time" + ); + } + + @Test + public void testCreateSourceConnectorWithStoppedInitialStateAndModifyOffsets() throws Exception { + connect = connectBuilder.build(); + // start the clusters + connect.start(); + + connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS, + "Initial group of workers did not start in time."); + + Map props = defaultSourceConnectorProps(TOPIC_NAME); + + // Configure the connector to produce a maximum of 10 messages + props.put("max.messages", "10"); + props.put(TASKS_MAX_CONFIG, "1"); + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + props, + CreateConnectorRequest.InitialState.STOPPED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is STOPPED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorIsStopped( + CONNECTOR_NAME, + "Connector was not created in a stopped state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that the offsets can be modified for a source connector created in the STOPPED state + + // Alter the offsets so that only 5 messages are produced + connect.alterSourceConnectorOffset( + CONNECTOR_NAME, + Collections.singletonMap("task.id", CONNECTOR_NAME + "-0"), + Collections.singletonMap("saved", 5L) + ); + + // Verify that a connector created in the STOPPED state can be resumed successfully + connect.resumeConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( + CONNECTOR_NAME, + 1, + "Connector or tasks did not start running healthily in time" + ); + + // Verify that only 5 messages were produced. We verify this by consuming all the messages from the topic after we've already ensured that at + // least 5 messages can be consumed. + long timeoutMs = TimeUnit.SECONDS.toMillis(10); + connect.kafka().consume(5, timeoutMs, TOPIC_NAME); + assertEquals(5, connect.kafka().consumeAll(timeoutMs, TOPIC_NAME).count()); + } + + @Test + public void testCreateSinkConnectorWithStoppedInitialStateAndModifyOffsets() throws Exception { + connect = connectBuilder.build(); + // start the clusters + connect.start(); + + connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS, + "Initial group of workers did not start in time."); + + // Create topic and produce 10 messages + connect.kafka().createTopic(TOPIC_NAME); + for (int i = 0; i < 10; i++) { + connect.kafka().produce(TOPIC_NAME, "Message " + i); + } + + Map props = defaultSinkConnectorProps(TOPIC_NAME); + props.put(TASKS_MAX_CONFIG, "1"); + + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + props, + CreateConnectorRequest.InitialState.STOPPED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is STOPPED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorIsStopped( + CONNECTOR_NAME, + "Connector was not created in a stopped state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that the offsets can be modified for a sink connector created in the STOPPED state + + // Alter the offsets so that the first 5 messages in the topic are skipped + connect.alterSinkConnectorOffset(CONNECTOR_NAME, new TopicPartition(TOPIC_NAME, 0), 5L); + + // This will cause the connector task to fail if it encounters a record with offset < 5 + TaskHandle taskHandle = RuntimeHandles.get().connectorHandle(CONNECTOR_NAME).taskHandle(CONNECTOR_NAME + "-0", + sinkRecord -> { + if (sinkRecord.kafkaOffset() < 5L) { + throw new ConnectException("Unexpected record encountered: " + sinkRecord); + } + }); + + // We produced 10 records and altered the connector offsets to skip over the first 5, so we expect 5 records to be consumed + taskHandle.expectedRecords(5); + + // Verify that a connector created in the STOPPED state can be resumed successfully + connect.resumeConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( + CONNECTOR_NAME, + 1, + "Connector or tasks did not start running healthily in time" + ); + + taskHandle.awaitRecords(TimeUnit.SECONDS.toMillis(10)); + + // Confirm that the task is still running (i.e. it didn't fail due to encountering any records with offset < 5) + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( + CONNECTOR_NAME, + 1, + "Connector or tasks did not start running healthily in time" + ); + } + + @Test + public void testDeleteConnectorCreatedWithPausedOrStoppedInitialState() throws Exception { + connect = connectBuilder.build(); + // start the clusters + connect.start(); + + connect.assertions().assertAtLeastNumWorkersAreUp(NUM_WORKERS, + "Initial group of workers did not start in time."); + + // Create a connector with PAUSED initial state + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + defaultSourceConnectorProps(TOPIC_NAME), + CreateConnectorRequest.InitialState.PAUSED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is PAUSED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorAndExactlyNumTasksArePaused( + CONNECTOR_NAME, + 0, + "Connector was not created in a paused state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that a connector created in the PAUSED state can be deleted successfully + connect.deleteConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorDoesNotExist(CONNECTOR_NAME, "Connector wasn't deleted in time"); + + + // Create a connector with STOPPED initial state + createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + defaultSourceConnectorProps(TOPIC_NAME), + CreateConnectorRequest.InitialState.STOPPED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is STOPPED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorIsStopped( + CONNECTOR_NAME, + "Connector was not created in a stopped state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that a connector created in the STOPPED state can be deleted successfully + connect.deleteConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorDoesNotExist(CONNECTOR_NAME, "Connector wasn't deleted in time"); + } + + private Map defaultSinkConnectorProps(String topics) { + // setup props for the sink connector + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSinkConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); + props.put(TOPICS_CONFIG, topics); + + return props; } private Map defaultSourceConnectorProps(String topic) { - // setup up props for the source connector + // setup props for the source connector Map props = new HashMap<>(); props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestForwardingIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestForwardingIntegrationTest.java index 7c9e2f2c51e35..0bbad10a57d91 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestForwardingIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RestForwardingIntegrationTest.java @@ -70,6 +70,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; @@ -187,7 +188,7 @@ public void testRestForwardToLeader(boolean dualListener, boolean followerSsl, b followerCallbackCaptor.getValue().onCompletion(forwardException, null); return null; }).when(followerHerder) - .putConnectorConfig(any(), any(), anyBoolean(), followerCallbackCaptor.capture()); + .putConnectorConfig(any(), any(), isNull(), anyBoolean(), followerCallbackCaptor.capture()); // Leader will reply ConnectorInfo connectorInfo = new ConnectorInfo("blah", Collections.emptyMap(), Collections.emptyList(), ConnectorType.SOURCE); @@ -197,7 +198,7 @@ public void testRestForwardToLeader(boolean dualListener, boolean followerSsl, b leaderCallbackCaptor.getValue().onCompletion(null, leaderAnswer); return null; }).when(leaderHerder) - .putConnectorConfig(any(), any(), anyBoolean(), leaderCallbackCaptor.capture()); + .putConnectorConfig(any(), any(), isNull(), anyBoolean(), leaderCallbackCaptor.capture()); // Client makes request to the follower URI followerUrl = followerServer.advertisedUrl(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StandaloneWorkerIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StandaloneWorkerIntegrationTest.java index ea938f9a4f68f..e47fe4304d366 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StandaloneWorkerIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/StandaloneWorkerIntegrationTest.java @@ -17,7 +17,9 @@ package org.apache.kafka.connect.integration; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest; import org.apache.kafka.connect.runtime.rest.entities.LoggerLevel; +import org.apache.kafka.connect.storage.StringConverter; import org.apache.kafka.connect.util.clusters.EmbeddedConnectStandalone; import org.apache.kafka.test.IntegrationTest; import org.junit.After; @@ -26,12 +28,21 @@ import org.junit.experimental.categories.Category; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.kafka.connect.integration.MonitorableSourceConnector.TOPIC_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.DEFAULT_TOPIC_CREATION_PREFIX; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.PARTITIONS_CONFIG; +import static org.apache.kafka.connect.runtime.TopicCreationConfig.REPLICATION_FACTOR_CONFIG; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -41,6 +52,10 @@ @Category(IntegrationTest.class) public class StandaloneWorkerIntegrationTest { + private static final String CONNECTOR_NAME = "test-connector"; + private static final int NUM_TASKS = 4; + private static final String TOPIC_NAME = "test-topic"; + private EmbeddedConnectStandalone connect; @Before @@ -202,4 +217,42 @@ private static String level(Map.Entry entry) { return entry.getValue().level(); } + @Test + public void testCreateConnectorWithStoppedInitialState() throws Exception { + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest( + CONNECTOR_NAME, + defaultSourceConnectorProps(TOPIC_NAME), + CreateConnectorRequest.InitialState.STOPPED + ); + connect.configureConnector(createConnectorRequest); + + // Verify that the connector's status is STOPPED and also that no tasks were spawned for the connector + connect.assertions().assertConnectorIsStopped( + CONNECTOR_NAME, + "Connector was not created in a stopped state" + ); + assertEquals(Collections.emptyList(), connect.connectorInfo(CONNECTOR_NAME).tasks()); + assertEquals(Collections.emptyList(), connect.taskConfigs(CONNECTOR_NAME)); + + // Verify that a connector created in the STOPPED state can be resumed successfully + connect.resumeConnector(CONNECTOR_NAME); + connect.assertions().assertConnectorAndExactlyNumTasksAreRunning( + CONNECTOR_NAME, + NUM_TASKS, + "Connector or tasks did not start running healthily in time" + ); + } + + private Map defaultSourceConnectorProps(String topic) { + // setup props for the source connector + Map props = new HashMap<>(); + props.put(CONNECTOR_CLASS_CONFIG, MonitorableSourceConnector.class.getSimpleName()); + props.put(TASKS_MAX_CONFIG, String.valueOf(NUM_TASKS)); + props.put(TOPIC_CONFIG, topic); + props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + REPLICATION_FACTOR_CONFIG, String.valueOf(1)); + props.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(1)); + return props; + } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java index 4c08ea2a44101..5a22e6c10f0d7 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/DistributedHerderTest.java @@ -690,7 +690,7 @@ public void testCreateConnector() throws Exception { }).when(herder).validateConnectorConfig(eq(CONN2_CONFIG), validateCallback.capture()); // CONN2 is new, should succeed - doNothing().when(configBackingStore).putConnectorConfig(CONN2, CONN2_CONFIG); + doNothing().when(configBackingStore).putConnectorConfig(eq(CONN2), eq(CONN2_CONFIG), isNull()); // This will occur just before/during the second tick doNothing().when(member).ensureActive(); @@ -713,6 +713,51 @@ public void testCreateConnector() throws Exception { verifyNoMoreInteractions(worker, member, configBackingStore, statusBackingStore, putConnectorCallback); } + @Test + public void testCreateConnectorWithInitialState() throws Exception { + when(member.memberId()).thenReturn("leader"); + when(member.currentProtocolVersion()).thenReturn(CONNECT_PROTOCOL_V0); + expectRebalance(1, Collections.emptyList(), Collections.emptyList(), true); + expectConfigRefreshAndSnapshot(SNAPSHOT); + + when(statusBackingStore.connectors()).thenReturn(Collections.emptySet()); + doNothing().when(member).poll(anyLong()); + + // Initial rebalance where this member becomes the leader + herder.tick(); + + // mock the actual validation since its asynchronous nature is difficult to test and should + // be covered sufficiently by the unit tests for the AbstractHerder class + ArgumentCaptor> validateCallback = ArgumentCaptor.forClass(Callback.class); + doAnswer(invocation -> { + validateCallback.getValue().onCompletion(null, CONN2_CONFIG_INFOS); + return null; + }).when(herder).validateConnectorConfig(eq(CONN2_CONFIG), validateCallback.capture()); + + // CONN2 is new, should succeed + doNothing().when(configBackingStore).putConnectorConfig(eq(CONN2), eq(CONN2_CONFIG), eq(TargetState.STOPPED)); + + // This will occur just before/during the second tick + doNothing().when(member).ensureActive(); + + // No immediate action besides this -- change will be picked up via the config log + + herder.putConnectorConfig(CONN2, CONN2_CONFIG, TargetState.STOPPED, false, putConnectorCallback); + // This tick runs the initial herder request, which issues an asynchronous request for + // connector validation + herder.tick(); + + // Once that validation is complete, another request is added to the herder request queue + // for actually performing the config write; this tick is for that request + herder.tick(); + time.sleep(1000L); + assertStatistics(3, 1, 100, 1000L); + + ConnectorInfo info = new ConnectorInfo(CONN2, CONN2_CONFIG, Collections.emptyList(), ConnectorType.SOURCE); + verify(putConnectorCallback).onCompletion(isNull(), eq(new Herder.Created<>(true, info))); + verifyNoMoreInteractions(worker, member, configBackingStore, statusBackingStore, putConnectorCallback); + } + @Test public void testCreateConnectorConfigBackingStoreError() { when(member.memberId()).thenReturn("leader"); @@ -735,7 +780,7 @@ public void testCreateConnectorConfigBackingStoreError() { }).when(herder).validateConnectorConfig(eq(CONN2_CONFIG), validateCallback.capture()); doThrow(new ConnectException("Error writing connector configuration to Kafka")) - .when(configBackingStore).putConnectorConfig(CONN2, CONN2_CONFIG); + .when(configBackingStore).putConnectorConfig(eq(CONN2), eq(CONN2_CONFIG), isNull()); // This will occur just before/during the second tick doNothing().when(member).ensureActive(); @@ -2184,7 +2229,7 @@ public void testPutConnectorConfig() throws Exception { // Simulate response to writing config + waiting until end of log to be read configUpdateListener.onConnectorConfigUpdate(CONN1); return null; - }).when(configBackingStore).putConnectorConfig(eq(CONN1), eq(CONN1_CONFIG_UPDATED)); + }).when(configBackingStore).putConnectorConfig(eq(CONN1), eq(CONN1_CONFIG_UPDATED), isNull()); // As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart // connector without rebalance diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequestTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequestTest.java new file mode 100644 index 0000000000000..1d32479f82c22 --- /dev/null +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/entities/CreateConnectorRequestTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.connect.runtime.rest.entities; + +import org.apache.kafka.connect.runtime.TargetState; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class CreateConnectorRequestTest { + + @Test + public void testToTargetState() { + assertEquals(TargetState.STARTED, CreateConnectorRequest.InitialState.RUNNING.toTargetState()); + assertEquals(TargetState.PAUSED, CreateConnectorRequest.InitialState.PAUSED.toTargetState()); + assertEquals(TargetState.STOPPED, CreateConnectorRequest.InitialState.STOPPED.toTargetState()); + + CreateConnectorRequest createConnectorRequest = new CreateConnectorRequest("test-name", Collections.emptyMap(), null); + assertNull(createConnectorRequest.initialTargetState()); + } + + @Test + public void testForValue() { + assertEquals(CreateConnectorRequest.InitialState.RUNNING, CreateConnectorRequest.InitialState.forValue("running")); + assertEquals(CreateConnectorRequest.InitialState.RUNNING, CreateConnectorRequest.InitialState.forValue("Running")); + assertEquals(CreateConnectorRequest.InitialState.RUNNING, CreateConnectorRequest.InitialState.forValue("RUNNING")); + + assertEquals(CreateConnectorRequest.InitialState.PAUSED, CreateConnectorRequest.InitialState.forValue("paused")); + assertEquals(CreateConnectorRequest.InitialState.PAUSED, CreateConnectorRequest.InitialState.forValue("Paused")); + assertEquals(CreateConnectorRequest.InitialState.PAUSED, CreateConnectorRequest.InitialState.forValue("PAUSED")); + + assertEquals(CreateConnectorRequest.InitialState.STOPPED, CreateConnectorRequest.InitialState.forValue("stopped")); + assertEquals(CreateConnectorRequest.InitialState.STOPPED, CreateConnectorRequest.InitialState.forValue("Stopped")); + assertEquals(CreateConnectorRequest.InitialState.STOPPED, CreateConnectorRequest.InitialState.forValue("STOPPED")); + } +} diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java index 0183e251600fb..aed081cf4d6f4 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResourceTest.java @@ -23,6 +23,7 @@ import org.apache.kafka.connect.runtime.ConnectorConfig; import org.apache.kafka.connect.runtime.Herder; import org.apache.kafka.connect.runtime.RestartRequest; +import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.distributed.NotAssignedException; import org.apache.kafka.connect.runtime.distributed.NotLeaderException; import org.apache.kafka.connect.runtime.distributed.RebalanceNeededException; @@ -272,23 +273,64 @@ public void testExpandConnectorsWithConnectorNotFound() { @Test public void testCreateConnector() throws Throwable { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), isNull(), eq(false), cb.capture()); + + connectorsResource.createConnector(FORWARD, NULL_HEADERS, body); + } + + @Test + public void testCreateConnectorWithPausedInitialState() throws Throwable { + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), CreateConnectorRequest.InitialState.PAUSED); + + final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); + expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, + CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(TargetState.PAUSED), eq(false), cb.capture()); + + connectorsResource.createConnector(FORWARD, NULL_HEADERS, body); + } + + @Test + public void testCreateConnectorWithStoppedInitialState() throws Throwable { + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), CreateConnectorRequest.InitialState.STOPPED); + + final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); + expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, + CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(TargetState.STOPPED), eq(false), cb.capture()); + + connectorsResource.createConnector(FORWARD, NULL_HEADERS, body); + } + + @Test + public void testCreateConnectorWithRunningInitialState() throws Throwable { + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), CreateConnectorRequest.InitialState.RUNNING); + + final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); + expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, + CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(TargetState.STARTED), eq(false), cb.capture()); connectorsResource.createConnector(FORWARD, NULL_HEADERS, body); } @Test public void testCreateConnectorNotLeader() throws Throwable { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackNotLeaderException(cb).when(herder) - .putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(false), cb.capture()); + .putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), isNull(), eq(false), cb.capture()); when(restClient.httpRequest(eq(LEADER_URL + "connectors?forward=false"), eq("POST"), isNull(), eq(body), any())) .thenReturn(new RestClient.HttpResponse<>(201, new HashMap<>(), new ConnectorInfo(CONNECTOR_NAME, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES, ConnectorType.SOURCE))); @@ -297,11 +339,12 @@ public void testCreateConnectorNotLeader() throws Throwable { @Test public void testCreateConnectorWithHeaders() throws Throwable { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); HttpHeaders httpHeaders = mock(HttpHeaders.class); expectAndCallbackNotLeaderException(cb) - .when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(false), cb.capture()); + .when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), isNull(), eq(false), cb.capture()); when(restClient.httpRequest(eq(LEADER_URL + "connectors?forward=false"), eq("POST"), eq(httpHeaders), any(), any())) .thenReturn(new RestClient.HttpResponse<>(202, new HashMap<>(), null)); @@ -310,11 +353,12 @@ public void testCreateConnectorWithHeaders() throws Throwable { @Test public void testCreateConnectorExists() { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackException(cb, new AlreadyExistsException("already exists")) - .when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), eq(false), cb.capture()); + .when(herder).putConnectorConfig(eq(CONNECTOR_NAME), eq(body.config()), isNull(), eq(false), cb.capture()); assertThrows(AlreadyExistsException.class, () -> connectorsResource.createConnector(FORWARD, NULL_HEADERS, body)); } @@ -323,13 +367,13 @@ public void testCreateConnectorNameTrimWhitespaces() throws Throwable { // Clone CONNECTOR_CONFIG_WITHOUT_NAME Map, as createConnector changes it (puts the name in it) and this // will affect later tests Map inputConfig = getConnectorConfig(CONNECTOR_CONFIG_WITHOUT_NAME); - final CreateConnectorRequest bodyIn = new CreateConnectorRequest(CONNECTOR_NAME_PADDING_WHITESPACES, inputConfig); - final CreateConnectorRequest bodyOut = new CreateConnectorRequest(CONNECTOR_NAME, CONNECTOR_CONFIG); + final CreateConnectorRequest bodyIn = new CreateConnectorRequest(CONNECTOR_NAME_PADDING_WHITESPACES, inputConfig, null); + final CreateConnectorRequest bodyOut = new CreateConnectorRequest(CONNECTOR_NAME, CONNECTOR_CONFIG, null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(bodyOut.name(), bodyOut.config(), CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), isNull(), eq(false), cb.capture()); connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn); } @@ -339,13 +383,13 @@ public void testCreateConnectorNameAllWhitespaces() throws Throwable { // Clone CONNECTOR_CONFIG_WITHOUT_NAME Map, as createConnector changes it (puts the name in it) and this // will affect later tests Map inputConfig = getConnectorConfig(CONNECTOR_CONFIG_WITHOUT_NAME); - final CreateConnectorRequest bodyIn = new CreateConnectorRequest(CONNECTOR_NAME_ALL_WHITESPACES, inputConfig); - final CreateConnectorRequest bodyOut = new CreateConnectorRequest("", CONNECTOR_CONFIG_WITH_EMPTY_NAME); + final CreateConnectorRequest bodyIn = new CreateConnectorRequest(CONNECTOR_NAME_ALL_WHITESPACES, inputConfig, null); + final CreateConnectorRequest bodyOut = new CreateConnectorRequest("", CONNECTOR_CONFIG_WITH_EMPTY_NAME, null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(bodyOut.name(), bodyOut.config(), CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), isNull(), eq(false), cb.capture()); connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn); } @@ -355,13 +399,13 @@ public void testCreateConnectorNoName() throws Throwable { // Clone CONNECTOR_CONFIG_WITHOUT_NAME Map, as createConnector changes it (puts the name in it) and this // will affect later tests Map inputConfig = getConnectorConfig(CONNECTOR_CONFIG_WITHOUT_NAME); - final CreateConnectorRequest bodyIn = new CreateConnectorRequest(null, inputConfig); - final CreateConnectorRequest bodyOut = new CreateConnectorRequest("", CONNECTOR_CONFIG_WITH_EMPTY_NAME); + final CreateConnectorRequest bodyIn = new CreateConnectorRequest(null, inputConfig, null); + final CreateConnectorRequest bodyOut = new CreateConnectorRequest("", CONNECTOR_CONFIG_WITH_EMPTY_NAME, null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(bodyOut.name(), bodyOut.config(), CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(bodyOut.name()), eq(bodyOut.config()), isNull(), eq(false), cb.capture()); connectorsResource.createConnector(FORWARD, NULL_HEADERS, bodyIn); } @@ -476,12 +520,13 @@ public void testPutConnectorConfig() throws Throwable { @Test public void testCreateConnectorWithSpecialCharsInName() throws Throwable { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME_SPECIAL_CHARS, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME_SPECIAL_CHARS)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME_SPECIAL_CHARS, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME_SPECIAL_CHARS), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME_SPECIAL_CHARS, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME_SPECIAL_CHARS), eq(body.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME_SPECIAL_CHARS), eq(body.config()), isNull(), eq(false), cb.capture()); String rspLocation = connectorsResource.createConnector(FORWARD, NULL_HEADERS, body).getLocation().toString(); String decoded = new URI(rspLocation).getPath(); @@ -490,12 +535,13 @@ public void testCreateConnectorWithSpecialCharsInName() throws Throwable { @Test public void testCreateConnectorWithControlSequenceInName() throws Throwable { - CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME_CONTROL_SEQUENCES1, Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME_CONTROL_SEQUENCES1)); + CreateConnectorRequest body = new CreateConnectorRequest(CONNECTOR_NAME_CONTROL_SEQUENCES1, + Collections.singletonMap(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME_CONTROL_SEQUENCES1), null); final ArgumentCaptor>> cb = ArgumentCaptor.forClass(Callback.class); expectAndCallbackResult(cb, new Herder.Created<>(true, new ConnectorInfo(CONNECTOR_NAME_CONTROL_SEQUENCES1, CONNECTOR_CONFIG, CONNECTOR_TASK_NAMES, ConnectorType.SOURCE)) - ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME_CONTROL_SEQUENCES1), eq(body.config()), eq(false), cb.capture()); + ).when(herder).putConnectorConfig(eq(CONNECTOR_NAME_CONTROL_SEQUENCES1), eq(body.config()), isNull(), eq(false), cb.capture()); String rspLocation = connectorsResource.createConnector(FORWARD, NULL_HEADERS, body).getLocation().toString(); String decoded = new URI(rspLocation).getPath(); @@ -540,7 +586,7 @@ public void testPutConnectorConfigNameMismatch() { public void testCreateConnectorConfigNameMismatch() { Map connConfig = new HashMap<>(); connConfig.put(ConnectorConfig.NAME_CONFIG, "mismatched-name"); - CreateConnectorRequest request = new CreateConnectorRequest(CONNECTOR_NAME, connConfig); + CreateConnectorRequest request = new CreateConnectorRequest(CONNECTOR_NAME, connConfig, null); assertThrows(BadRequestException.class, () -> connectorsResource.createConnector(FORWARD, NULL_HEADERS, request)); } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java index d9c64f5e36caf..7e4126aa141f9 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerderTest.java @@ -247,6 +247,37 @@ public void testCreateSinkConnector() throws Exception { PowerMock.verifyAll(); } + @Test + public void testCreateConnectorWithStoppedInitialState() throws Exception { + connector = PowerMock.createMock(BogusSinkConnector.class); + Map config = connectorConfig(SourceSink.SINK); + Connector connectorMock = PowerMock.createMock(SinkConnector.class); + expectConfigValidation(connectorMock, false, config); + EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); + + // Only the connector should be created; we expect no tasks to be spawned for a connector created with a paused or stopped initial state + Capture> onStart = EasyMock.newCapture(); + worker.startConnector(eq(CONNECTOR_NAME), eq(config), EasyMock.anyObject(HerderConnectorContext.class), + eq(herder), eq(TargetState.STOPPED), EasyMock.capture(onStart)); + EasyMock.expectLastCall().andAnswer(() -> { + onStart.getValue().onCompletion(null, TargetState.STOPPED); + return true; + }); + EasyMock.expect(worker.isRunning(CONNECTOR_NAME)).andReturn(true).anyTimes(); + EasyMock.expect(herder.connectorType(anyObject())).andReturn(ConnectorType.SINK); + + PowerMock.replayAll(); + + herder.putConnectorConfig(CONNECTOR_NAME, config, TargetState.STOPPED, false, createCallback); + Herder.Created connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS); + assertEquals( + new ConnectorInfo(CONNECTOR_NAME, connectorConfig(SourceSink.SINK), Collections.emptyList(), ConnectorType.SINK), + connectorInfo.result() + ); + + PowerMock.verifyAll(); + } + @Test public void testDestroyConnector() throws Exception { connector = PowerMock.createMock(BogusSourceConnector.class); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java index 59420e8faf047..a03224149511f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/KafkaConfigBackingStoreTest.java @@ -24,11 +24,11 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.ProducerFencedException; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; -import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.data.Field; @@ -77,12 +77,12 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; +import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.GROUP_ID_CONFIG; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.INCLUDE_TASKS_FIELD_NAME; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.ONLY_FAILED_FIELD_NAME; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.READ_WRITE_TOTAL_TIMEOUT_MS; import static org.apache.kafka.connect.storage.KafkaConfigBackingStore.RESTART_KEY; -import static org.apache.kafka.connect.runtime.distributed.DistributedConfig.EXACTLY_ONCE_SOURCE_SUPPORT_CONFIG; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; @@ -177,6 +177,10 @@ public class KafkaConfigBackingStoreTest { "config-bytes-7".getBytes(), "config-bytes-8".getBytes(), "config-bytes-9".getBytes() ); + private static final List TARGET_STATES_SERIALIZED = Arrays.asList( + "started".getBytes(), "paused".getBytes(), "stopped".getBytes() + ); + @Mock private Converter converter; @Mock @@ -320,14 +324,14 @@ public void testPutConnectorConfig() throws Exception { assertNull(configState.connectorConfig(CONNECTOR_IDS.get(1))); // Writing should block until it is written and read back from Kafka - configStorage.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStorage.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); configState = configStorage.snapshot(); assertEquals(1, configState.offset()); assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0))); assertNull(configState.connectorConfig(CONNECTOR_IDS.get(1))); // Second should also block and all configs should still be available - configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(1)); + configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(1), null); configState = configStorage.snapshot(); assertEquals(2, configState.offset()); assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0))); @@ -346,6 +350,55 @@ public void testPutConnectorConfig() throws Exception { PowerMock.verifyAll(); } + @Test + public void testPutConnectorConfigWithTargetState() throws Exception { + expectConfigure(); + expectStart(Collections.emptyList(), Collections.emptyMap()); + + // We expect to write the target state first, followed by the config write and then a read to end + + expectConvertWriteRead( + TARGET_STATE_KEYS.get(0), KafkaConfigBackingStore.TARGET_STATE_V1, TARGET_STATES_SERIALIZED.get(2), + "state.v2", TargetState.STOPPED.name()); + // We don't expect the config update listener's onConnectorTargetStateChange hook to be invoked + + expectConvertWriteRead( + CONNECTOR_CONFIG_KEYS.get(0), KafkaConfigBackingStore.CONNECTOR_CONFIGURATION_V0, CONFIGS_SERIALIZED.get(0), + "properties", SAMPLE_CONFIGS.get(0)); + configUpdateListener.onConnectorConfigUpdate(CONNECTOR_IDS.get(0)); + EasyMock.expectLastCall(); + + LinkedHashMap recordsToRead = new LinkedHashMap<>(); + recordsToRead.put(TARGET_STATE_KEYS.get(0), TARGET_STATES_SERIALIZED.get(2)); + recordsToRead.put(CONNECTOR_CONFIG_KEYS.get(0), CONFIGS_SERIALIZED.get(0)); + expectReadToEnd(recordsToRead); + + expectPartitionCount(1); + expectStop(); + + PowerMock.replayAll(); + + configStorage.setupAndCreateKafkaBasedLog(TOPIC, config); + configStorage.start(); + + // Null before writing + ClusterConfigState configState = configStorage.snapshot(); + assertEquals(-1, configState.offset()); + assertNull(configState.connectorConfig(CONNECTOR_IDS.get(0))); + assertNull(configState.targetState(CONNECTOR_IDS.get(0))); + + // Writing should block until it is written and read back from Kafka + configStorage.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), TargetState.STOPPED); + configState = configStorage.snapshot(); + assertEquals(2, configState.offset()); + assertEquals(TargetState.STOPPED, configState.targetState(CONNECTOR_IDS.get(0))); + assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0))); + + configStorage.stop(); + + PowerMock.verifyAll(); + } + @Test public void testPutConnectorConfigProducerError() throws Exception { expectConfigure(); @@ -373,7 +426,8 @@ public void testPutConnectorConfigProducerError() throws Exception { assertEquals(0, configState.connectors().size()); // verify that the producer exception from KafkaBasedLog::send is propagated - ConnectException e = assertThrows(ConnectException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0))); + ConnectException e = assertThrows(ConnectException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(0), + SAMPLE_CONFIGS.get(0), null)); assertTrue(e.getMessage().contains("Error writing connector configuration to Kafka")); configStorage.stop(); @@ -505,16 +559,16 @@ public void testWritePrivileges() throws Exception { configStorage.putTaskCountRecord(CONNECTOR_IDS.get(0), 6); // Should fail again when we get fenced out - assertThrows(PrivilegedWriteException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0))); + assertThrows(PrivilegedWriteException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0), null)); // Should fail if we retry without reclaiming write privileges - assertThrows(IllegalStateException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0))); + assertThrows(IllegalStateException.class, () -> configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0), null)); // Should succeed even without write privileges (target states can be written by anyone) configStorage.putTargetState(CONNECTOR_IDS.get(1), TargetState.PAUSED); // Should succeed if we re-claim write privileges configStorage.claimWritePrivileges(); - configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0)); + configStorage.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(0), null); configStorage.stop(); @@ -891,7 +945,6 @@ public void testBackgroundUpdateTargetState() throws Exception { expectRead(serializedAfterStartup, deserializedAfterStartup); configUpdateListener.onConnectorTargetStateChange(CONNECTOR_IDS.get(0)); - configUpdateListener.onConnectorTargetStateChange(CONNECTOR_IDS.get(1)); EasyMock.expectLastCall(); expectPartitionCount(1); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/MemoryConfigBackingStoreTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/MemoryConfigBackingStoreTest.java index 3e449b44cf918..185cb604cbf97 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/storage/MemoryConfigBackingStoreTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/storage/MemoryConfigBackingStoreTest.java @@ -40,6 +40,7 @@ import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -66,7 +67,7 @@ public void setUp() { @Test public void testPutConnectorConfig() { - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); ClusterConfigState configState = configStore.snapshot(); assertTrue(configState.contains(CONNECTOR_IDS.get(0))); @@ -78,9 +79,24 @@ public void testPutConnectorConfig() { verify(configUpdateListener).onConnectorConfigUpdate(eq(CONNECTOR_IDS.get(0))); } + @Test + public void testPutConnectorConfigWithTargetState() { + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), TargetState.PAUSED); + ClusterConfigState configState = configStore.snapshot(); + + assertTrue(configState.contains(CONNECTOR_IDS.get(0))); + assertEquals(TargetState.PAUSED, configState.targetState(CONNECTOR_IDS.get(0))); + assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0))); + assertEquals(1, configState.connectors().size()); + + verify(configUpdateListener).onConnectorConfigUpdate(eq(CONNECTOR_IDS.get(0))); + // onConnectorTargetStateChange hook shouldn't be called when a connector is created with a specific initial target state + verify(configUpdateListener, never()).onConnectorTargetStateChange(eq(CONNECTOR_IDS.get(0))); + } + @Test public void testPutConnectorConfigUpdateExisting() { - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); ClusterConfigState configState = configStore.snapshot(); assertTrue(configState.contains(CONNECTOR_IDS.get(0))); @@ -89,7 +105,7 @@ public void testPutConnectorConfigUpdateExisting() { assertEquals(SAMPLE_CONFIGS.get(0), configState.connectorConfig(CONNECTOR_IDS.get(0))); assertEquals(1, configState.connectors().size()); - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(1)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(1), null); configState = configStore.snapshot(); assertEquals(SAMPLE_CONFIGS.get(1), configState.connectorConfig(CONNECTOR_IDS.get(0))); @@ -98,8 +114,8 @@ public void testPutConnectorConfigUpdateExisting() { @Test public void testRemoveConnectorConfig() { - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); - configStore.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(1)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); + configStore.putConnectorConfig(CONNECTOR_IDS.get(1), SAMPLE_CONFIGS.get(1), null); ClusterConfigState configState = configStore.snapshot(); Set expectedConnectors = new HashSet<>(); @@ -124,7 +140,7 @@ public void testPutTaskConfigs() { assertThrows(IllegalArgumentException.class, () -> configStore.putTaskConfigs(CONNECTOR_IDS.get(0), Collections.singletonList(SAMPLE_CONFIGS.get(1)))); - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); configStore.putTaskConfigs(CONNECTOR_IDS.get(0), Collections.singletonList(SAMPLE_CONFIGS.get(1))); ClusterConfigState configState = configStore.snapshot(); @@ -151,7 +167,7 @@ public void testRemoveTaskConfigs() { return null; }).when(configUpdateListener).onTaskConfigUpdate(anySet()); - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); configStore.putTaskConfigs(CONNECTOR_IDS.get(0), Collections.singletonList(SAMPLE_CONFIGS.get(1))); configStore.removeTaskConfigs(CONNECTOR_IDS.get(0)); ClusterConfigState configState = configStore.snapshot(); @@ -171,7 +187,7 @@ public void testPutTargetState() { // Can't write target state for non-existent connector assertThrows(IllegalArgumentException.class, () -> configStore.putTargetState(CONNECTOR_IDS.get(0), TargetState.PAUSED)); - configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0)); + configStore.putConnectorConfig(CONNECTOR_IDS.get(0), SAMPLE_CONFIGS.get(0), null); configStore.putTargetState(CONNECTOR_IDS.get(0), TargetState.PAUSED); // Ensure that ConfigBackingStore.UpdateListener::onConnectorTargetStateChange is called only once if the same state is written twice configStore.putTargetState(CONNECTOR_IDS.get(0), TargetState.PAUSED); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/ConnectAssertions.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/ConnectAssertions.java index 1d1b042bc8f98..bf3b7361b55bf 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/ConnectAssertions.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/ConnectAssertions.java @@ -524,7 +524,7 @@ public void assertConnectorIsStopped(String connectorName, String detailMessage) * @param connectorState * @param numTasks the expected number of tasks * @param tasksState - * @return true if the connector and tasks are in RUNNING state; false otherwise + * @return true if the connector and tasks are in the expected state; false otherwise */ protected Optional checkConnectorState( String connectorName, @@ -554,7 +554,7 @@ protected Optional checkConnectorState( * @param connectorState * @param numTasks the expected number of tasks * @param tasksState - * @return true if the connector and tasks are in RUNNING state; false otherwise + * @return true if the connector and tasks are in the expected state; false otherwise */ protected Optional checkConnectorState( String connectorName, diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnect.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnect.java index 3b8c504e60e46..147e435adf6f1 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnect.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnect.java @@ -30,6 +30,7 @@ import org.apache.kafka.connect.runtime.rest.entities.ConnectorOffset; import org.apache.kafka.connect.runtime.rest.entities.ConnectorOffsets; import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; +import org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest; import org.apache.kafka.connect.runtime.rest.entities.LoggerLevel; import org.apache.kafka.connect.runtime.rest.entities.ServerInfo; import org.apache.kafka.connect.runtime.rest.entities.TaskInfo; @@ -187,7 +188,7 @@ public void requestTimeout(long requestTimeoutMs) { * * @param connName the name of the connector * @param connConfig the intended configuration - * @throws ConnectRestException if the REST api returns error status + * @throws ConnectRestException if the REST API returns error status * @throws ConnectException if the configuration fails to be serialized or if the request could not be sent */ public String configureConnector(String connName, Map connConfig) { @@ -195,6 +196,36 @@ public String configureConnector(String connName, Map connConfig return putConnectorConfig(url, connConfig); } + /** + * Configure a new connector using the POST /connectors endpoint. If the connector already exists, a + * {@link ConnectRestException} will be thrown. + * + * @param createConnectorRequest the connector creation request + * @throws ConnectRestException if the REST API returns error status + * @throws ConnectException if the request could not be sent + */ + public String configureConnector(CreateConnectorRequest createConnectorRequest) { + String url = endpointForResource("connectors"); + ObjectMapper objectMapper = new ObjectMapper(); + + String requestBody; + try { + requestBody = objectMapper.writeValueAsString(createConnectorRequest); + } catch (IOException e) { + throw new ConnectException("Failed to serialize connector creation request: " + createConnectorRequest); + } + + Response response = requestPost(url, requestBody, Collections.emptyMap()); + if (response.getStatus() < Response.Status.BAD_REQUEST.getStatusCode()) { + return responseToString(response); + } else { + throw new ConnectRestException( + response.getStatus(), + "Could not execute 'POST /connectors' request. Error response: " + responseToString(response) + ); + } + } + /** * Validate a given connector configuration. If the configuration validates or * has a configuration error, an instance of {@link ConfigInfos} is returned. If the validation fails diff --git a/core/src/main/java/kafka/log/remote/RemoteLogManager.java b/core/src/main/java/kafka/log/remote/RemoteLogManager.java index 1fcc9049fc03e..5063900627c62 100644 --- a/core/src/main/java/kafka/log/remote/RemoteLogManager.java +++ b/core/src/main/java/kafka/log/remote/RemoteLogManager.java @@ -41,6 +41,7 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.server.common.OffsetAndEpoch; import org.apache.kafka.server.log.remote.metadata.storage.ClassLoaderAwareRemoteLogMetadataManager; import org.apache.kafka.server.log.remote.storage.ClassLoaderAwareRemoteStorageManager; import org.apache.kafka.server.log.remote.storage.LogSegmentData; @@ -515,11 +516,15 @@ public Optional findOffsetByTimestamp(TopicParti while (maybeEpoch.isPresent()) { int epoch = maybeEpoch.getAsInt(); + // KAFKA-15802: Add a new API for RLMM to choose how to implement the predicate. + // currently, all segments are returned and then iterated, and filtered Iterator iterator = remoteLogMetadataManager.listRemoteLogSegments(topicIdPartition, epoch); while (iterator.hasNext()) { RemoteLogSegmentMetadata rlsMetadata = iterator.next(); - if (rlsMetadata.maxTimestampMs() >= timestamp && rlsMetadata.endOffset() >= startingOffset && - isRemoteSegmentWithinLeaderEpochs(rlsMetadata, unifiedLog.logEndOffset(), epochWithOffsets)) { + if (rlsMetadata.maxTimestampMs() >= timestamp + && rlsMetadata.endOffset() >= startingOffset + && isRemoteSegmentWithinLeaderEpochs(rlsMetadata, unifiedLog.logEndOffset(), epochWithOffsets) + && rlsMetadata.state().equals(RemoteLogSegmentState.COPY_SEGMENT_FINISHED)) { return lookupTimestamp(rlsMetadata, timestamp, startingOffset); } } @@ -586,7 +591,7 @@ boolean isLeader() { // The copied and log-start offset is empty initially for a new leader RLMTask, and needs to be fetched inside // the task's run() method. - private volatile OptionalLong copiedOffsetOption = OptionalLong.empty(); + private volatile Optional copiedOffsetOption = Optional.empty(); private volatile boolean isLogStartOffsetUpdatedOnBecomingLeader = false; public void convertToLeader(int leaderEpochVal) { @@ -597,7 +602,7 @@ public void convertToLeader(int leaderEpochVal) { leaderEpoch = leaderEpochVal; } // Reset copied and log-start offset, so that it is set in next run of RLMTask - copiedOffsetOption = OptionalLong.empty(); + copiedOffsetOption = Optional.empty(); isLogStartOffsetUpdatedOnBecomingLeader = false; } @@ -621,9 +626,10 @@ private void maybeUpdateCopiedOffset(UnifiedLog log) throws RemoteStorageExcepti // of a segment with that epoch copied into remote storage. If it can not find an entry then it checks for the // previous leader epoch till it finds an entry, If there are no entries till the earliest leader epoch in leader // epoch cache then it starts copying the segments from the earliest epoch entry's offset. - copiedOffsetOption = OptionalLong.of(findHighestRemoteOffset(topicIdPartition, log)); + copiedOffsetOption = Optional.of(findHighestRemoteOffset(topicIdPartition, log)); logger.info("Found the highest copiedRemoteOffset: {} for partition: {} after becoming leader, " + "leaderEpoch: {}", copiedOffsetOption, topicIdPartition, leaderEpoch); + copiedOffsetOption.ifPresent(offsetAndEpoch -> log.updateHighestOffsetInRemoteStorage(offsetAndEpoch.offset())); } } @@ -660,7 +666,7 @@ public void copyLogSegmentsToRemote(UnifiedLog log) throws InterruptedException try { maybeUpdateLogStartOffsetOnBecomingLeader(log); maybeUpdateCopiedOffset(log); - long copiedOffset = copiedOffsetOption.getAsLong(); + long copiedOffset = copiedOffsetOption.get().offset(); // LSO indicates the offset below are ready to be consumed (high-watermark or committed) long lso = log.lastStableOffset(); @@ -763,7 +769,11 @@ private void copyLogSegment(UnifiedLog log, LogSegment segment, long nextSegment brokerTopicStats.topicStats(log.topicPartition().topic()) .remoteCopyBytesRate().mark(copySegmentStartedRlsm.segmentSizeInBytes()); brokerTopicStats.allTopicsStats().remoteCopyBytesRate().mark(copySegmentStartedRlsm.segmentSizeInBytes()); - copiedOffsetOption = OptionalLong.of(endOffset); + + // `epochEntries` cannot be empty, there is a pre-condition validation in RemoteLogSegmentMetadata + // constructor + int lastEpochInSegment = epochEntries.get(epochEntries.size() - 1).epoch; + copiedOffsetOption = Optional.of(new OffsetAndEpoch(endOffset, lastEpochInSegment)); // Update the highest offset in remote storage for this partition's log so that the local log segments // are not deleted before they are copied to remote storage. log.updateHighestOffsetInRemoteStorage(endOffset); @@ -792,10 +802,10 @@ public void run() { // Cleanup/delete expired remote log segments cleanupExpiredRemoteLogSegments(); } else { - long offset = findHighestRemoteOffset(topicIdPartition, log); + OffsetAndEpoch offsetAndEpoch = findHighestRemoteOffset(topicIdPartition, log); // Update the highest offset in remote storage for this partition's log so that the local log segments // are not deleted before they are copied to remote storage. - log.updateHighestOffsetInRemoteStorage(offset); + log.updateHighestOffsetInRemoteStorage(offsetAndEpoch.offset()); } } catch (InterruptedException ex) { if (!isCancelled()) { @@ -989,6 +999,10 @@ void cleanupExpiredRemoteLogSegments() throws RemoteStorageException, ExecutionE return; } RemoteLogSegmentMetadata metadata = segmentsIterator.next(); + + if (RemoteLogSegmentState.DELETE_SEGMENT_FINISHED.equals(metadata.state())) { + continue; + } if (segmentsToDelete.contains(metadata)) { continue; } @@ -1414,20 +1428,37 @@ RecordBatch findFirstBatch(RemoteLogInputStream remoteLogInputStream, long offse return nextBatch; } - long findHighestRemoteOffset(TopicIdPartition topicIdPartition, UnifiedLog log) throws RemoteStorageException { - Optional offset = Optional.empty(); - - Option maybeLeaderEpochFileCache = log.leaderEpochCache(); - if (maybeLeaderEpochFileCache.isDefined()) { - LeaderEpochFileCache cache = maybeLeaderEpochFileCache.get(); - OptionalInt epoch = cache.latestEpoch(); - while (!offset.isPresent() && epoch.isPresent()) { - offset = remoteLogMetadataManager.highestOffsetForEpoch(topicIdPartition, epoch.getAsInt()); - epoch = cache.previousEpoch(epoch.getAsInt()); + OffsetAndEpoch findHighestRemoteOffset(TopicIdPartition topicIdPartition, UnifiedLog log) throws RemoteStorageException { + OffsetAndEpoch offsetAndEpoch = null; + Option leaderEpochCacheOpt = log.leaderEpochCache(); + if (leaderEpochCacheOpt.isDefined()) { + LeaderEpochFileCache cache = leaderEpochCacheOpt.get(); + Optional maybeEpochEntry = cache.latestEntry(); + while (offsetAndEpoch == null && maybeEpochEntry.isPresent()) { + int epoch = maybeEpochEntry.get().epoch; + Optional highestRemoteOffsetOpt = + remoteLogMetadataManager.highestOffsetForEpoch(topicIdPartition, epoch); + if (highestRemoteOffsetOpt.isPresent()) { + Map.Entry entry = cache.endOffsetFor(epoch, log.logEndOffset()); + int requestedEpoch = entry.getKey(); + long endOffset = entry.getValue(); + long highestRemoteOffset = highestRemoteOffsetOpt.get(); + if (endOffset <= highestRemoteOffset) { + LOGGER.info("The end-offset for epoch {}: ({}, {}) is less than or equal to the " + + "highest-remote-offset: {} for partition: {}", epoch, requestedEpoch, endOffset, + highestRemoteOffset, topicIdPartition); + offsetAndEpoch = new OffsetAndEpoch(endOffset - 1, requestedEpoch); + } else { + offsetAndEpoch = new OffsetAndEpoch(highestRemoteOffset, epoch); + } + } + maybeEpochEntry = cache.previousEntry(epoch); } } - - return offset.orElse(-1L); + if (offsetAndEpoch == null) { + offsetAndEpoch = new OffsetAndEpoch(-1L, RecordBatch.NO_PARTITION_LEADER_EPOCH); + } + return offsetAndEpoch; } long findLogStartOffset(TopicIdPartition topicIdPartition, UnifiedLog log) throws RemoteStorageException { @@ -1659,5 +1690,4 @@ public String toString() { '}'; } } - } diff --git a/core/src/main/java/kafka/server/ClientMetricsManager.java b/core/src/main/java/kafka/server/ClientMetricsManager.java deleted file mode 100644 index aecb33f7cb2f2..0000000000000 --- a/core/src/main/java/kafka/server/ClientMetricsManager.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Properties; - -/** - * Handles client telemetry metrics requests/responses, subscriptions and instance information. - */ -public class ClientMetricsManager implements Closeable { - - private static final Logger log = LoggerFactory.getLogger(ClientMetricsManager.class); - private static final ClientMetricsManager INSTANCE = new ClientMetricsManager(); - - public static ClientMetricsManager instance() { - return INSTANCE; - } - - public void updateSubscription(String subscriptionName, Properties properties) { - // TODO: Implement the update logic to manage subscriptions. - } - - @Override - public void close() throws IOException { - // TODO: Implement the close logic to close the client metrics manager. - } -} diff --git a/core/src/main/java/kafka/server/builders/KafkaApisBuilder.java b/core/src/main/java/kafka/server/builders/KafkaApisBuilder.java index 09ae8e6aa155f..9182f9c4bcd1f 100644 --- a/core/src/main/java/kafka/server/builders/KafkaApisBuilder.java +++ b/core/src/main/java/kafka/server/builders/KafkaApisBuilder.java @@ -22,7 +22,6 @@ import kafka.server.ApiVersionManager; import kafka.server.AutoTopicCreationManager; import kafka.server.BrokerTopicStats; -import kafka.server.ClientMetricsManager; import kafka.server.DelegationTokenManager; import kafka.server.FetchManager; import kafka.server.KafkaApis; @@ -35,6 +34,7 @@ import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.Time; import org.apache.kafka.coordinator.group.GroupCoordinator; +import org.apache.kafka.server.ClientMetricsManager; import org.apache.kafka.server.authorizer.Authorizer; import java.util.Collections; diff --git a/core/src/main/java/kafka/server/builders/ReplicaManagerBuilder.java b/core/src/main/java/kafka/server/builders/ReplicaManagerBuilder.java index 2566e4bcfcbcb..6859900100e7b 100644 --- a/core/src/main/java/kafka/server/builders/ReplicaManagerBuilder.java +++ b/core/src/main/java/kafka/server/builders/ReplicaManagerBuilder.java @@ -35,6 +35,7 @@ import kafka.zk.KafkaZkClient; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.server.common.DirectoryEventHandler; import org.apache.kafka.storage.internals.log.LogDirFailureChannel; import org.apache.kafka.server.util.Scheduler; import scala.compat.java8.OptionConverters; @@ -66,6 +67,7 @@ public class ReplicaManagerBuilder { private Optional threadNamePrefix = Optional.empty(); private Long brokerEpoch = -1L; private Optional addPartitionsToTxnManager = Optional.empty(); + private DirectoryEventHandler directoryEventHandler = DirectoryEventHandler.NOOP; public ReplicaManagerBuilder setConfig(KafkaConfig config) { this.config = config; @@ -172,6 +174,11 @@ public ReplicaManagerBuilder setAddPartitionsToTransactionManager(AddPartitionsT return this; } + public ReplicaManagerBuilder setDirectoryEventHandler(DirectoryEventHandler directoryEventHandler) { + this.directoryEventHandler = directoryEventHandler; + return this; + } + public ReplicaManager build() { if (config == null) config = new KafkaConfig(Collections.emptyMap()); if (metrics == null) metrics = new Metrics(); @@ -200,6 +207,7 @@ public ReplicaManager build() { OptionConverters.toScala(delayedRemoteFetchPurgatory), OptionConverters.toScala(threadNamePrefix), () -> brokerEpoch, - OptionConverters.toScala(addPartitionsToTxnManager)); + OptionConverters.toScala(addPartitionsToTxnManager), + directoryEventHandler); } } diff --git a/core/src/main/scala/kafka/Kafka.scala b/core/src/main/scala/kafka/Kafka.scala index a1791ccbe0bf1..efa6286051494 100755 --- a/core/src/main/scala/kafka/Kafka.scala +++ b/core/src/main/scala/kafka/Kafka.scala @@ -18,7 +18,6 @@ package kafka import java.util.Properties - import joptsimple.OptionParser import kafka.server.{KafkaConfig, KafkaRaftServer, KafkaServer, Server} import kafka.utils.Implicits._ diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 25d400918f691..37d41458c39ad 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -29,7 +29,7 @@ import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.clients.admin.{Admin, AlterClientQuotasOptions, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeClusterOptions, DescribeConfigsOptions, ListTopicsOptions, ScramCredentialInfo, UserScramCredentialDeletion, UserScramCredentialUpsertion, Config => JConfig, ScramMechanism => PublicScramMechanism} import org.apache.kafka.common.config.{ConfigResource, TopicConfig} import org.apache.kafka.common.config.types.Password -import org.apache.kafka.common.errors.InvalidConfigurationException +import org.apache.kafka.common.errors.{InvalidConfigurationException, InvalidRequestException} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, ClientQuotaFilter, ClientQuotaFilterComponent} import org.apache.kafka.common.security.JaasUtils @@ -44,7 +44,7 @@ import scala.jdk.CollectionConverters._ import scala.collection._ /** - * This script can be used to change configs for topics/clients/users/brokers/ips dynamically + * This script can be used to change configs for topics/clients/users/brokers/ips/client-metrics dynamically * An entity described or altered by the command may be one of: *
      *
    • topic: --topic OR --entity-type topics --entity-name @@ -55,6 +55,7 @@ import scala.collection._ *
    • broker: --broker OR --entity-type brokers --entity-name *
    • broker-logger: --broker-logger OR --entity-type broker-loggers --entity-name *
    • ip: --ip OR --entity-type ips --entity-name + *
    • client-metrics: --client-metrics OR --entity-type client-metrics --entity-name *
    * --entity-type --entity-default may be specified in place of --entity-type --entity-name * when describing or altering default configuration for users, clients, brokers, or ips, respectively. @@ -76,7 +77,7 @@ object ConfigCommand extends Logging { val BrokerDefaultEntityName = "" val BrokerLoggerConfigType = "broker-loggers" - val BrokerSupportedConfigTypes = ConfigType.all :+ BrokerLoggerConfigType + val BrokerSupportedConfigTypes = ConfigType.all :+ BrokerLoggerConfigType :+ ConfigType.ClientMetrics val ZkSupportedConfigTypes = Seq(ConfigType.User, ConfigType.Broker) val DefaultScramIterations = 4096 @@ -84,7 +85,7 @@ object ConfigCommand extends Logging { try { val opts = new ConfigCommandOptions(args) - CommandLineUtils.maybePrintHelpOrVersion(opts, "This tool helps to manipulate and describe entity config for a topic, client, user, broker or ip") + CommandLineUtils.maybePrintHelpOrVersion(opts, "This tool helps to manipulate and describe entity config for a topic, client, user, broker, ip or client-metrics") opts.checkArgs() @@ -445,6 +446,21 @@ object ConfigCommand extends Logging { if (unknownConfigs.nonEmpty) throw new IllegalArgumentException(s"Only connection quota configs can be added for '${ConfigType.Ip}' using --bootstrap-server. Unexpected config names: ${unknownConfigs.mkString(",")}") alterQuotaConfigs(adminClient, entityTypes, entityNames, configsToBeAddedMap, configsToBeDeleted) + case ConfigType.ClientMetrics => + val oldConfig = getResourceConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) + .map { entry => (entry.name, entry) }.toMap + + // fail the command if any of the configs to be deleted does not exist + val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains) + if (invalidConfigs.nonEmpty) + throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}") + + val configResource = new ConfigResource(ConfigResource.Type.CLIENT_METRICS, entityNameHead) + val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false) + val alterEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET)) + ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) } + ).asJavaCollection + adminClient.incrementalAlterConfigs(Map(configResource -> alterEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case _ => throw new IllegalArgumentException(s"Unsupported entity type: $entityTypeHead") } @@ -518,7 +534,7 @@ object ConfigCommand extends Logging { val describeAll = opts.options.has(opts.allOpt) entityTypes.head match { - case ConfigType.Topic | ConfigType.Broker | BrokerLoggerConfigType => + case ConfigType.Topic | ConfigType.Broker | BrokerLoggerConfigType | ConfigType.ClientMetrics => describeResourceConfig(adminClient, entityTypes.head, entityNames.headOption, describeAll) case ConfigType.User | ConfigType.Client => describeClientQuotaAndUserScramCredentialConfigs(adminClient, entityTypes, entityNames) @@ -536,6 +552,8 @@ object ConfigCommand extends Logging { adminClient.listTopics(new ListTopicsOptions().listInternal(true)).names().get().asScala.toSeq case ConfigType.Broker | BrokerLoggerConfigType => adminClient.describeCluster(new DescribeClusterOptions()).nodes().get().asScala.map(_.idString).toSeq :+ BrokerDefaultEntityName + case ConfigType.ClientMetrics => + throw new InvalidRequestException("Client metrics entity-name is required") case entityType => throw new IllegalArgumentException(s"Invalid entity type: $entityType") }) @@ -576,6 +594,8 @@ object ConfigCommand extends Logging { if (entityName.nonEmpty) validateBrokerId() (ConfigResource.Type.BROKER_LOGGER, None) + case ConfigType.ClientMetrics => + (ConfigResource.Type.CLIENT_METRICS, Some(ConfigEntry.ConfigSource.DYNAMIC_CLIENT_METRICS_CONFIG)) case entityType => throw new IllegalArgumentException(s"Invalid entity type: $entityType") } @@ -791,10 +811,10 @@ object ConfigCommand extends Logging { val describeOpt = parser.accepts("describe", "List configs for the given entity.") val allOpt = parser.accepts("all", "List all configs for the given topic, broker, or broker-logger entity (includes static configuration when the entity type is brokers)") - val entityType = parser.accepts("entity-type", "Type of entity (topics/clients/users/brokers/broker-loggers/ips)") + val entityType = parser.accepts("entity-type", "Type of entity (topics/clients/users/brokers/broker-loggers/ips/client-metrics)") .withRequiredArg .ofType(classOf[String]) - val entityName = parser.accepts("entity-name", "Name of entity (topic name/client id/user principal name/broker id/ip)") + val entityName = parser.accepts("entity-name", "Name of entity (topic name/client id/user principal name/broker id/ip/client metrics)") .withRequiredArg .ofType(classOf[String]) val entityDefault = parser.accepts("entity-default", "Default entity name for clients/users/brokers/ips (applies to corresponding entity type in command line)") @@ -806,6 +826,7 @@ object ConfigCommand extends Logging { "For entity-type '" + ConfigType.User + "': " + DynamicConfig.User.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) + "For entity-type '" + ConfigType.Client + "': " + DynamicConfig.Client.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) + "For entity-type '" + ConfigType.Ip + "': " + DynamicConfig.Ip.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) + + "For entity-type '" + ConfigType.ClientMetrics + "': " + DynamicConfig.ClientMetrics.names.asScala.toSeq.sorted.map("\t" + _).mkString(nl, nl, nl) + s"Entity types '${ConfigType.User}' and '${ConfigType.Client}' may be specified together to update config for clients of a specific user.") .withRequiredArg .ofType(classOf[String]) @@ -939,7 +960,8 @@ object ConfigCommand extends Logging { } } - if (options.has(describeOpt) && entityTypeVals.contains(BrokerLoggerConfigType) && !hasEntityName) + if (options.has(describeOpt) && (entityTypeVals.contains(BrokerLoggerConfigType) || + entityTypeVals.contains(ConfigType.ClientMetrics)) && !hasEntityName) throw new IllegalArgumentException(s"an entity name must be specified with --describe of ${entityTypeVals.mkString(",")}") if (options.has(alterOpt)) { diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 251f198dc573d..abb69926eb38c 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -85,6 +85,7 @@ trait AlterPartitionListener { def markIsrExpand(): Unit def markIsrShrink(): Unit def markFailed(): Unit + def assignDir(dir: String): Unit } class DelayedOperations(topicPartition: TopicPartition, @@ -119,6 +120,10 @@ object Partition { } override def markFailed(): Unit = replicaManager.failedIsrUpdatesRate.mark() + + override def assignDir(dir: String): Unit = { + replicaManager.maybeNotifyPartitionAssignedToDirectory(topicPartition, dir) + } } val delayedOperations = new DelayedOperations( @@ -480,6 +485,7 @@ class Partition(val topicPartition: TopicPartition, if (!isFutureReplica) log.setLogOffsetsListener(logOffsetsListener) maybeLog = Some(log) updateHighWatermark(log) + alterPartitionListener.assignDir(log.parentDir) log } finally { logManager.finishedInitializingLog(topicPartition, maybeLog) diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 33a335dcc2f60..18e211f62bd54 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -377,7 +377,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, val stopReplicaRequestMap = mutable.Map.empty[Int, mutable.Map[TopicPartition, StopReplicaPartitionState]] val updateMetadataRequestBrokerSet = mutable.Set.empty[Int] val updateMetadataRequestPartitionInfoMap = mutable.Map.empty[TopicPartition, UpdateMetadataPartitionState] - private var updateType: LeaderAndIsrRequest.Type = LeaderAndIsrRequest.Type.UNKNOWN + private var updateType: AbstractControlRequest.Type = AbstractControlRequest.Type.UNKNOWN private var metadataInstance: ControllerChannelContext = _ def sendRequest(brokerId: Int, @@ -399,7 +399,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, metadataInstance = metadataProvider() } - def setUpdateType(updateType: LeaderAndIsrRequest.Type): Unit = { + def setUpdateType(updateType: AbstractControlRequest.Type): Unit = { this.updateType = updateType } @@ -409,7 +409,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, updateMetadataRequestBrokerSet.clear() updateMetadataRequestPartitionInfoMap.clear() metadataInstance = null - updateType = LeaderAndIsrRequest.Type.UNKNOWN + updateType = AbstractControlRequest.Type.UNKNOWN } def addLeaderAndIsrRequestForBrokers(brokerIds: Seq[Int], @@ -567,7 +567,6 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, } } leaderAndIsrRequestMap.clear() - updateType = LeaderAndIsrRequest.Type.UNKNOWN } def handleLeaderAndIsrResponse(response: LeaderAndIsrResponse, broker: Int): Unit @@ -621,8 +620,17 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, .distinct .filter(metadataInstance.topicIds.contains) .map(topic => (topic, metadataInstance.topicIds(topic))).toMap - val updateMetadataRequestBuilder = new UpdateMetadataRequest.Builder(updateMetadataRequestVersion, - controllerId, controllerEpoch, brokerEpoch, partitionStates.asJava, liveBrokers.asJava, topicIds.asJava, kraftController) + val updateMetadataRequestBuilder = new UpdateMetadataRequest.Builder( + updateMetadataRequestVersion, + controllerId, + controllerEpoch, + brokerEpoch, + partitionStates.asJava, + liveBrokers.asJava, + topicIds.asJava, + kraftController, + updateType + ) sendRequest(broker, updateMetadataRequestBuilder, (r: AbstractResponse) => { val updateMetadataResponse = r.asInstanceOf[UpdateMetadataResponse] handleUpdateMetadataResponse(updateMetadataResponse, broker) @@ -736,6 +744,7 @@ abstract class AbstractControllerBrokerRequestBatch(config: KafkaConfig, sendLeaderAndIsrRequest(controllerEpoch, stateChangeLog) sendUpdateMetadataRequests(controllerEpoch, stateChangeLog) sendStopReplicaRequests(controllerEpoch, stateChangeLog) + this.updateType = AbstractControlRequest.Type.UNKNOWN } catch { case e: Throwable => if (leaderAndIsrRequestMap.nonEmpty) { diff --git a/core/src/main/scala/kafka/coordinator/group/CoordinatorLoaderImpl.scala b/core/src/main/scala/kafka/coordinator/group/CoordinatorLoaderImpl.scala index e2ac2f66ead27..a9a50b77583bd 100644 --- a/core/src/main/scala/kafka/coordinator/group/CoordinatorLoaderImpl.scala +++ b/core/src/main/scala/kafka/coordinator/group/CoordinatorLoaderImpl.scala @@ -139,7 +139,11 @@ class CoordinatorLoaderImpl[T]( batch.asScala.foreach { record => numRecords = numRecords + 1 try { - coordinator.replay(deserializer.deserialize(record.key, record.value)) + coordinator.replay( + batch.producerId, + batch.producerEpoch, + deserializer.deserialize(record.key, record.value) + ) } catch { case ex: UnknownRecordTypeException => warn(s"Unknown record type ${ex.unknownType} while loading offsets and group metadata " + diff --git a/core/src/main/scala/kafka/coordinator/group/CoordinatorPartitionWriter.scala b/core/src/main/scala/kafka/coordinator/group/CoordinatorPartitionWriter.scala index 5ee576ff3c52e..2c12a30e0f0fa 100644 --- a/core/src/main/scala/kafka/coordinator/group/CoordinatorPartitionWriter.scala +++ b/core/src/main/scala/kafka/coordinator/group/CoordinatorPartitionWriter.scala @@ -21,14 +21,13 @@ import kafka.server.{ActionQueue, ReplicaManager} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.RecordTooLargeException import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, TimestampType} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, TimestampType} import org.apache.kafka.common.record.Record.EMPTY_HEADERS import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse -import org.apache.kafka.common.utils.Time +import org.apache.kafka.common.utils.{BufferSupplier, Time} import org.apache.kafka.coordinator.group.runtime.PartitionWriter import org.apache.kafka.storage.internals.log.AppendOrigin -import java.nio.ByteBuffer import java.util import scala.collection.Map @@ -66,6 +65,10 @@ class CoordinatorPartitionWriter[T]( compressionType: CompressionType, time: Time ) extends PartitionWriter[T] { + private val threadLocalBufferSupplier = ThreadLocal.withInitial( + () => new BufferSupplier.GrowableBufferSupplier() + ) + // We use an action queue which directly executes actions. This is possible // here because we don't hold any conflicting locks. private val directActionQueue = new ActionQueue { @@ -106,13 +109,17 @@ class CoordinatorPartitionWriter[T]( * Write records to the partitions. Records are written in one batch so * atomicity is guaranteed. * - * @param tp The partition to write records to. - * @param records The list of records. The records are written in a single batch. + * @param tp The partition to write records to. + * @param producerId The producer id. + * @param producerEpoch The producer epoch. + * @param records The list of records. The records are written in a single batch. * @return The log end offset right after the written records. * @throws KafkaException Any KafkaException caught during the write operation. */ override def append( tp: TopicPartition, + producerId: Long, + producerEpoch: Short, records: util.List[T] ): Long = { if (records.isEmpty) throw new IllegalStateException("records must be non-empty.") @@ -122,52 +129,63 @@ class CoordinatorPartitionWriter[T]( val magic = logConfig.recordVersion.value val maxBatchSize = logConfig.maxMessageSize val currentTimeMs = time.milliseconds() - - val recordsBuilder = MemoryRecords.builder( - ByteBuffer.allocate(math.min(16384, maxBatchSize)), - magic, - compressionType, - TimestampType.CREATE_TIME, - 0L, - maxBatchSize - ) - - records.forEach { record => - val keyBytes = serializer.serializeKey(record) - val valueBytes = serializer.serializeValue(record) - - if (recordsBuilder.hasRoomFor(currentTimeMs, keyBytes, valueBytes, EMPTY_HEADERS)) recordsBuilder.append( - currentTimeMs, - keyBytes, - valueBytes, - EMPTY_HEADERS - ) else throw new RecordTooLargeException(s"Message batch size is ${recordsBuilder.estimatedSizeInBytes()} bytes " + - s"in append to partition $tp which exceeds the maximum configured size of $maxBatchSize.") + val bufferSupplier = threadLocalBufferSupplier.get() + val buffer = bufferSupplier.get(math.min(16384, maxBatchSize)) + + try { + val recordsBuilder = MemoryRecords.builder( + buffer, + magic, + compressionType, + TimestampType.CREATE_TIME, + 0L, + time.milliseconds(), + producerId, + producerEpoch, + 0, + producerId != RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PARTITION_LEADER_EPOCH + ) + + records.forEach { record => + val keyBytes = serializer.serializeKey(record) + val valueBytes = serializer.serializeValue(record) + + if (recordsBuilder.hasRoomFor(currentTimeMs, keyBytes, valueBytes, EMPTY_HEADERS)) recordsBuilder.append( + currentTimeMs, + keyBytes, + valueBytes, + EMPTY_HEADERS + ) else throw new RecordTooLargeException(s"Message batch size is ${recordsBuilder.estimatedSizeInBytes()} bytes " + + s"in append to partition $tp which exceeds the maximum configured size of $maxBatchSize.") + } + + var appendResults: Map[TopicPartition, PartitionResponse] = Map.empty + replicaManager.appendRecords( + timeout = 0L, + requiredAcks = 1, + internalTopicsAllowed = true, + origin = AppendOrigin.COORDINATOR, + entriesPerPartition = Map(tp -> recordsBuilder.build()), + responseCallback = results => appendResults = results, + // We can directly complete the purgatories here because we don't hold + // any conflicting locks. + actionQueue = directActionQueue + ) + + val partitionResult = appendResults.getOrElse(tp, + throw new IllegalStateException(s"Append status $appendResults should have partition $tp.")) + + if (partitionResult.error != Errors.NONE) { + throw partitionResult.error.exception() + } + + // Required offset. + partitionResult.lastOffset + 1 + } finally { + bufferSupplier.release(buffer) } - var appendResults: Map[TopicPartition, PartitionResponse] = Map.empty - replicaManager.appendRecords( - timeout = 0L, - requiredAcks = 1, - internalTopicsAllowed = true, - origin = AppendOrigin.COORDINATOR, - entriesPerPartition = Map(tp -> recordsBuilder.build()), - responseCallback = results => appendResults = results, - // We can directly complete the purgatories here because we don't hold - // any conflicting locks. - actionQueue = directActionQueue - ) - - val partitionResult = appendResults.getOrElse(tp, - throw new IllegalStateException(s"Append status $appendResults should have partition $tp.")) - - if (partitionResult.error != Errors.NONE) { - throw partitionResult.error.exception() - } - - // Required offset. - partitionResult.lastOffset + 1 - case None => throw Errors.NOT_LEADER_OR_FOLLOWER.exception() } diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinatorAdapter.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinatorAdapter.scala index 409c1e1d849b3..678844cc66255 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinatorAdapter.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinatorAdapter.scala @@ -20,7 +20,7 @@ import kafka.common.OffsetAndMetadata import kafka.server.{KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils.Implicits.MapExtensionMethods import org.apache.kafka.common.{TopicIdPartition, TopicPartition, Uuid} -import org.apache.kafka.common.message.{ConsumerGroupHeartbeatRequestData, ConsumerGroupHeartbeatResponseData, DeleteGroupsResponseData, DescribeGroupsResponseData, HeartbeatRequestData, HeartbeatResponseData, JoinGroupRequestData, JoinGroupResponseData, LeaveGroupRequestData, LeaveGroupResponseData, ListGroupsRequestData, ListGroupsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteRequestData, OffsetDeleteResponseData, OffsetFetchRequestData, OffsetFetchResponseData, SyncGroupRequestData, SyncGroupResponseData, TxnOffsetCommitRequestData, TxnOffsetCommitResponseData} +import org.apache.kafka.common.message.{ConsumerGroupDescribeResponseData, ConsumerGroupHeartbeatRequestData, ConsumerGroupHeartbeatResponseData, DeleteGroupsResponseData, DescribeGroupsResponseData, HeartbeatRequestData, HeartbeatResponseData, JoinGroupRequestData, JoinGroupResponseData, LeaveGroupRequestData, LeaveGroupResponseData, ListGroupsRequestData, ListGroupsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteRequestData, OffsetDeleteResponseData, OffsetFetchRequestData, OffsetFetchResponseData, SyncGroupRequestData, SyncGroupResponseData, TxnOffsetCommitRequestData, TxnOffsetCommitResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.RecordBatch @@ -601,4 +601,13 @@ private[group] class GroupCoordinatorAdapter( override def shutdown(): Unit = { coordinator.shutdown() } + + override def consumerGroupDescribe( + context: RequestContext, + groupIds: util.List[String] + ): CompletableFuture[util.List[ConsumerGroupDescribeResponseData.DescribedGroup]] = { + FutureUtils.failedFuture(Errors.UNSUPPORTED_VERSION.exception( + s"The old group coordinator does not support ${ApiKeys.CONSUMER_GROUP_DESCRIBE.name} API." + )) + } } diff --git a/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala b/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala index 9e7b88d2ed776..06e358d7b6cba 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/ProducerIdManager.scala @@ -17,7 +17,6 @@ package kafka.coordinator.transaction import kafka.coordinator.transaction.ProducerIdManager.{IterationLimit, NoRetry, RetryBackoffMs} -import kafka.server.{NodeToControllerChannelManager, ControllerRequestCompletionHandler} import kafka.utils.Logging import kafka.zk.{KafkaZkClient, ProducerIdBlockZNode} import org.apache.kafka.clients.ClientResponse @@ -26,6 +25,7 @@ import org.apache.kafka.common.message.AllocateProducerIdsRequestData import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{AllocateProducerIdsRequest, AllocateProducerIdsResponse} import org.apache.kafka.common.utils.Time +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.ProducerIdsBlock import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong, AtomicReference} diff --git a/core/src/main/scala/kafka/log/LocalLog.scala b/core/src/main/scala/kafka/log/LocalLog.scala index 27b89864ffc83..92758d01144aa 100644 --- a/core/src/main/scala/kafka/log/LocalLog.scala +++ b/core/src/main/scala/kafka/log/LocalLog.scala @@ -32,7 +32,7 @@ import java.util import java.util.concurrent.atomic.AtomicLong import java.util.regex.Pattern import java.util.{Collections, Optional} -import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import scala.collection.mutable.ListBuffer import scala.collection.{Seq, immutable} import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ @@ -257,36 +257,6 @@ class LocalLog(@volatile private var _dir: File, } } - /** - * Find segments starting from the oldest until the user-supplied predicate is false. - * A final segment that is empty will never be returned. - * - * @param predicate A function that takes in a candidate log segment, the next higher segment - * (if there is one). It returns true iff the segment is deletable. - * @return the segments ready to be deleted - */ - private[log] def deletableSegments(predicate: (LogSegment, Option[LogSegment]) => Boolean): Iterable[LogSegment] = { - if (segments.isEmpty) { - Seq.empty - } else { - val deletable = ArrayBuffer.empty[LogSegment] - val segmentsIterator = segments.values.iterator - var segmentOpt = nextOption(segmentsIterator) - while (segmentOpt.isDefined) { - val segment = segmentOpt.get - val nextSegmentOpt = nextOption(segmentsIterator) - val isLastSegmentAndEmpty = nextSegmentOpt.isEmpty && segment.size == 0 - if (predicate(segment, nextSegmentOpt) && !isLastSegmentAndEmpty) { - deletable += segment - segmentOpt = nextSegmentOpt - } else { - segmentOpt = Option.empty - } - } - deletable - } - } - /** * This method deletes the given log segments by doing the following for each of them: * - It removes the segment from the segment map so that it will no longer be used for reads. @@ -982,7 +952,7 @@ object LocalLog extends Logging { * @tparam T the type of object held within the iterator * @return Some(iterator.next) if a next element exists, None otherwise. */ - private def nextOption[T](iterator: util.Iterator[T]): Option[T] = { + private[log] def nextOption[T](iterator: util.Iterator[T]): Option[T] = { if (iterator.hasNext) Some(iterator.next()) else diff --git a/core/src/main/scala/kafka/log/UnifiedLog.scala b/core/src/main/scala/kafka/log/UnifiedLog.scala index d1fff6783d8e8..8ca58ad20f0ce 100644 --- a/core/src/main/scala/kafka/log/UnifiedLog.scala +++ b/core/src/main/scala/kafka/log/UnifiedLog.scala @@ -19,6 +19,7 @@ package kafka.log import com.yammer.metrics.core.MetricName import kafka.common.{OffsetsOutOfOrderException, UnexpectedAppendOffsetException} +import kafka.log.LocalLog.nextOption import kafka.log.remote.RemoteLogManager import kafka.server.{BrokerTopicMetrics, BrokerTopicStats, RequestLocal} import kafka.utils._ @@ -43,13 +44,13 @@ import org.apache.kafka.storage.internals.epoch.LeaderEpochFileCache import org.apache.kafka.storage.internals.log.{AbortedTxn, AppendOrigin, BatchMetadata, CompletedTxn, EpochEntry, FetchDataInfo, FetchIsolation, LastRecord, LeaderHwChange, LogAppendInfo, LogConfig, LogDirFailureChannel, LogFileUtils, LogOffsetMetadata, LogOffsetSnapshot, LogOffsetsListener, LogSegment, LogSegments, LogStartOffsetIncrementReason, LogValidator, ProducerAppendInfo, ProducerStateManager, ProducerStateManagerConfig, RollParams, VerificationGuard} import java.io.{File, IOException} -import java.nio.file.Files +import java.nio.file.{Files, Path} import java.util import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap} import java.util.stream.Collectors import java.util.{Collections, Optional, OptionalInt, OptionalLong} import scala.annotation.nowarn -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ArrayBuffer, ListBuffer} import scala.collection.{Seq, immutable, mutable} import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ @@ -818,7 +819,7 @@ class UnifiedLog(@volatile var logStartOffset: Long, appendInfo.setMaxTimestamp(validateAndOffsetAssignResult.maxTimestampMs) appendInfo.setOffsetOfMaxTimestamp(validateAndOffsetAssignResult.shallowOffsetOfMaxTimestampMs) appendInfo.setLastOffset(offset.value - 1) - appendInfo.setRecordConversionStats(validateAndOffsetAssignResult.recordConversionStats) + appendInfo.setRecordValidationStats(validateAndOffsetAssignResult.recordValidationStats) if (config.messageTimestampType == TimestampType.LOG_APPEND_TIME) appendInfo.setLogAppendTime(validateAndOffsetAssignResult.logAppendTimeMs) @@ -1188,7 +1189,7 @@ class UnifiedLog(@volatile var logStartOffset: Long, OptionalInt.empty() new LogAppendInfo(firstOffset, lastOffset, lastLeaderEpochOpt, maxTimestamp, offsetOfMaxTimestamp, - RecordBatch.NO_TIMESTAMP, logStartOffset, RecordConversionStats.EMPTY, sourceCompression, + RecordBatch.NO_TIMESTAMP, logStartOffset, RecordValidationStats.EMPTY, sourceCompression, validBytesCount, lastOffsetOfFirstBatch, Collections.emptyList[RecordError], LeaderHwChange.NONE) } @@ -1424,18 +1425,8 @@ class UnifiedLog(@volatile var logStartOffset: Long, */ private def deleteOldSegments(predicate: (LogSegment, Option[LogSegment]) => Boolean, reason: SegmentDeletionReason): Int = { - def shouldDelete(segment: LogSegment, nextSegmentOpt: Option[LogSegment]): Boolean = { - val upperBoundOffset = nextSegmentOpt.map(_.baseOffset).getOrElse(localLog.logEndOffset) - - // Check not to delete segments which are not yet copied to tiered storage if remote log is enabled. - (!remoteLogEnabled() || (upperBoundOffset > 0 && upperBoundOffset - 1 <= highestOffsetInRemoteStorage)) && - // We don't delete segments with offsets at or beyond the high watermark to ensure that the log start - // offset can never exceed it. - highWatermark >= upperBoundOffset && - predicate(segment, nextSegmentOpt) - } lock synchronized { - val deletable = localLog.deletableSegments(shouldDelete) + val deletable = deletableSegments(predicate) if (deletable.nonEmpty) deleteSegments(deletable, reason) else @@ -1443,6 +1434,61 @@ class UnifiedLog(@volatile var logStartOffset: Long, } } + /** + * Find segments starting from the oldest until the user-supplied predicate is false. + * A final segment that is empty will never be returned. + * + * @param predicate A function that takes in a candidate log segment, the next higher segment + * (if there is one). It returns true iff the segment is deletable. + * @return the segments ready to be deleted + */ + private[log] def deletableSegments(predicate: (LogSegment, Option[LogSegment]) => Boolean): Iterable[LogSegment] = { + def isSegmentEligibleForDeletion(upperBoundOffset: Long): Boolean = { + // Segments are eligible for deletion when: + // 1. they are uploaded to the remote storage + if (remoteLogEnabled()) { + upperBoundOffset > 0 && upperBoundOffset - 1 <= highestOffsetInRemoteStorage + } else { + true + } + } + + if (localLog.segments.isEmpty) { + Seq.empty + } else { + val deletable = ArrayBuffer.empty[LogSegment] + val segmentsIterator = localLog.segments.values.iterator + var segmentOpt = nextOption(segmentsIterator) + var shouldRoll = false + while (segmentOpt.isDefined) { + val segment = segmentOpt.get + val nextSegmentOpt = nextOption(segmentsIterator) + val isLastSegmentAndEmpty = nextSegmentOpt.isEmpty && segment.size == 0 + val upperBoundOffset = if (nextSegmentOpt.nonEmpty) nextSegmentOpt.get.baseOffset() else logEndOffset + // We don't delete segments with offsets at or beyond the high watermark to ensure that the log start + // offset can never exceed it. + val predicateResult = highWatermark >= upperBoundOffset && predicate(segment, nextSegmentOpt) + + // Roll the active segment when it breaches the configured retention policy. The rolled segment will be + // eligible for deletion and gets removed in the next iteration. + if (predicateResult && remoteLogEnabled() && nextSegmentOpt.isEmpty && segment.size > 0) { + shouldRoll = true + } + if (predicateResult && !isLastSegmentAndEmpty && isSegmentEligibleForDeletion(upperBoundOffset)) { + deletable += segment + segmentOpt = nextSegmentOpt + } else { + segmentOpt = Option.empty + } + } + if (shouldRoll) { + info("Rolling the active segment to make it eligible for deletion") + roll() + } + deletable + } + } + private def incrementStartOffset(startOffset: Long, reason: LogStartOffsetIncrementReason): Unit = { if (remoteLogEnabled()) maybeIncrementLocalLogStartOffset(startOffset, reason) else maybeIncrementLogStartOffset(startOffset, reason) @@ -1610,10 +1656,16 @@ class UnifiedLog(@volatile var logStartOffset: Long, // may actually be ahead of the current producer state end offset (which corresponds to the log end offset), // we manually override the state offset here prior to taking the snapshot. producerStateManager.updateMapEndOffset(newSegment.baseOffset) - producerStateManager.takeSnapshot() + // We avoid potentially-costly fsync call, since we acquire UnifiedLog#lock here + // which could block subsequent produces in the meantime. + // flush is done in the scheduler thread along with segment flushing below + val maybeSnapshot = producerStateManager.takeSnapshot(false) updateHighWatermarkWithLogEndOffset() // Schedule an asynchronous flush of the old segment - scheduler.scheduleOnce("flush-log", () => flushUptoOffsetExclusive(newSegment.baseOffset)) + scheduler.scheduleOnce("flush-log", () => { + maybeSnapshot.ifPresent(f => flushProducerStateSnapshot(f.toPath)) + flushUptoOffsetExclusive(newSegment.baseOffset) + }) newSegment } @@ -1696,6 +1748,12 @@ class UnifiedLog(@volatile var logStartOffset: Long, producerStateManager.mapEndOffset } + private[log] def flushProducerStateSnapshot(snapshot: Path): Unit = { + maybeHandleIOException(s"Error while deleting producer state snapshot $snapshot for $topicPartition in dir ${dir.getParent}") { + Utils.flushFileIfExists(snapshot) + } + } + /** * Truncate this log so that it ends with the greatest offset < targetOffset. * diff --git a/core/src/main/scala/kafka/migration/MigrationPropagator.scala b/core/src/main/scala/kafka/migration/MigrationPropagator.scala index 1a18ca42fcb0c..2a02f5891ecc6 100644 --- a/core/src/main/scala/kafka/migration/MigrationPropagator.scala +++ b/core/src/main/scala/kafka/migration/MigrationPropagator.scala @@ -22,7 +22,7 @@ import kafka.controller.{ControllerChannelContext, ControllerChannelManager, Rep import kafka.server.KafkaConfig import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.requests.LeaderAndIsrRequest +import org.apache.kafka.common.requests.AbstractControlRequest import org.apache.kafka.common.utils.Time import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, TopicsImage} import org.apache.kafka.metadata.PartitionRegistration @@ -138,6 +138,7 @@ class MigrationPropagator( } requestBatch.sendRequestsToBrokers(zkControllerEpoch) requestBatch.newBatch() + requestBatch.setUpdateType(AbstractControlRequest.Type.INCREMENTAL) // Now send LISR, UMR and StopReplica requests for both new zk brokers and existing zk // brokers based on the topic changes. @@ -226,7 +227,7 @@ class MigrationPropagator( requestBatch.sendRequestsToBrokers(zkControllerEpoch) requestBatch.newBatch() - requestBatch.setUpdateType(LeaderAndIsrRequest.Type.FULL) + requestBatch.setUpdateType(AbstractControlRequest.Type.FULL) // When we need to send RPCs from the image, we're sending 'full' requests meaning we let // every broker know about all the metadata and all the LISR requests it needs to handle. // Note that we cannot send StopReplica requests from the image. We don't have any state diff --git a/core/src/main/scala/kafka/network/RequestConvertToJson.scala b/core/src/main/scala/kafka/network/RequestConvertToJson.scala index 6844acf66cba7..4e4f89611ede9 100644 --- a/core/src/main/scala/kafka/network/RequestConvertToJson.scala +++ b/core/src/main/scala/kafka/network/RequestConvertToJson.scala @@ -32,15 +32,19 @@ object RequestConvertToJson { case req: AllocateProducerIdsRequest => AllocateProducerIdsRequestDataJsonConverter.write(req.data, request.version) case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version) case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version) - case req: AlterPartitionRequest => AlterPartitionRequestDataJsonConverter.write(req.data, request.version) case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version) + case req: AlterPartitionRequest => AlterPartitionRequestDataJsonConverter.write(req.data, request.version) case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version) case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version) case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version) + case req: AssignReplicasToDirsRequest => AssignReplicasToDirsRequestDataJsonConverter.write(req.data, request.version) case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version) case req: BrokerHeartbeatRequest => BrokerHeartbeatRequestDataJsonConverter.write(req.data, request.version) case req: BrokerRegistrationRequest => BrokerRegistrationRequestDataJsonConverter.write(req.data, request.version) + case req: ConsumerGroupDescribeRequest => ConsumerGroupDescribeRequestDataJsonConverter.write(req.data, request.version) + case req: ConsumerGroupHeartbeatRequest => ConsumerGroupHeartbeatRequestDataJsonConverter.write(req.data, request.version) case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version) + case req: ControllerRegistrationRequest => ControllerRegistrationRequestDataJsonConverter.write(req.data, request.version) case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version) case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version) case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version) @@ -51,18 +55,22 @@ object RequestConvertToJson { case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version) case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version) case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version) + case req: DescribeClusterRequest => DescribeClusterRequestDataJsonConverter.write(req.data, request.version) case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version) case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version) case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version) case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version) + case req: DescribeProducersRequest => DescribeProducersRequestDataJsonConverter.write(req.data, request.version) case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version) + case req: DescribeTransactionsRequest => DescribeTransactionsRequestDataJsonConverter.write(req.data, request.version) case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version) case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version) - case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version) case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version) + case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version) case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version) case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version) case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version) + case req: FetchSnapshotRequest => FetchSnapshotRequestDataJsonConverter.write(req.data, request.version) case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version) case req: GetTelemetrySubscriptionsRequest => GetTelemetrySubscriptionsRequestDataJsonConverter.write(req.data, request.version) case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version) @@ -71,9 +79,11 @@ object RequestConvertToJson { case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version) case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version) case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version) + case req: ListClientMetricsResourcesRequest => ListClientMetricsResourcesRequestDataJsonConverter.write(req.data, request.version) case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version) case req: ListOffsetsRequest => ListOffsetsRequestDataJsonConverter.write(req.data, request.version) case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version) + case req: ListTransactionsRequest => ListTransactionsRequestDataJsonConverter.write(req.data, request.version) case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version) case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version) case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version) @@ -92,15 +102,6 @@ object RequestConvertToJson { case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version) case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version) case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version) - case req: FetchSnapshotRequest => FetchSnapshotRequestDataJsonConverter.write(req.data, request.version) - case req: DescribeClusterRequest => DescribeClusterRequestDataJsonConverter.write(req.data, request.version) - case req: DescribeProducersRequest => DescribeProducersRequestDataJsonConverter.write(req.data, request.version) - case req: DescribeTransactionsRequest => DescribeTransactionsRequestDataJsonConverter.write(req.data, request.version) - case req: ListTransactionsRequest => ListTransactionsRequestDataJsonConverter.write(req.data, request.version) - case req: ConsumerGroupHeartbeatRequest => ConsumerGroupHeartbeatRequestDataJsonConverter.write(req.data, request.version) - case req: ConsumerGroupDescribeRequest => ConsumerGroupDescribeRequestDataJsonConverter.write(req.data, request.version) - case req: ControllerRegistrationRequest => ControllerRegistrationRequestDataJsonConverter.write(req.data, request.version) - case req: AssignReplicasToDirsRequest => AssignReplicasToDirsRequestDataJsonConverter.write(req.data, request.version) case _ => throw new IllegalStateException(s"ApiKey ${request.apiKey} is not currently handled in `request`, the " + "code should be updated to do so."); } @@ -113,15 +114,19 @@ object RequestConvertToJson { case res: AllocateProducerIdsResponse => AllocateProducerIdsResponseDataJsonConverter.write(res.data, version) case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version) case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version) - case res: AlterPartitionResponse => AlterPartitionResponseDataJsonConverter.write(res.data, version) case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version) + case res: AlterPartitionResponse => AlterPartitionResponseDataJsonConverter.write(res.data, version) case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version) case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version) case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version) + case res: AssignReplicasToDirsResponse => AssignReplicasToDirsResponseDataJsonConverter.write(res.data, version) case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version) case res: BrokerHeartbeatResponse => BrokerHeartbeatResponseDataJsonConverter.write(res.data, version) case res: BrokerRegistrationResponse => BrokerRegistrationResponseDataJsonConverter.write(res.data, version) + case res: ConsumerGroupDescribeResponse => ConsumerGroupDescribeResponseDataJsonConverter.write(res.data, version) + case res: ConsumerGroupHeartbeatResponse => ConsumerGroupHeartbeatResponseDataJsonConverter.write(res.data, version) case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version) + case req: ControllerRegistrationResponse => ControllerRegistrationResponseDataJsonConverter.write(req.data, version) case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version) case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version) case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version) @@ -132,18 +137,22 @@ object RequestConvertToJson { case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version) case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version) case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version) + case res: DescribeClusterResponse => DescribeClusterResponseDataJsonConverter.write(res.data, version) case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version) case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version) case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version) case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version) + case res: DescribeProducersResponse => DescribeProducersResponseDataJsonConverter.write(res.data, version) case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version) + case res: DescribeTransactionsResponse => DescribeTransactionsResponseDataJsonConverter.write(res.data, version) case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version) case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version) - case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version) case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version) + case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version) case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version) case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version) case res: FetchResponse => FetchResponseDataJsonConverter.write(res.data, version, false) + case res: FetchSnapshotResponse => FetchSnapshotResponseDataJsonConverter.write(res.data, version) case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version) case res: GetTelemetrySubscriptionsResponse => GetTelemetrySubscriptionsResponseDataJsonConverter.write(res.data, version) case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version) @@ -152,9 +161,11 @@ object RequestConvertToJson { case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version) case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version) case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version) + case res: ListClientMetricsResourcesResponse => ListClientMetricsResourcesResponseDataJsonConverter.write(res.data, version) case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version) case res: ListOffsetsResponse => ListOffsetsResponseDataJsonConverter.write(res.data, version) case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version) + case res: ListTransactionsResponse => ListTransactionsResponseDataJsonConverter.write(res.data, version) case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version) case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version) case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version) @@ -173,15 +184,6 @@ object RequestConvertToJson { case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version) case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version) case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version) - case res: FetchSnapshotResponse => FetchSnapshotResponseDataJsonConverter.write(res.data, version) - case res: DescribeClusterResponse => DescribeClusterResponseDataJsonConverter.write(res.data, version) - case res: DescribeProducersResponse => DescribeProducersResponseDataJsonConverter.write(res.data, version) - case res: DescribeTransactionsResponse => DescribeTransactionsResponseDataJsonConverter.write(res.data, version) - case res: ListTransactionsResponse => ListTransactionsResponseDataJsonConverter.write(res.data, version) - case res: ConsumerGroupHeartbeatResponse => ConsumerGroupHeartbeatResponseDataJsonConverter.write(res.data, version) - case res: ConsumerGroupDescribeResponse => ConsumerGroupDescribeResponseDataJsonConverter.write(res.data, version) - case req: ControllerRegistrationResponse => ControllerRegistrationResponseDataJsonConverter.write(req.data, version) - case res: AssignReplicasToDirsResponse => AssignReplicasToDirsResponseDataJsonConverter.write(res.data, version) case _ => throw new IllegalStateException(s"ApiKey ${response.apiKey} is not currently handled in `response`, the " + "code should be updated to do so."); } diff --git a/core/src/main/scala/kafka/server/AlterPartitionManager.scala b/core/src/main/scala/kafka/server/AlterPartitionManager.scala index 1c5021b69bfdb..661f8afe5f22f 100644 --- a/core/src/main/scala/kafka/server/AlterPartitionManager.scala +++ b/core/src/main/scala/kafka/server/AlterPartitionManager.scala @@ -34,6 +34,7 @@ import org.apache.kafka.common.requests.RequestHeader import org.apache.kafka.common.requests.{AlterPartitionRequest, AlterPartitionResponse} import org.apache.kafka.common.utils.Time import org.apache.kafka.metadata.LeaderRecoveryState +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.util.Scheduler @@ -84,7 +85,7 @@ object AlterPartitionManager { threadNamePrefix: String, brokerEpochSupplier: () => Long, ): AlterPartitionManager = { - val channelManager = NodeToControllerChannelManager( + val channelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider, time = time, metrics = metrics, diff --git a/core/src/main/scala/kafka/server/ApiVersionManager.scala b/core/src/main/scala/kafka/server/ApiVersionManager.scala index fa796bc668853..6e1057469df83 100644 --- a/core/src/main/scala/kafka/server/ApiVersionManager.scala +++ b/core/src/main/scala/kafka/server/ApiVersionManager.scala @@ -22,6 +22,7 @@ import org.apache.kafka.common.feature.SupportedVersionRange import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.ApiVersionsResponse +import org.apache.kafka.server.ClientMetricsManager import org.apache.kafka.server.common.Features import scala.jdk.CollectionConverters._ @@ -47,7 +48,8 @@ object ApiVersionManager { config: KafkaConfig, forwardingManager: Option[ForwardingManager], supportedFeatures: BrokerFeatures, - metadataCache: MetadataCache + metadataCache: MetadataCache, + clientMetricsManager: Option[ClientMetricsManager] ): ApiVersionManager = { new DefaultApiVersionManager( listenerType, @@ -55,7 +57,8 @@ object ApiVersionManager { supportedFeatures, metadataCache, config.unstableApiVersionsEnabled, - config.migrationEnabled + config.migrationEnabled, + clientMetricsManager ) } } @@ -123,6 +126,7 @@ class SimpleApiVersionManager( * @param metadataCache the metadata cache, used to get the finalized features and the metadata version * @param enableUnstableLastVersion whether to enable unstable last version, see [[KafkaConfig.unstableApiVersionsEnabled]] * @param zkMigrationEnabled whether to enable zk migration, see [[KafkaConfig.migrationEnabled]] + * @param clientMetricsManager the client metrics manager, helps to determine whether client telemetry is enabled */ class DefaultApiVersionManager( val listenerType: ListenerType, @@ -130,7 +134,8 @@ class DefaultApiVersionManager( brokerFeatures: BrokerFeatures, metadataCache: MetadataCache, val enableUnstableLastVersion: Boolean, - val zkMigrationEnabled: Boolean = false + val zkMigrationEnabled: Boolean = false, + val clientMetricsManager: Option[ClientMetricsManager] = None ) extends ApiVersionManager { val enabledApis = ApiKeys.apisForListener(listenerType).asScala @@ -139,6 +144,10 @@ class DefaultApiVersionManager( val supportedFeatures = brokerFeatures.supportedFeatures val finalizedFeatures = metadataCache.features() val controllerApiVersions = forwardingManager.flatMap(_.controllerApiVersions) + val clientTelemetryEnabled = clientMetricsManager match { + case Some(manager) => manager.isTelemetryReceiverConfigured + case None => false + } ApiVersionsResponse.createApiVersionsResponse( throttleTimeMs, @@ -149,7 +158,8 @@ class DefaultApiVersionManager( controllerApiVersions.orNull, listenerType, enableUnstableLastVersion, - zkMigrationEnabled + zkMigrationEnabled, + clientTelemetryEnabled ) } diff --git a/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala b/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala index 25e934d888cfe..a9b06dcbc51ef 100644 --- a/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala +++ b/core/src/main/scala/kafka/server/AutoTopicCreationManager.scala @@ -20,7 +20,6 @@ package kafka.server import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference import java.util.{Collections, Properties} - import kafka.controller.KafkaController import kafka.coordinator.transaction.TransactionCoordinator import kafka.utils.Logging @@ -34,8 +33,10 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopi import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{ApiError, CreateTopicsRequest, RequestContext, RequestHeader} import org.apache.kafka.coordinator.group.GroupCoordinator +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import scala.collection.{Map, Seq, Set, mutable} +import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ trait AutoTopicCreationManager { @@ -193,7 +194,7 @@ class DefaultAutoTopicCreationManager( val request = metadataRequestContext.map { context => val requestVersion = - channelManager.controllerApiVersions() match { + channelManager.controllerApiVersions.asScala match { case None => // We will rely on the Metadata request to be retried in the case // that the latest version is not usable by the controller. diff --git a/core/src/main/scala/kafka/server/BrokerLifecycleManager.scala b/core/src/main/scala/kafka/server/BrokerLifecycleManager.scala index bfe5ca0fde726..fd2c2cc8e4573 100644 --- a/core/src/main/scala/kafka/server/BrokerLifecycleManager.scala +++ b/core/src/main/scala/kafka/server/BrokerLifecycleManager.scala @@ -30,6 +30,7 @@ import org.apache.kafka.metadata.{BrokerState, VersionRange} import org.apache.kafka.queue.EventQueue.DeadlineFunction import org.apache.kafka.common.utils.{ExponentialBackoff, LogContext, Time} import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import java.util.{Comparator, OptionalLong} import scala.jdk.CollectionConverters._ @@ -148,10 +149,10 @@ class BrokerLifecycleManager( private var readyToUnfence = false /** - * List of offline directories pending to be sent. + * List of accumulated offline directories. * This variable can only be read or written from the event queue thread. */ - private var offlineDirsPending = Set[Uuid]() + private var offlineDirs = Set[Uuid]() /** * True if we sent a event queue to the active controller requesting controlled @@ -299,10 +300,10 @@ class BrokerLifecycleManager( private class OfflineDirEvent(val dir: Uuid) extends EventQueue.Event { override def run(): Unit = { - if (offlineDirsPending.isEmpty) { - offlineDirsPending = Set(dir) + if (offlineDirs.isEmpty) { + offlineDirs = Set(dir) } else { - offlineDirsPending = offlineDirsPending + dir + offlineDirs = offlineDirs + dir } if (registered) { scheduleNextCommunicationImmediately() @@ -423,15 +424,15 @@ class BrokerLifecycleManager( setCurrentMetadataOffset(metadataOffset). setWantFence(!readyToUnfence). setWantShutDown(_state == BrokerState.PENDING_CONTROLLED_SHUTDOWN). - setOfflineLogDirs(offlineDirsPending.toSeq.asJava) + setOfflineLogDirs(offlineDirs.toSeq.asJava) if (isTraceEnabled) { trace(s"Sending broker heartbeat $data") } - val handler = new BrokerHeartbeatResponseHandler(offlineDirsPending) + val handler = new BrokerHeartbeatResponseHandler() _channelManager.sendRequest(new BrokerHeartbeatRequest.Builder(data), handler) } - private class BrokerHeartbeatResponseHandler(dirsInFlight: Set[Uuid]) extends ControllerRequestCompletionHandler { + private class BrokerHeartbeatResponseHandler() extends ControllerRequestCompletionHandler { override def onComplete(response: ClientResponse): Unit = { if (response.authenticationException() != null) { error(s"Unable to send broker heartbeat for $nodeId because of an " + @@ -455,7 +456,7 @@ class BrokerLifecycleManager( // this response handler is not invoked from the event handler thread, // and processing a successful heartbeat response requires updating // state, so to continue we need to schedule an event - eventQueue.prepend(new BrokerHeartbeatResponseEvent(message.data(), dirsInFlight)) + eventQueue.prepend(new BrokerHeartbeatResponseEvent(message.data())) } else { warn(s"Broker $nodeId sent a heartbeat request but received error $errorCode.") scheduleNextCommunicationAfterFailure() @@ -469,10 +470,9 @@ class BrokerLifecycleManager( } } - private class BrokerHeartbeatResponseEvent(response: BrokerHeartbeatResponseData, dirsInFlight: Set[Uuid]) extends EventQueue.Event { + private class BrokerHeartbeatResponseEvent(response: BrokerHeartbeatResponseData) extends EventQueue.Event { override def run(): Unit = { failedAttempts = 0 - offlineDirsPending = offlineDirsPending.diff(dirsInFlight) _state match { case BrokerState.STARTING => if (response.isCaughtUp) { diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index e34fe4c89c9f5..836818ef8b485 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -34,18 +34,19 @@ import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache import org.apache.kafka.common.utils.{LogContext, Time} -import org.apache.kafka.common.{ClusterResource, KafkaException, TopicPartition} +import org.apache.kafka.common.{ClusterResource, KafkaException, TopicPartition, Uuid} import org.apache.kafka.coordinator.group -import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorRuntimeMetrics +import org.apache.kafka.coordinator.group.metrics.{GroupCoordinatorMetrics, GroupCoordinatorRuntimeMetrics} import org.apache.kafka.coordinator.group.util.SystemTimerReaper import org.apache.kafka.coordinator.group.{GroupCoordinator, GroupCoordinatorConfig, GroupCoordinatorService, RecordSerde} import org.apache.kafka.image.publisher.MetadataPublisher import org.apache.kafka.metadata.{BrokerState, ListenerInfo, VersionRange} import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.{AssignmentsManager, ClientMetricsManager, NodeToControllerChannelManager} import org.apache.kafka.server.authorizer.Authorizer -import org.apache.kafka.server.common.ApiMessageAndVersion +import org.apache.kafka.server.common.{ApiMessageAndVersion, DirectoryEventHandler, TopicIdPartition} import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig -import org.apache.kafka.server.metrics.KafkaYammerMetrics +import org.apache.kafka.server.metrics.{ClientMetricsReceiverPlugin, KafkaYammerMetrics} import org.apache.kafka.server.network.{EndpointReadyFutures, KafkaAuthorizerServerInfo} import org.apache.kafka.server.util.timer.SystemTimer import org.apache.kafka.server.util.{Deadline, FutureUtils, KafkaScheduler} @@ -85,6 +86,8 @@ class BrokerServer( @volatile var lifecycleManager: BrokerLifecycleManager = _ + var assignmentsManager: AssignmentsManager = _ + private val isShuttingDown = new AtomicBoolean(false) val lock = new ReentrantLock() @@ -174,7 +177,8 @@ class BrokerServer( info("Starting broker") - config.dynamicConfig.initialize(zkClientOpt = None) + val clientMetricsReceiverPlugin = new ClientMetricsReceiverPlugin() + config.dynamicConfig.initialize(zkClientOpt = None, Some(clientMetricsReceiverPlugin)) /* start scheduler */ kafkaScheduler = new KafkaScheduler(config.backgroundThreads) @@ -220,7 +224,7 @@ class BrokerServer( val controllerNodes = RaftConfig.voterConnectionsToNodes(voterConnections).asScala val controllerNodeProvider = RaftControllerNodeProvider(raftManager, config, controllerNodes) - clientToControllerChannelManager = NodeToControllerChannelManager( + clientToControllerChannelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider, time, metrics, @@ -231,14 +235,15 @@ class BrokerServer( ) clientToControllerChannelManager.start() forwardingManager = new ForwardingManagerImpl(clientToControllerChannelManager) - + clientMetricsManager = new ClientMetricsManager(clientMetricsReceiverPlugin, config.clientTelemetryMaxBytes, time) val apiVersionManager = ApiVersionManager( ListenerType.BROKER, config, Some(forwardingManager), brokerFeatures, - metadataCache + metadataCache, + Some(clientMetricsManager) ) // Create and start the socket server acceptor threads so that the bound port is known. @@ -277,6 +282,28 @@ class BrokerServer( time ) + val assignmentsChannelManager = new NodeToControllerChannelManagerImpl( + controllerNodeProvider, + time, + metrics, + config, + "directory-assignments", + s"broker-${config.nodeId}-", + retryTimeoutMs = 60000 + ) + assignmentsManager = new AssignmentsManager( + time, + assignmentsChannelManager, + config.brokerId, + () => lifecycleManager.brokerEpoch + ) + val directoryEventHandler = new DirectoryEventHandler { + override def handleAssignment(partition: TopicIdPartition, directoryId: Uuid): Unit = + assignmentsManager.onAssignment(partition, directoryId) + override def handleFailure(directoryId: Uuid): Unit = + lifecycleManager.propagateDirectoryFailure(directoryId) + } + this._replicaManager = new ReplicaManager( config = config, metrics = metrics, @@ -294,7 +321,8 @@ class BrokerServer( threadNamePrefix = None, // The ReplicaManager only runs on the broker, and already includes the ID in thread names. delayedRemoteFetchPurgatoryParam = None, brokerEpochSupplier = () => lifecycleManager.brokerEpoch, - addPartitionsToTxnManager = Some(addPartitionsToTxnManager) + addPartitionsToTxnManager = Some(addPartitionsToTxnManager), + directoryEventHandler = directoryEventHandler ) /* start token manager */ @@ -320,8 +348,6 @@ class BrokerServer( config, Some(clientToControllerChannelManager), None, None, groupCoordinator, transactionCoordinator) - clientMetricsManager = ClientMetricsManager.instance() - dynamicConfigHandlers = Map[String, ConfigHandler]( ConfigType.Topic -> new TopicConfigHandler(replicaManager, config, quotaManagers, None), ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers), @@ -332,7 +358,7 @@ class BrokerServer( k -> VersionRange.of(v.min, v.max) }.asJava - val brokerLifecycleChannelManager = NodeToControllerChannelManager( + val brokerLifecycleChannelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider, time, metrics, @@ -562,6 +588,7 @@ class BrokerServer( .withLoader(loader) .withWriter(writer) .withCoordinatorRuntimeMetrics(new GroupCoordinatorRuntimeMetrics(metrics)) + .withGroupCoordinatorMetrics(new GroupCoordinatorMetrics(KafkaYammerMetrics.defaultRegistry, metrics)) .build() } else { GroupCoordinatorAdapter( @@ -648,6 +675,9 @@ class BrokerServer( if (tokenManager != null) CoreUtils.swallow(tokenManager.shutdown(), this) + if (assignmentsManager != null) + CoreUtils.swallow(assignmentsManager.close(), this) + if (replicaManager != null) CoreUtils.swallow(replicaManager.shutdown(), this) diff --git a/core/src/main/scala/kafka/server/ConfigHandler.scala b/core/src/main/scala/kafka/server/ConfigHandler.scala index a770e17f0c071..5a2c11cb91c35 100644 --- a/core/src/main/scala/kafka/server/ConfigHandler.scala +++ b/core/src/main/scala/kafka/server/ConfigHandler.scala @@ -33,6 +33,7 @@ import org.apache.kafka.common.config.internals.QuotaConfigs import org.apache.kafka.common.metrics.Quota import org.apache.kafka.common.metrics.Quota._ import org.apache.kafka.common.utils.Sanitizer +import org.apache.kafka.server.ClientMetricsManager import org.apache.kafka.storage.internals.log.{LogConfig, ThrottledReplicaListValidator} import org.apache.kafka.storage.internals.log.LogConfig.MessageFormatVersion diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index 7278bf417ade4..aef1924b89a5c 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -205,16 +205,14 @@ class ControllerApis( names => authHelper.filterByAuthorized(request.context, DESCRIBE, TOPIC, names)(n => n), names => authHelper.filterByAuthorized(request.context, DELETE, TOPIC, names)(n => n)) future.handle[Unit] { (results, exception) => - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, throttleTimeMs => { - if (exception != null) { - deleteTopicsRequest.getErrorResponse(throttleTimeMs, exception) - } else { - val responseData = new DeleteTopicsResponseData(). - setResponses(new DeletableTopicResultCollection(results.iterator)). - setThrottleTimeMs(throttleTimeMs) - new DeleteTopicsResponse(responseData) - } - }) + val response = if (exception != null) { + deleteTopicsRequest.getErrorResponse(exception) + } else { + val responseData = new DeleteTopicsResponseData() + .setResponses(new DeletableTopicResultCollection(results.iterator)) + new DeleteTopicsResponse(responseData) + } + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } } @@ -371,14 +369,12 @@ class ControllerApis( names => authHelper.filterByAuthorized(request.context, DESCRIBE_CONFIGS, TOPIC, names, logIfDenied = false)(identity)) future.handle[Unit] { (result, exception) => - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, throttleTimeMs => { - if (exception != null) { - createTopicsRequest.getErrorResponse(throttleTimeMs, exception) - } else { - result.setThrottleTimeMs(throttleTimeMs) - new CreateTopicsResponse(result) - } - }) + val response = if (exception != null) { + createTopicsRequest.getErrorResponse(exception) + } else { + new CreateTopicsResponse(result) + } + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } } @@ -802,16 +798,13 @@ class ControllerApis( createPartitionsRequest.data(), filterAlterAuthorizedTopics) future.handle[Unit] { (responses, exception) => - if (exception != null) { - requestHelper.handleError(request, exception) + val response = if (exception != null) { + createPartitionsRequest.getErrorResponse(exception) } else { - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, requestThrottleMs => { - val responseData = new CreatePartitionsResponseData(). - setResults(responses). - setThrottleTimeMs(requestThrottleMs) - new CreatePartitionsResponse(responseData) - }) + val responseData = new CreatePartitionsResponseData().setResults(responses) + new CreatePartitionsResponse(responseData) } + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } } diff --git a/core/src/main/scala/kafka/server/ControllerConfigurationValidator.scala b/core/src/main/scala/kafka/server/ControllerConfigurationValidator.scala index 73a609ee7ebac..28ddac0c5e6b9 100644 --- a/core/src/main/scala/kafka/server/ControllerConfigurationValidator.scala +++ b/core/src/main/scala/kafka/server/ControllerConfigurationValidator.scala @@ -17,8 +17,6 @@ package kafka.server -import kafka.metrics.ClientMetricsConfigs - import java.util import java.util.Properties import org.apache.kafka.common.config.ConfigResource @@ -26,6 +24,7 @@ import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, CLIENT_METRIC import org.apache.kafka.controller.ConfigurationValidator import org.apache.kafka.common.errors.{InvalidConfigurationException, InvalidRequestException} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.server.metrics.ClientMetricsConfigs import org.apache.kafka.storage.internals.log.LogConfig import scala.collection.mutable diff --git a/core/src/main/scala/kafka/server/ControllerRegistrationManager.scala b/core/src/main/scala/kafka/server/ControllerRegistrationManager.scala index 6cf42690384d5..20edae23ca829 100644 --- a/core/src/main/scala/kafka/server/ControllerRegistrationManager.scala +++ b/core/src/main/scala/kafka/server/ControllerRegistrationManager.scala @@ -31,6 +31,7 @@ import org.apache.kafka.image.{MetadataDelta, MetadataImage} import org.apache.kafka.image.publisher.MetadataPublisher import org.apache.kafka.queue.EventQueue.DeadlineFunction import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.MetadataVersion import scala.jdk.CollectionConverters._ diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 7b28c3372e1ed..db386e4a92e60 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -44,6 +44,7 @@ import org.apache.kafka.metadata.bootstrap.BootstrapMetadata import org.apache.kafka.metadata.migration.{KRaftMigrationDriver, LegacyPropagator} import org.apache.kafka.metadata.publisher.FeaturesPublisher import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.NodeToControllerChannelManager import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.server.metrics.{KafkaMetricsGroup, KafkaYammerMetrics} @@ -83,7 +84,7 @@ case class ControllerMigrationSupport( class ControllerServer( val sharedServer: SharedServer, val configSchema: KafkaConfigSchema, - val bootstrapMetadata: BootstrapMetadata, + val bootstrapMetadata: BootstrapMetadata ) extends Logging { import kafka.server.Server._ @@ -124,7 +125,6 @@ class ControllerServer( @volatile var incarnationId: Uuid = _ @volatile var registrationManager: ControllerRegistrationManager = _ @volatile var registrationChannelManager: NodeToControllerChannelManager = _ - var clientMetricsManager: ClientMetricsManager = _ private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { lock.lock() @@ -146,7 +146,7 @@ class ControllerServer( try { this.logIdent = logContext.logPrefix() info("Starting controller") - config.dynamicConfig.initialize(zkClientOpt = None) + config.dynamicConfig.initialize(zkClientOpt = None, clientMetricsReceiverPluginOpt = None) maybeChangeStatus(STARTING, STARTED) @@ -216,7 +216,7 @@ class ControllerServer( startupDeadline, time) val controllerNodes = RaftConfig.voterConnectionsToNodes(voterConnections) val quorumFeatures = new QuorumFeatures(config.nodeId, - QuorumFeatures.defaultFeatureMap(), + QuorumFeatures.defaultFeatureMap(config.unstableMetadataVersionsEnabled), controllerNodes.asScala.map(node => Integer.valueOf(node.id())).asJava) val delegationTokenKeyString = { @@ -333,8 +333,6 @@ class ControllerServer( DataPlaneAcceptor.ThreadPrefix, "controller") - clientMetricsManager = ClientMetricsManager.instance() - // Set up the metadata cache publisher. metadataPublishers.add(metadataCachePublisher) @@ -349,7 +347,7 @@ class ControllerServer( clusterId, time, s"controller-${config.nodeId}-", - QuorumFeatures.defaultFeatureMap(), + QuorumFeatures.defaultFeatureMap(config.unstableMetadataVersionsEnabled), config.migrationEnabled, incarnationId, listenerInfo) @@ -365,8 +363,7 @@ class ControllerServer( sharedServer.metadataPublishingFaultHandler, immutable.Map[String, ConfigHandler]( // controllers don't host topics, so no need to do anything with dynamic topic config changes here - ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers), - ConfigType.ClientMetrics -> new ClientMetricsConfigHandler(clientMetricsManager) + ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers) ), "controller")) @@ -432,7 +429,7 @@ class ControllerServer( * Start the KIP-919 controller registration manager. */ val controllerNodeProvider = RaftControllerNodeProvider(raftManager, config, controllerNodes.asScala) - registrationChannelManager = NodeToControllerChannelManager( + registrationChannelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider, time, metrics, diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index eac519b1dece8..2acbee89756c7 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -38,6 +38,8 @@ import org.apache.kafka.common.security.authenticator.LoginManager import org.apache.kafka.common.utils.{ConfigUtils, Utils} import org.apache.kafka.server.config.ServerTopicConfigSynonyms import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig +import org.apache.kafka.server.metrics.ClientMetricsReceiverPlugin +import org.apache.kafka.server.telemetry.ClientTelemetry import org.apache.kafka.storage.internals.log.{LogConfig, ProducerStateManagerConfig} import scala.annotation.nowarn @@ -216,6 +218,7 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging private[server] val reconfigurables = new CopyOnWriteArrayList[Reconfigurable]() private val brokerReconfigurables = new CopyOnWriteArrayList[BrokerReconfigurable]() private val lock = new ReentrantReadWriteLock + private var metricsReceiverPluginOpt: Option[ClientMetricsReceiverPlugin] = _ private var currentConfig: KafkaConfig = _ private val dynamicConfigPasswordEncoder = if (kafkaConfig.processRoles.isEmpty) { maybeCreatePasswordEncoder(kafkaConfig.passwordEncoderSecret) @@ -223,8 +226,9 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging Some(PasswordEncoder.noop()) } - private[server] def initialize(zkClientOpt: Option[KafkaZkClient]): Unit = { + private[server] def initialize(zkClientOpt: Option[KafkaZkClient], clientMetricsReceiverPluginOpt: Option[ClientMetricsReceiverPlugin]): Unit = { currentConfig = new KafkaConfig(kafkaConfig.props, false, None) + metricsReceiverPluginOpt = clientMetricsReceiverPluginOpt zkClientOpt.foreach { zkClient => val adminZkClient = new AdminZkClient(zkClient) @@ -327,6 +331,10 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging dynamicDefaultConfigs.clone() } + private[server] def clientMetricsReceiverPlugin: Option[ClientMetricsReceiverPlugin] = CoreUtils.inReadLock(lock) { + metricsReceiverPluginOpt + } + private[server] def updateBrokerConfig(brokerId: Int, persistentProps: Properties, doLog: Boolean = true): Unit = CoreUtils.inWriteLock(lock) { try { val props = fromPersistentProps(persistentProps, perBrokerConfig = true) @@ -913,6 +921,19 @@ class DynamicMetricReporterState(brokerId: Int, config: KafkaConfig, metrics: Me reporters.forEach { reporter => metrics.addReporter(reporter) currentReporters += reporter.getClass.getName -> reporter + val clientTelemetryReceiver = reporter match { + case telemetry: ClientTelemetry => telemetry.clientReceiver() + case _ => null + } + + if (clientTelemetryReceiver != null) { + dynamicConfig.clientMetricsReceiverPlugin match { + case Some(receiverPlugin) => + receiverPlugin.add(clientTelemetryReceiver) + case None => + // Do nothing + } + } } KafkaBroker.notifyClusterListeners(clusterId, reporters.asScala) } diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala index 8af2dece0421d..d91a06d8bd3c6 100644 --- a/core/src/main/scala/kafka/server/DynamicConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicConfig.scala @@ -25,6 +25,7 @@ import org.apache.kafka.common.config.ConfigDef.Range._ import org.apache.kafka.common.config.ConfigDef.Type._ import org.apache.kafka.storage.internals.log.LogConfig +import java.util import scala.jdk.CollectionConverters._ /** @@ -111,6 +112,12 @@ object DynamicConfig { } } + object ClientMetrics { + private val clientConfigs = org.apache.kafka.server.metrics.ClientMetricsConfigs.configDef() + + def names: util.Set[String] = clientConfigs.names + } + private def validate(configDef: ConfigDef, props: Properties, customPropsAllowed: Boolean) = { // Validate Names val names = configDef.names() diff --git a/core/src/main/scala/kafka/server/ForwardingManager.scala b/core/src/main/scala/kafka/server/ForwardingManager.scala index b0b13dfecdaa0..7d7b6eba02daa 100644 --- a/core/src/main/scala/kafka/server/ForwardingManager.scala +++ b/core/src/main/scala/kafka/server/ForwardingManager.scala @@ -18,13 +18,13 @@ package kafka.server import java.nio.ByteBuffer - import kafka.network.RequestChannel import kafka.utils.Logging import org.apache.kafka.clients.{ClientResponse, NodeApiVersions} import org.apache.kafka.common.errors.TimeoutException import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, EnvelopeRequest, EnvelopeResponse, RequestContext, RequestHeader} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import scala.compat.java8.OptionConverters._ @@ -165,7 +165,7 @@ class ForwardingManagerImpl( } override def controllerApiVersions: Option[NodeApiVersions] = - channelManager.controllerApiVersions() + channelManager.controllerApiVersions.asScala private def parseResponse( buffer: ByteBuffer, diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index bd0959d40d2f8..ec79d7fc85210 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -67,6 +67,7 @@ import org.apache.kafka.common.security.token.delegation.{DelegationToken, Token import org.apache.kafka.common.utils.{ProducerIdAndEpoch, Time} import org.apache.kafka.common.{Node, TopicIdPartition, TopicPartition, Uuid} import org.apache.kafka.coordinator.group.GroupCoordinator +import org.apache.kafka.server.ClientMetricsManager import org.apache.kafka.server.authorizer._ import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.common.MetadataVersion.{IBP_0_11_0_IV0, IBP_2_3_IV0} @@ -110,7 +111,7 @@ class KafkaApis(val requestChannel: RequestChannel, val clientMetricsManager: Option[ClientMetricsManager] ) extends ApiRequestHandler with Logging { - type FetchResponseStats = Map[TopicPartition, RecordConversionStats] + type FetchResponseStats = Map[TopicPartition, RecordValidationStats] this.logIdent = "[KafkaApi-%d] ".format(brokerId) val configHelper = new ConfigHelper(metadataCache, config, configRepository) val authHelper = new AuthHelper(authorizer) @@ -246,6 +247,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.CONSUMER_GROUP_DESCRIBE => handleConsumerGroupDescribe(request).exceptionally(handleError) case ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS => handleGetTelemetrySubscriptionsRequest(request) case ApiKeys.PUSH_TELEMETRY => handlePushTelemetryRequest(request) + case ApiKeys.LIST_CLIENT_METRICS_RESOURCES => handleListClientMetricsResources(request) case _ => throw new IllegalStateException(s"No handler for request api key ${request.header.apiKey}") } } catch { @@ -562,6 +564,23 @@ class KafkaApis(val requestChannel: RequestChannel, } } + case class LeaderNode(leaderId: Int, leaderEpoch: Int, node: Option[Node]) + + private def getCurrentLeader(tp: TopicPartition, ln: ListenerName): LeaderNode = { + val partitionInfoOrError = replicaManager.getPartitionOrError(tp) + val (leaderId, leaderEpoch) = partitionInfoOrError match { + case Right(x) => + (x.leaderReplicaIdOpt.getOrElse(-1), x.getLeaderEpoch) + case Left(x) => + debug(s"Unable to retrieve local leaderId and Epoch with error $x, falling back to metadata cache") + metadataCache.getPartitionInfo(tp.topic, tp.partition) match { + case Some(pinfo) => (pinfo.leader(), pinfo.leaderEpoch()) + case None => (-1, -1) + } + } + LeaderNode(leaderId, leaderEpoch, metadataCache.getAliveBrokerNode(leaderId, ln)) + } + /** * Handle a produce request */ @@ -614,6 +633,7 @@ class KafkaApis(val requestChannel: RequestChannel, val mergedResponseStatus = responseStatus ++ unauthorizedTopicResponses ++ nonExistingTopicResponses ++ invalidRequestResponses var errorInResponse = false + val nodeEndpoints = new mutable.HashMap[Int, Node] mergedResponseStatus.forKeyValue { (topicPartition, status) => if (status.error != Errors.NONE) { errorInResponse = true @@ -622,6 +642,20 @@ class KafkaApis(val requestChannel: RequestChannel, request.header.clientId, topicPartition, status.error.exceptionName)) + + if (request.header.apiVersion >= 10) { + status.error match { + case Errors.NOT_LEADER_OR_FOLLOWER => + val leaderNode = getCurrentLeader(topicPartition, request.context.listenerName) + leaderNode.node.foreach { node => + nodeEndpoints.put(node.id(), node) + } + status.currentLeader + .setLeaderId(leaderNode.leaderId) + .setLeaderEpoch(leaderNode.leaderEpoch) + case _ => + } + } } } @@ -665,7 +699,7 @@ class KafkaApis(val requestChannel: RequestChannel, requestHelper.sendNoOpResponseExemptThrottle(request) } } else { - requestChannel.sendResponse(request, new ProduceResponse(mergedResponseStatus.asJava, maxThrottleTimeMs), None) + requestChannel.sendResponse(request, new ProduceResponse(mergedResponseStatus.asJava, maxThrottleTimeMs, nodeEndpoints.values.toList.asJava), None) } } @@ -689,7 +723,7 @@ class KafkaApis(val requestChannel: RequestChannel, entriesPerPartition = authorizedRequestInfo, requestLocal = requestLocal, responseCallback = sendResponseCallback, - recordConversionStatsCallback = processingStatsCallback, + recordValidationStatsCallback = processingStatsCallback, transactionalId = produceRequest.transactionalId() ) @@ -843,6 +877,7 @@ class KafkaApis(val requestChannel: RequestChannel, .setRecords(unconvertedRecords) .setPreferredReadReplica(partitionData.preferredReadReplica) .setDivergingEpoch(partitionData.divergingEpoch) + .setCurrentLeader(partitionData.currentLeader()) } } } @@ -851,6 +886,7 @@ class KafkaApis(val requestChannel: RequestChannel, def processResponseCallback(responsePartitionData: Seq[(TopicIdPartition, FetchPartitionData)]): Unit = { val partitions = new util.LinkedHashMap[TopicIdPartition, FetchResponseData.PartitionData] val reassigningPartitions = mutable.Set[TopicIdPartition]() + val nodeEndpoints = new mutable.HashMap[Int, Node] responsePartitionData.foreach { case (tp, data) => val abortedTransactions = data.abortedTransactions.orElse(null) val lastStableOffset: Long = data.lastStableOffset.orElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) @@ -864,6 +900,21 @@ class KafkaApis(val requestChannel: RequestChannel, .setAbortedTransactions(abortedTransactions) .setRecords(data.records) .setPreferredReadReplica(data.preferredReadReplica.orElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID)) + + if (versionId >= 16) { + data.error match { + case Errors.NOT_LEADER_OR_FOLLOWER | Errors.FENCED_LEADER_EPOCH => + val leaderNode = getCurrentLeader(tp.topicPartition(), request.context.listenerName) + leaderNode.node.foreach { node => + nodeEndpoints.put(node.id(), node) + } + partitionData.currentLeader() + .setLeaderId(leaderNode.leaderId) + .setLeaderEpoch(leaderNode.leaderEpoch) + case _ => + } + } + data.divergingEpoch.ifPresent(partitionData.setDivergingEpoch(_)) partitions.put(tp, partitionData) } @@ -887,7 +938,7 @@ class KafkaApis(val requestChannel: RequestChannel, // Prepare fetch response from converted data val response = - FetchResponse.of(unconvertedFetchResponse.error, throttleTimeMs, unconvertedFetchResponse.sessionId, convertedData) + FetchResponse.of(unconvertedFetchResponse.error, throttleTimeMs, unconvertedFetchResponse.sessionId, convertedData, nodeEndpoints.values.toList.asJava) // record the bytes out metrics only when the response is being sent response.data.responses.forEach { topicResponse => topicResponse.partitions.forEach { data => @@ -1891,16 +1942,12 @@ class KafkaApis(val requestChannel: RequestChannel, val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request, strictSinceVersion = 6) def sendResponseCallback(results: CreatableTopicResultCollection): Unit = { - def createResponse(requestThrottleMs: Int): AbstractResponse = { - val responseData = new CreateTopicsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setTopics(results) - val responseBody = new CreateTopicsResponse(responseData) - trace(s"Sending create topics response $responseData for correlation id " + - s"${request.header.correlationId} to client ${request.header.clientId}.") - responseBody - } - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, createResponse) + val responseData = new CreateTopicsResponseData() + .setTopics(results) + val response = new CreateTopicsResponse(responseData) + trace(s"Sending create topics response $responseData for correlation id " + + s"${request.header.correlationId} to client ${request.header.clientId}.") + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } val createTopicsRequest = request.body[CreateTopicsRequest] @@ -1997,21 +2044,17 @@ class KafkaApis(val requestChannel: RequestChannel, val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request, strictSinceVersion = 3) def sendResponseCallback(results: Map[String, ApiError]): Unit = { - def createResponse(requestThrottleMs: Int): AbstractResponse = { - val createPartitionsResults = results.map { - case (topic, error) => new CreatePartitionsTopicResult() - .setName(topic) - .setErrorCode(error.error.code) - .setErrorMessage(error.message) - }.toSeq - val responseBody = new CreatePartitionsResponse(new CreatePartitionsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setResults(createPartitionsResults.asJava)) - trace(s"Sending create partitions response $responseBody for correlation id ${request.header.correlationId} to " + - s"client ${request.header.clientId}.") - responseBody - } - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, createResponse) + val createPartitionsResults = results.map { + case (topic, error) => new CreatePartitionsTopicResult() + .setName(topic) + .setErrorCode(error.error.code) + .setErrorMessage(error.message) + }.toSeq + val response = new CreatePartitionsResponse(new CreatePartitionsResponseData() + .setResults(createPartitionsResults.asJava)) + trace(s"Sending create partitions response $response for correlation id ${request.header.correlationId} to " + + s"client ${request.header.clientId}.") + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } if (!zkSupport.controller.isActive) { @@ -2051,15 +2094,11 @@ class KafkaApis(val requestChannel: RequestChannel, val controllerMutationQuota = quotas.controllerMutation.newQuotaFor(request, strictSinceVersion = 5) def sendResponseCallback(results: DeletableTopicResultCollection): Unit = { - def createResponse(requestThrottleMs: Int): AbstractResponse = { - val responseData = new DeleteTopicsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setResponses(results) - val responseBody = new DeleteTopicsResponse(responseData) - trace(s"Sending delete topics response $responseBody for correlation id ${request.header.correlationId} to client ${request.header.clientId}.") - responseBody - } - requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, createResponse) + val responseData = new DeleteTopicsResponseData() + .setResponses(results) + val response = new DeleteTopicsResponse(responseData) + trace(s"Sending delete topics response $response for correlation id ${request.header.correlationId} to client ${request.header.clientId}.") + requestHelper.sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota, request, response) } val deleteTopicRequest = request.body[DeleteTopicsRequest] @@ -3693,25 +3732,100 @@ class KafkaApis(val requestChannel: RequestChannel, } def handleConsumerGroupDescribe(request: RequestChannel.Request): CompletableFuture[Unit] = { - requestHelper.sendMaybeThrottle(request, request.body[ConsumerGroupDescribeRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) - CompletableFuture.completedFuture[Unit](()) + val consumerGroupDescribeRequest = request.body[ConsumerGroupDescribeRequest] + + if (!config.isNewGroupCoordinatorEnabled) { + // The API is not supported by the "old" group coordinator (the default). If the + // new one is not enabled, we fail directly here. + requestHelper.sendMaybeThrottle(request, request.body[ConsumerGroupDescribeRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) + CompletableFuture.completedFuture[Unit](()) + } else { + val response = new ConsumerGroupDescribeResponseData() + + val authorizedGroups = new ArrayBuffer[String]() + consumerGroupDescribeRequest.data.groupIds.forEach { groupId => + if (!authHelper.authorize(request.context, DESCRIBE, GROUP, groupId)) { + response.groups.add(new ConsumerGroupDescribeResponseData.DescribedGroup() + .setGroupId(groupId) + .setErrorCode(Errors.GROUP_AUTHORIZATION_FAILED.code) + ) + } else { + authorizedGroups += groupId + } + } + + groupCoordinator.consumerGroupDescribe( + request.context, + authorizedGroups.asJava + ).handle[Unit] { (results, exception) => + if (exception != null) { + requestHelper.sendMaybeThrottle(request, consumerGroupDescribeRequest.getErrorResponse(exception)) + } else { + if (response.groups.isEmpty) { + // If the response is empty, we can directly reuse the results. + response.setGroups(results) + } else { + // Otherwise, we have to copy the results into the existing ones. + response.groups.addAll(results) + } + + requestHelper.sendMaybeThrottle(request, new ConsumerGroupDescribeResponse(response)) + } + } + } + } - // Just a place holder for now. def handleGetTelemetrySubscriptionsRequest(request: RequestChannel.Request): Unit = { - requestHelper.sendMaybeThrottle(request, request.body[GetTelemetrySubscriptionsRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) - CompletableFuture.completedFuture[Unit](()) + val subscriptionRequest = request.body[GetTelemetrySubscriptionsRequest] + + clientMetricsManager match { + case Some(metricsManager) => + try { + requestHelper.sendMaybeThrottle(request, metricsManager.processGetTelemetrySubscriptionRequest(subscriptionRequest, request.context)) + } catch { + case _: Exception => + requestHelper.sendMaybeThrottle(request, subscriptionRequest.getErrorResponse(Errors.INVALID_REQUEST.exception)) + } + case None => + info("Received get telemetry client request for zookeeper based cluster") + requestHelper.sendMaybeThrottle(request, subscriptionRequest.getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) + } } - // Just a place holder for now. def handlePushTelemetryRequest(request: RequestChannel.Request): Unit = { - requestHelper.sendMaybeThrottle(request, request.body[PushTelemetryRequest].getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) - CompletableFuture.completedFuture[Unit](()) + val pushTelemetryRequest = request.body[PushTelemetryRequest] + + clientMetricsManager match { + case Some(metricsManager) => + try { + requestHelper.sendMaybeThrottle(request, metricsManager.processPushTelemetryRequest(pushTelemetryRequest, request.context)) + } catch { + case _: Exception => + requestHelper.sendMaybeThrottle(request, pushTelemetryRequest.getErrorResponse(Errors.INVALID_REQUEST.exception)) + } + case None => + info("Received push telemetry client request for zookeeper based cluster") + requestHelper.sendMaybeThrottle(request, pushTelemetryRequest.getErrorResponse(Errors.UNSUPPORTED_VERSION.exception)) + } + } + + // Just a placeholder for now. + def handleListClientMetricsResources(request: RequestChannel.Request): Unit = { + val listClientMetricsResourcesRequest = request.body[ListClientMetricsResourcesRequest] + + if (!authHelper.authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { + requestHelper.sendMaybeThrottle(request, listClientMetricsResourcesRequest.getErrorResponse(Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) + } else { + // Just return an empty list in the placeholder + val data = new ListClientMetricsResourcesResponseData() + requestHelper.sendMaybeThrottle(request, new ListClientMetricsResourcesResponse(data)) + } } private def updateRecordConversionStats(request: RequestChannel.Request, tp: TopicPartition, - conversionStats: RecordConversionStats): Unit = { + conversionStats: RecordValidationStats): Unit = { val conversionCount = conversionStats.numRecordsConverted if (conversionCount > 0) { request.header.apiKey match { diff --git a/core/src/main/scala/kafka/server/KafkaBroker.scala b/core/src/main/scala/kafka/server/KafkaBroker.scala index ea1d6cf8ed029..e281087f12f51 100644 --- a/core/src/main/scala/kafka/server/KafkaBroker.scala +++ b/core/src/main/scala/kafka/server/KafkaBroker.scala @@ -32,6 +32,7 @@ import org.apache.kafka.common.security.token.delegation.internals.DelegationTok import org.apache.kafka.common.utils.Time import org.apache.kafka.coordinator.group.GroupCoordinator import org.apache.kafka.metadata.BrokerState +import org.apache.kafka.server.NodeToControllerChannelManager import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.metrics.{KafkaMetricsGroup, KafkaYammerMetrics} import org.apache.kafka.server.util.Scheduler diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 4afa60d3d27e6..8aef3cab22b64 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -42,7 +42,7 @@ import org.apache.kafka.common.security.auth.KafkaPrincipalSerde import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder import org.apache.kafka.common.utils.Utils -import org.apache.kafka.coordinator.group.assignor.{PartitionAssignor, RangeAssignor} +import org.apache.kafka.coordinator.group.assignor.{PartitionAssignor, RangeAssignor, UniformAssignor} import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.{MetadataVersion, MetadataVersionValidator} @@ -176,7 +176,7 @@ object Defaults { val ConsumerGroupMinHeartbeatIntervalMs = 5000 val ConsumerGroupMaxHeartbeatIntervalMs = 15000 val ConsumerGroupMaxSize = Int.MaxValue - val ConsumerGroupAssignors = List(classOf[RangeAssignor].getName).asJava + val ConsumerGroupAssignors = List(classOf[UniformAssignor].getName, classOf[RangeAssignor].getName).asJava /** ********* Offset management configuration ***********/ val OffsetMetadataMaxSize = OffsetConfig.DefaultMaxMetadataSize @@ -232,6 +232,9 @@ object Defaults { val KafkaMetricReporterClasses = "" val KafkaMetricsPollingIntervalSeconds = 10 + /** ********* Kafka Client Telemetry Metrics Configuration ***********/ + val ClientTelemetryMaxBytes = 1024 * 1024 + /** ********* SSL configuration ***********/ val SslProtocol = SslConfigs.DEFAULT_SSL_PROTOCOL val SslEnabledProtocols = SslConfigs.DEFAULT_SSL_ENABLED_PROTOCOLS @@ -589,6 +592,9 @@ object KafkaConfig { val KafkaMetricsReporterClassesProp = "kafka.metrics.reporters" val KafkaMetricsPollingIntervalSecondsProp = "kafka.metrics.polling.interval.secs" + /** ********* Kafka Client Telemetry Metrics Configuration ***********/ + val ClientTelemetryMaxBytesProp = "telemetry.max.bytes" + /** ******** Common Security Configuration *************/ val PrincipalBuilderClassProp = BrokerSecurityConfigs.PRINCIPAL_BUILDER_CLASS_CONFIG val ConnectionsMaxReauthMsProp = BrokerSecurityConfigs.CONNECTIONS_MAX_REAUTH_MS @@ -617,6 +623,8 @@ object KafkaConfig { val SslClientAuthProp = BrokerSecurityConfigs.SSL_CLIENT_AUTH_CONFIG val SslPrincipalMappingRulesProp = BrokerSecurityConfigs.SSL_PRINCIPAL_MAPPING_RULES_CONFIG var SslEngineFactoryClassProp = SslConfigs.SSL_ENGINE_FACTORY_CLASS_CONFIG + var SslAllowDnChangesProp = BrokerSecurityConfigs.SSL_ALLOW_DN_CHANGES_CONFIG + var SslAllowSanChangesProp = BrokerSecurityConfigs.SSL_ALLOW_SAN_CHANGES_CONFIG /** ********* SASL Configuration ****************/ val SaslMechanismInterBrokerProtocolProp = "sasl.mechanism.inter.broker.protocol" @@ -669,6 +677,7 @@ object KafkaConfig { /** Internal Configurations **/ val UnstableApiVersionsEnableProp = "unstable.api.versions.enable" + val UnstableMetadataVersionsEnableProp = "unstable.metadata.versions.enable" /* Documentation */ /** ********* Zookeeper Configuration ***********/ @@ -1090,6 +1099,10 @@ object KafkaConfig { val KafkaMetricsPollingIntervalSecondsDoc = s"The metrics polling interval (in seconds) which can be used" + s" in $KafkaMetricsReporterClassesProp implementations." + /** ********* Kafka Client Telemetry Metrics Configuration ***********/ + val ClientTelemetryMaxBytesDoc = "The maximum size (after compression if compression is used) of" + + " telemetry metrics pushed from a client to the broker. The default value is 1048576 (1 MB)." + /** ******** Common Security Configuration *************/ val PrincipalBuilderClassDoc = BrokerSecurityConfigs.PRINCIPAL_BUILDER_CLASS_DOC val ConnectionsMaxReauthMsDoc = BrokerSecurityConfigs.CONNECTIONS_MAX_REAUTH_MS_DOC @@ -1118,6 +1131,8 @@ object KafkaConfig { val SslClientAuthDoc = BrokerSecurityConfigs.SSL_CLIENT_AUTH_DOC val SslPrincipalMappingRulesDoc = BrokerSecurityConfigs.SSL_PRINCIPAL_MAPPING_RULES_DOC val SslEngineFactoryClassDoc = SslConfigs.SSL_ENGINE_FACTORY_CLASS_DOC + val SslAllowDnChangesDoc = BrokerSecurityConfigs.SSL_ALLOW_DN_CHANGES_DOC + val SslAllowSanChangesDoc = BrokerSecurityConfigs.SSL_ALLOW_SAN_CHANGES_DOC /** ********* Sasl Configuration ****************/ val SaslMechanismInterBrokerProtocolDoc = "SASL mechanism used for inter-broker communication. Default is GSSAPI." @@ -1415,6 +1430,9 @@ object KafkaConfig { .define(KafkaMetricsReporterClassesProp, LIST, Defaults.KafkaMetricReporterClasses, LOW, KafkaMetricsReporterClassesDoc) .define(KafkaMetricsPollingIntervalSecondsProp, INT, Defaults.KafkaMetricsPollingIntervalSeconds, atLeast(1), LOW, KafkaMetricsPollingIntervalSecondsDoc) + /** ********* Kafka Client Telemetry Metrics Configuration ***********/ + .define(ClientTelemetryMaxBytesProp, INT, Defaults.ClientTelemetryMaxBytes, atLeast(1), LOW, ClientTelemetryMaxBytesDoc) + /** ********* Quota configuration ***********/ .define(NumQuotaSamplesProp, INT, Defaults.NumQuotaSamples, atLeast(1), LOW, NumQuotaSamplesDoc) .define(NumReplicationQuotaSamplesProp, INT, Defaults.NumReplicationQuotaSamples, atLeast(1), LOW, NumReplicationQuotaSamplesDoc) @@ -1454,6 +1472,8 @@ object KafkaConfig { .define(SslCipherSuitesProp, LIST, Collections.emptyList(), MEDIUM, SslCipherSuitesDoc) .define(SslPrincipalMappingRulesProp, STRING, Defaults.SslPrincipalMappingRules, LOW, SslPrincipalMappingRulesDoc) .define(SslEngineFactoryClassProp, CLASS, null, LOW, SslEngineFactoryClassDoc) + .define(SslAllowDnChangesProp, BOOLEAN, BrokerSecurityConfigs.DEFAULT_SSL_ALLOW_DN_CHANGES_VALUE, LOW, SslAllowDnChangesDoc) + .define(SslAllowSanChangesProp, BOOLEAN, BrokerSecurityConfigs.DEFAULT_SSL_ALLOW_SAN_CHANGES_VALUE, LOW, SslAllowSanChangesDoc) /** ********* Sasl Configuration ****************/ .define(SaslMechanismInterBrokerProtocolProp, STRING, Defaults.SaslMechanismInterBrokerProtocol, MEDIUM, SaslMechanismInterBrokerProtocolDoc) @@ -1513,8 +1533,10 @@ object KafkaConfig { .define(RaftConfig.QUORUM_RETRY_BACKOFF_MS_CONFIG, INT, Defaults.QuorumRetryBackoffMs, null, LOW, RaftConfig.QUORUM_RETRY_BACKOFF_MS_DOC) /** Internal Configurations **/ - // This indicates whether unreleased APIs should be advertised by this broker. - .defineInternal(UnstableApiVersionsEnableProp, BOOLEAN, false, LOW) + // This indicates whether unreleased APIs should be advertised by this node. + .defineInternal(UnstableApiVersionsEnableProp, BOOLEAN, false, HIGH) + // This indicates whether unreleased MetadataVersions should be enabled on this node. + .defineInternal(UnstableMetadataVersionsEnableProp, BOOLEAN, false, HIGH) } /** ********* Remote Log Management Configuration *********/ @@ -1580,6 +1602,7 @@ object KafkaConfig { case ConfigResource.Type.BROKER => KafkaConfig.maybeSensitive(KafkaConfig.configType(name)) case ConfigResource.Type.TOPIC => KafkaConfig.maybeSensitive(LogConfig.configType(name).asScala) case ConfigResource.Type.BROKER_LOGGER => false + case ConfigResource.Type.CLIENT_METRICS => false case _ => true } if (maybeSensitive) Password.HIDDEN else value @@ -2027,6 +2050,9 @@ class KafkaConfig private(doLog: Boolean, val props: java.util.Map[_, _], dynami val metricSampleWindowMs = getLong(KafkaConfig.MetricSampleWindowMsProp) val metricRecordingLevel = getString(KafkaConfig.MetricRecordingLevelProp) + /** ********* Kafka Client Telemetry Metrics Configuration ***********/ + val clientTelemetryMaxBytes: Int = getInt(KafkaConfig.ClientTelemetryMaxBytesProp) + /** ********* SSL/SASL Configuration **************/ // Security configs may be overridden for listeners, so it is not safe to use the base values // Hence the base SSL/SASL configs are not fields of KafkaConfig, listener configs should be @@ -2090,6 +2116,7 @@ class KafkaConfig private(doLog: Boolean, val props: java.util.Map[_, _], dynami /** Internal Configurations **/ val unstableApiVersionsEnabled = getBoolean(KafkaConfig.UnstableApiVersionsEnableProp) + val unstableMetadataVersionsEnabled = getBoolean(KafkaConfig.UnstableMetadataVersionsEnableProp) def addReconfigurable(reconfigurable: Reconfigurable): Unit = { dynamicConfig.addReconfigurable(reconfigurable) diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 322611ef3a3a0..3c9c607662381 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -52,6 +52,7 @@ import org.apache.kafka.metadata.properties.MetaPropertiesEnsemble.VerificationF import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesEnsemble} import org.apache.kafka.metadata.{BrokerState, MetadataRecordSerde, VersionRange} import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.NodeToControllerChannelManager import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.MetadataVersion._ import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} @@ -196,6 +197,7 @@ class KafkaServer( def kafkaController: KafkaController = _kafkaController var lifecycleManager: BrokerLifecycleManager = _ + private var raftManager: KafkaRaftManager[ApiMessageAndVersion] = _ @volatile var brokerEpochManager: ZkBrokerEpochManager = _ @@ -253,7 +255,7 @@ class KafkaServer( // initialize dynamic broker configs from ZooKeeper. Any updates made after this will be // applied after ZkConfigManager starts. - config.dynamicConfig.initialize(Some(zkClient)) + config.dynamicConfig.initialize(Some(zkClient), clientMetricsReceiverPluginOpt = None) /* start scheduler */ kafkaScheduler = new KafkaScheduler(config.backgroundThreads) @@ -281,7 +283,9 @@ class KafkaServer( setClusterId(_clusterId). setNodeId(config.brokerId) if (!builder.directoryId().isPresent()) { - builder.setDirectoryId(copier.generateValidDirectoryId()) + if (config.migrationEnabled) { + builder.setDirectoryId(copier.generateValidDirectoryId()) + } } copier.setLogDirProps(logDir, builder.build()) }) @@ -323,7 +327,8 @@ class KafkaServer( config.brokerId, config.interBrokerProtocolVersion, brokerFeatures, - kraftControllerNodes) + kraftControllerNodes, + config.migrationEnabled) val controllerNodeProvider = new MetadataCacheControllerNodeProvider(metadataCache, config) /* initialize feature change listener */ @@ -337,7 +342,7 @@ class KafkaServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - clientToControllerChannelManager = NodeToControllerChannelManager( + clientToControllerChannelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider = controllerNodeProvider, time = time, metrics = metrics, @@ -360,7 +365,8 @@ class KafkaServer( config, forwardingManager, brokerFeatures, - metadataCache + metadataCache, + None ) // Create and start the socket server acceptor threads so that the bound port is known. @@ -413,7 +419,7 @@ class KafkaServer( // If the ZK broker is in migration mode, start up a RaftManager to learn about the new KRaft controller val controllerQuorumVotersFuture = CompletableFuture.completedFuture( RaftConfig.parseVoterConnections(config.quorumVoters)) - val raftManager = new KafkaRaftManager[ApiMessageAndVersion]( + raftManager = new KafkaRaftManager[ApiMessageAndVersion]( metaPropsEnsemble.clusterId().get(), config, new MetadataRecordSerde, @@ -427,7 +433,7 @@ class KafkaServer( ) val controllerNodes = RaftConfig.voterConnectionsToNodes(controllerQuorumVotersFuture.get()).asScala val quorumControllerNodeProvider = RaftControllerNodeProvider(raftManager, config, controllerNodes) - val brokerToQuorumChannelManager = NodeToControllerChannelManager( + val brokerToQuorumChannelManager = new NodeToControllerChannelManagerImpl( controllerNodeProvider = quorumControllerNodeProvider, time = time, metrics = metrics, @@ -1008,6 +1014,9 @@ class KafkaServer( // Clear all reconfigurable instances stored in DynamicBrokerConfig config.dynamicConfig.clear() + if (raftManager != null) + CoreUtils.swallow(raftManager.shutdown(), this) + if (lifecycleManager != null) { lifecycleManager.close() } diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala index 414e4fab3ca19..015e46a76523d 100755 --- a/core/src/main/scala/kafka/server/MetadataCache.scala +++ b/core/src/main/scala/kafka/server/MetadataCache.scala @@ -116,9 +116,10 @@ object MetadataCache { def zkMetadataCache(brokerId: Int, metadataVersion: MetadataVersion, brokerFeatures: BrokerFeatures = BrokerFeatures.createEmpty(), - kraftControllerNodes: collection.Seq[Node] = collection.Seq.empty[Node]) + kraftControllerNodes: collection.Seq[Node] = collection.Seq.empty[Node], + zkMigrationEnabled: Boolean = false) : ZkMetadataCache = { - new ZkMetadataCache(brokerId, metadataVersion, brokerFeatures, kraftControllerNodes) + new ZkMetadataCache(brokerId, metadataVersion, brokerFeatures, kraftControllerNodes, zkMigrationEnabled) } def kRaftMetadataCache(brokerId: Int): KRaftMetadataCache = { diff --git a/core/src/main/scala/kafka/server/NodeToControllerChannelManager.scala b/core/src/main/scala/kafka/server/NodeToControllerChannelManager.scala index ee69a74aede5f..19d19c87bb1a4 100644 --- a/core/src/main/scala/kafka/server/NodeToControllerChannelManager.scala +++ b/core/src/main/scala/kafka/server/NodeToControllerChannelManager.scala @@ -31,10 +31,12 @@ import org.apache.kafka.common.requests.AbstractRequest import org.apache.kafka.common.security.JaasContext import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.server.util.{InterBrokerSendThread, RequestAndCompletionHandler} import java.util +import java.util.Optional import scala.collection.Seq import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ @@ -130,38 +132,6 @@ class RaftControllerNodeProvider( listenerName, securityProtocol, saslMechanism, isZkController = false) } -object NodeToControllerChannelManager { - def apply( - controllerNodeProvider: ControllerNodeProvider, - time: Time, - metrics: Metrics, - config: KafkaConfig, - channelName: String, - threadNamePrefix: String, - retryTimeoutMs: Long - ): NodeToControllerChannelManager = { - new NodeToControllerChannelManagerImpl( - controllerNodeProvider, - time, - metrics, - config, - channelName, - threadNamePrefix, - retryTimeoutMs - ) - } -} - -trait NodeToControllerChannelManager { - def start(): Unit - def shutdown(): Unit - def controllerApiVersions(): Option[NodeApiVersions] - def sendRequest( - request: AbstractRequest.Builder[_ <: AbstractRequest], - callback: ControllerRequestCompletionHandler - ): Unit -} - /** * This class manages the connection between a broker and the controller. It runs a single * [[NodeToControllerRequestThread]] which uses the broker's metadata cache as its own metadata to find @@ -270,22 +240,13 @@ class NodeToControllerChannelManagerImpl( )) } - def controllerApiVersions(): Option[NodeApiVersions] = { + def controllerApiVersions(): Optional[NodeApiVersions] = { requestThread.activeControllerAddress().flatMap { activeController => Option(apiVersions.get(activeController.idString)) - } + }.asJava } } -abstract class ControllerRequestCompletionHandler extends RequestCompletionHandler { - - /** - * Fire when the request transmission time passes the caller defined deadline on the channel queue. - * It covers the total waiting time including retries which might be the result of individual request timeout. - */ - def onTimeout(): Unit -} - case class NodeToControllerQueueItem( createdTimeMs: Long, request: AbstractRequest.Builder[_ <: AbstractRequest], diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 3eaef6bb44426..681073311a792 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -27,7 +27,7 @@ import kafka.server.QuotaFactory.QuotaManagers import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName} import kafka.server.ReplicaManager.createLogReadResult import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile, OffsetCheckpoints} -import kafka.server.metadata.ZkMetadataCache +import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} import kafka.utils.Implicits._ import kafka.utils._ import kafka.zk.KafkaZkClient @@ -55,6 +55,8 @@ import org.apache.kafka.common.utils.Time import org.apache.kafka.common.{ElectionType, IsolationLevel, Node, TopicIdPartition, TopicPartition, Uuid} import org.apache.kafka.image.{LocalReplicaChanges, MetadataImage, TopicsDelta} import org.apache.kafka.metadata.LeaderConstants.NO_LEADER +import org.apache.kafka.server.common +import org.apache.kafka.server.common.DirectoryEventHandler import org.apache.kafka.server.common.MetadataVersion._ import org.apache.kafka.server.metrics.KafkaMetricsGroup import org.apache.kafka.server.util.{Scheduler, ShutdownableThread} @@ -269,7 +271,8 @@ class ReplicaManager(val config: KafkaConfig, delayedRemoteFetchPurgatoryParam: Option[DelayedOperationPurgatory[DelayedRemoteFetch]] = None, threadNamePrefix: Option[String] = None, val brokerEpochSupplier: () => Long = () => -1, - addPartitionsToTxnManager: Option[AddPartitionsToTxnManager] = None + addPartitionsToTxnManager: Option[AddPartitionsToTxnManager] = None, + directoryEventHandler: DirectoryEventHandler = DirectoryEventHandler.NOOP ) extends Logging { private val metricsGroup = new KafkaMetricsGroup(this.getClass) @@ -760,7 +763,7 @@ class ReplicaManager(val config: KafkaConfig, * @param entriesPerPartition the records per partition to be appended * @param responseCallback callback for sending the response * @param delayedProduceLock lock for the delayed actions - * @param recordConversionStatsCallback callback for updating stats on record conversions + * @param recordValidationStatsCallback callback for updating stats on record conversions * @param requestLocal container for the stateful instances scoped to this request * @param transactionalId transactional ID if the request is from a producer and the producer is transactional * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. @@ -772,7 +775,7 @@ class ReplicaManager(val config: KafkaConfig, entriesPerPartition: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock] = None, - recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), + recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, transactionalId: String = null, actionQueue: ActionQueue = this.defaultActionQueue): Unit = { @@ -792,7 +795,7 @@ class ReplicaManager(val config: KafkaConfig, if (notYetVerifiedEntriesPerPartition.isEmpty || addPartitionsToTxnManager.isEmpty) { appendEntries(verifiedEntriesPerPartition, internalTopicsAllowed, origin, requiredAcks, verificationGuards.toMap, - errorsPerPartition, recordConversionStatsCallback, timeout, responseCallback, delayedProduceLock, actionQueue)(requestLocal, Map.empty) + errorsPerPartition, recordValidationStatsCallback, timeout, responseCallback, delayedProduceLock, actionQueue)(requestLocal, Map.empty) } else { // For unverified entries, send a request to verify. When verified, the append process will proceed via the callback. // We verify above that all partitions use the same producer ID. @@ -810,7 +813,7 @@ class ReplicaManager(val config: KafkaConfig, requiredAcks, verificationGuards.toMap, errorsPerPartition, - recordConversionStatsCallback, + recordValidationStatsCallback, timeout, responseCallback, delayedProduceLock, @@ -844,7 +847,7 @@ class ReplicaManager(val config: KafkaConfig, requiredAcks: Short, verificationGuards: Map[TopicPartition, VerificationGuard], errorsPerPartition: Map[TopicPartition, Errors], - recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit, + recordConversionStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit, timeout: Long, responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock], @@ -917,7 +920,7 @@ class ReplicaManager(val config: KafkaConfig, } } - recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordConversionStats }) + recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) if (delayedProduceRequestRequired(requiredAcks, allEntries, allResults)) { // create delayed produce operation @@ -1883,7 +1886,7 @@ class ReplicaManager(val config: KafkaConfig, if ( config.migrationEnabled && leaderAndIsrRequest.isKRaftController && - leaderAndIsrRequest.requestType() == LeaderAndIsrRequest.Type.FULL + leaderAndIsrRequest.requestType() == AbstractControlRequest.Type.FULL ) { updateStrayLogs(findStrayPartitionsFromLeaderAndIsr(allTopicPartitionsInRequest)) } @@ -2292,9 +2295,9 @@ class ReplicaManager(val config: KafkaConfig, * The log directory failure handler for the replica * * @param dir the absolute path of the log directory - * @param sendZkNotification check if we need to send notification to zookeeper node (needed for unit test) + * @param notifyController check if we need to send notification to the Controller (needed for unit test) */ - def handleLogDirFailure(dir: String, sendZkNotification: Boolean = true): Unit = { + def handleLogDirFailure(dir: String, notifyController: Boolean = true): Unit = { if (!logManager.isLogDirOnline(dir)) return warn(s"Stopping serving replicas in dir $dir") @@ -2323,16 +2326,57 @@ class ReplicaManager(val config: KafkaConfig, s"for partitions ${partitionsWithOfflineFutureReplica.mkString(",")} because they are in the failed log directory $dir.") } logManager.handleLogDirFailure(dir) + if (dir == config.metadataLogDir) { + fatal(s"Shutdown broker because the metadata log dir $dir has failed") + Exit.halt(1) + } - if (sendZkNotification) + if (notifyController) { + if (config.migrationEnabled) { + fatal(s"Shutdown broker because some log directory has failed during migration mode: $dir") + Exit.halt(1) + } if (zkClient.isEmpty) { - warn("Unable to propagate log dir failure via Zookeeper in KRaft mode") + val uuid = logManager.directoryId(dir) + if (uuid.isDefined) { + directoryEventHandler.handleFailure(uuid.get) + } else { + fatal(s"Unable to propagate directory failure disabled because directory $dir has no UUID") + Exit.halt(1) + } } else { zkClient.get.propagateLogDirEvent(localBrokerId) } + } warn(s"Stopped serving replicas in dir $dir") } + /** + * Called when a topic partition is placed in a log directory. + * If a directory event listener is configured, + * and if the selected log directory has an assigned Uuid, + * then the listener is notified of the assignment. + */ + def maybeNotifyPartitionAssignedToDirectory(tp: TopicPartition, dir: String): Unit = { + if (metadataCache.isInstanceOf[KRaftMetadataCache]) { + logManager.directoryId(dir) match { + case None => throw new IllegalStateException(s"Assignment into unidentified directory: ${dir}") + case Some(dirId) => + getPartition(tp) match { + case HostedPartition.Offline | HostedPartition.None => + throw new IllegalStateException("Assignment for a partition that is not online") + case HostedPartition.Online(partition) => partition.topicId match { + case None => + throw new IllegalStateException(s"Assignment for topic without ID: ${tp.topic()}") + case Some(topicId) => + val topicIdPartition = new common.TopicIdPartition(topicId, tp.partition()) + directoryEventHandler.handleAssignment(topicIdPartition, dirId) + } + } + } + } + } + def removeMetrics(): Unit = { ReplicaManager.MetricNames.foreach(metricsGroup.removeMetric) } diff --git a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala index dc80b51484692..bfcb27adfeb16 100644 --- a/core/src/main/scala/kafka/server/RequestHandlerHelper.scala +++ b/core/src/main/scala/kafka/server/RequestHandlerHelper.scala @@ -148,9 +148,11 @@ class RequestHandlerHelper( * Throttle the channel if the controller mutations quota or the request quota have been violated. * Regardless of throttling, send the response immediately. */ - def sendResponseMaybeThrottleWithControllerQuota(controllerMutationQuota: ControllerMutationQuota, - request: RequestChannel.Request, - createResponse: Int => AbstractResponse): Unit = { + def sendResponseMaybeThrottleWithControllerQuota( + controllerMutationQuota: ControllerMutationQuota, + request: RequestChannel.Request, + response: AbstractResponse + ): Unit = { val timeMs = time.milliseconds val controllerThrottleTimeMs = controllerMutationQuota.throttleTime val requestThrottleTimeMs = quotas.request.maybeRecordAndGetThrottleTimeMs(request, timeMs) @@ -165,7 +167,8 @@ class RequestHandlerHelper( } } - requestChannel.sendResponse(request, createResponse(maxThrottleTimeMs), None) + response.maybeSetThrottleTimeMs(maxThrottleTimeMs) + requestChannel.sendResponse(request, response, None) } def sendResponseExemptThrottle(request: RequestChannel.Request, diff --git a/core/src/main/scala/kafka/server/SharedServer.scala b/core/src/main/scala/kafka/server/SharedServer.scala index abf7e0fa0decb..b093d66995f10 100644 --- a/core/src/main/scala/kafka/server/SharedServer.scala +++ b/core/src/main/scala/kafka/server/SharedServer.scala @@ -29,6 +29,7 @@ import org.apache.kafka.image.MetadataProvenance import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.image.loader.metrics.MetadataLoaderMetrics import org.apache.kafka.image.publisher.{SnapshotEmitter, SnapshotGenerator} +import org.apache.kafka.image.publisher.metrics.SnapshotEmitterMetrics import org.apache.kafka.metadata.MetadataRecordSerde import org.apache.kafka.metadata.properties.MetaPropertiesEnsemble import org.apache.kafka.raft.RaftConfig.AddressSpec @@ -244,7 +245,7 @@ class SharedServer( // This is only done in tests. metrics = new Metrics() } - sharedServerConfig.dynamicConfig.initialize(zkClientOpt = None) + sharedServerConfig.dynamicConfig.initialize(zkClientOpt = None, clientMetricsReceiverPluginOpt = None) if (sharedServerConfig.processRoles.contains(BrokerRole)) { brokerMetrics = BrokerServerMetrics(metrics) @@ -289,6 +290,8 @@ class SharedServer( snapshotEmitter = new SnapshotEmitter.Builder(). setNodeId(nodeId). setRaftClient(_raftManager.client). + setMetrics(new SnapshotEmitterMetrics( + Optional.of(KafkaYammerMetrics.defaultRegistry()), time)). build() snapshotGenerator = new SnapshotGenerator.Builder(snapshotEmitter). setNodeId(nodeId). diff --git a/core/src/main/scala/kafka/server/ZkConfigManager.scala b/core/src/main/scala/kafka/server/ZkConfigManager.scala index c782085980774..abead88b07ce4 100644 --- a/core/src/main/scala/kafka/server/ZkConfigManager.scala +++ b/core/src/main/scala/kafka/server/ZkConfigManager.scala @@ -39,7 +39,8 @@ object ConfigType { val Broker = "brokers" val Ip = "ips" val ClientMetrics = "client-metrics" - val all = Seq(Topic, Client, User, Broker, Ip, ClientMetrics) + // Do not include ClientMetrics in `all` as ClientMetrics is not supported on ZK. + val all = Seq(Topic, Client, User, Broker, Ip) } object ConfigEntityName { diff --git a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala index 084e46c5ef266..de3283d21fd42 100644 --- a/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala +++ b/core/src/main/scala/kafka/server/checkpoints/OffsetCheckpointFile.scala @@ -68,7 +68,7 @@ class OffsetCheckpointFile(val file: File, logDirFailureChannel: LogDirFailureCh def write(offsets: Map[TopicPartition, Long]): Unit = { val list: java.util.List[(TopicPartition, Long)] = new java.util.ArrayList[(TopicPartition, Long)](offsets.size) offsets.foreach(x => list.add(x)) - checkpoint.write(list) + checkpoint.write(list, true) } def read(): Map[TopicPartition, Long] = { diff --git a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala index b03648a3ef503..03122b5ef1ba2 100644 --- a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala +++ b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala @@ -103,9 +103,16 @@ class DynamicConfigPublisher( ) case CLIENT_METRICS => // Apply changes to client metrics subscription. - info(s"Updating client metrics subscription ${resource.name()} with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.ClientMetrics).processConfigChanges(resource.name(), props) + dynamicConfigHandlers.get(ConfigType.ClientMetrics).foreach(metricsConfigHandler => + try { + info(s"Updating client metrics ${resource.name()} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + metricsConfigHandler.processConfigChanges(resource.name(), props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating client metrics" + + s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in $deltaName", t) + }) case _ => // nothing to do } } diff --git a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala index 0d998ed84c5ed..485bba8812c46 100644 --- a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala @@ -36,7 +36,7 @@ import java.util.concurrent.ThreadLocalRandom import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.message.{DescribeClientQuotasRequestData, DescribeClientQuotasResponseData} import org.apache.kafka.common.message.{DescribeUserScramCredentialsRequestData, DescribeUserScramCredentialsResponseData} -import org.apache.kafka.metadata.{PartitionRegistration, Replicas} +import org.apache.kafka.metadata.{BrokerRegistration, PartitionRegistration, Replicas} import org.apache.kafka.server.common.{Features, MetadataVersion} import scala.collection.{Map, Seq, Set, mutable} @@ -143,14 +143,11 @@ class KRaftMetadataCache(val brokerId: Int) extends MetadataCache with Logging w private def getOfflineReplicas(image: MetadataImage, partition: PartitionRegistration, listenerName: ListenerName): util.List[Integer] = { - // TODO: in order to really implement this correctly, we would need JBOD support. - // That would require us to track which replicas were offline on a per-replica basis. - // See KAFKA-13005. val offlineReplicas = new util.ArrayList[Integer](0) for (brokerId <- partition.replicas) { Option(image.cluster().broker(brokerId)) match { case None => offlineReplicas.add(brokerId) - case Some(broker) => if (broker.fenced() || !broker.listeners().containsKey(listenerName.value())) { + case Some(broker) => if (isReplicaOffline(partition, listenerName, broker)) { offlineReplicas.add(brokerId) } } @@ -158,6 +155,12 @@ class KRaftMetadataCache(val brokerId: Int) extends MetadataCache with Logging w offlineReplicas } + private def isReplicaOffline(partition: PartitionRegistration, listenerName: ListenerName, broker: BrokerRegistration) = + broker.fenced() || !broker.listeners().containsKey(listenerName.value()) || isReplicaInOfflineDir(broker, partition) + + private def isReplicaInOfflineDir(broker: BrokerRegistration, partition: PartitionRegistration): Boolean = + !broker.hasOnlineDir(partition.directory(broker.id())) + /** * Get the endpoint matching the provided listener if the broker is alive. Note that listeners can * be added dynamically, so a broker with a missing listener could be a transient error. diff --git a/core/src/main/scala/kafka/server/metadata/ZkConfigRepository.scala b/core/src/main/scala/kafka/server/metadata/ZkConfigRepository.scala index 16842bcd11ffe..a2aabe656c56f 100644 --- a/core/src/main/scala/kafka/server/metadata/ZkConfigRepository.scala +++ b/core/src/main/scala/kafka/server/metadata/ZkConfigRepository.scala @@ -23,6 +23,7 @@ import kafka.server.{ConfigEntityName, ConfigType} import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.ConfigResource.Type +import org.apache.kafka.common.errors.InvalidRequestException object ZkConfigRepository { @@ -35,6 +36,7 @@ class ZkConfigRepository(adminZkClient: AdminZkClient) extends ConfigRepository val configTypeForZk = configResource.`type` match { case Type.TOPIC => ConfigType.Topic case Type.BROKER => ConfigType.Broker + case Type.CLIENT_METRICS => throw new InvalidRequestException("Config type client-metrics is only supported on KRaft clusters") case tpe => throw new IllegalArgumentException(s"Unsupported config type: $tpe") } // ZK stores cluster configs under "". diff --git a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala index 302d3fbf8de06..84ef973b8a6c5 100755 --- a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala @@ -31,13 +31,14 @@ import kafka.utils.Logging import kafka.utils.Implicits._ import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.internals.Topic -import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState +import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataPartitionState, UpdateMetadataTopicState} import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition, Uuid} import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition +import org.apache.kafka.common.message.UpdateMetadataRequestData import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.requests.{ApiVersionsResponse, MetadataResponse, UpdateMetadataRequest} +import org.apache.kafka.common.requests.{AbstractControlRequest, ApiVersionsResponse, MetadataResponse, UpdateMetadataRequest} import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.server.common.{Features, MetadataVersion} @@ -55,6 +56,60 @@ trait ZkFinalizedFeatureCache { def getFeatureOption: Option[Features] } +case class MetadataSnapshot(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]], + topicIds: Map[String, Uuid], + controllerId: Option[CachedControllerId], + aliveBrokers: mutable.LongMap[Broker], + aliveNodes: mutable.LongMap[collection.Map[ListenerName, Node]]) { + val topicNames: Map[Uuid, String] = topicIds.map { case (topicName, topicId) => (topicId, topicName) } +} + +object ZkMetadataCache { + /** + * Create topic deletions (leader=-2) for topics that are missing in a FULL UpdateMetadataRequest coming from a + * KRaft controller during a ZK migration. This will modify the UpdateMetadataRequest object passed into this method. + */ + def maybeInjectDeletedPartitionsFromFullMetadataRequest( + currentMetadata: MetadataSnapshot, + requestControllerEpoch: Int, + requestTopicStates: util.List[UpdateMetadataTopicState], + ): Seq[Uuid] = { + val prevTopicIds = currentMetadata.topicIds.values.toSet + val requestTopics = requestTopicStates.asScala.map { topicState => + topicState.topicName() -> topicState.topicId() + }.toMap + + val deleteTopics = prevTopicIds -- requestTopics.values.toSet + if (deleteTopics.isEmpty) { + return Seq.empty + } + + deleteTopics.foreach { deletedTopicId => + val topicName = currentMetadata.topicNames(deletedTopicId) + val topicState = new UpdateMetadataRequestData.UpdateMetadataTopicState() + .setTopicId(deletedTopicId) + .setTopicName(topicName) + .setPartitionStates(new util.ArrayList()) + + currentMetadata.partitionStates(topicName).foreach { case (partitionId, partitionState) => + val lisr = LeaderAndIsr.duringDelete(partitionState.isr().asScala.map(_.intValue()).toList) + val newPartitionState = new UpdateMetadataPartitionState() + .setPartitionIndex(partitionId.toInt) + .setTopicName(topicName) + .setLeader(lisr.leader) + .setLeaderEpoch(lisr.leaderEpoch) + .setControllerEpoch(requestControllerEpoch) + .setReplicas(partitionState.replicas()) + .setZkVersion(lisr.partitionEpoch) + .setIsr(lisr.isr.map(Integer.valueOf).asJava) + topicState.partitionStates().add(newPartitionState) + } + requestTopicStates.add(topicState) + } + deleteTopics.toSeq + } +} + /** * A cache for the state (e.g., current leader) of each partition. This cache is updated through * UpdateMetadataRequest from the controller. Every broker maintains the same cache, asynchronously. @@ -63,7 +118,8 @@ class ZkMetadataCache( brokerId: Int, metadataVersion: MetadataVersion, brokerFeatures: BrokerFeatures, - kraftControllerNodes: Seq[Node] = Seq.empty) + kraftControllerNodes: Seq[Node] = Seq.empty, + zkMigrationEnabled: Boolean = false) extends MetadataCache with ZkFinalizedFeatureCache with Logging { private val partitionMetadataLock = new ReentrantReadWriteLock() @@ -376,6 +432,25 @@ class ZkMetadataCache( // This method returns the deleted TopicPartitions received from UpdateMetadataRequest def updateMetadata(correlationId: Int, updateMetadataRequest: UpdateMetadataRequest): Seq[TopicPartition] = { inWriteLock(partitionMetadataLock) { + if ( + updateMetadataRequest.isKRaftController && + updateMetadataRequest.updateType() == AbstractControlRequest.Type.FULL + ) { + if (!zkMigrationEnabled) { + stateChangeLogger.error(s"Received UpdateMetadataRequest with Type=FULL (2), but ZK migrations " + + s"are not enabled on this broker. Not treating this as a full metadata update") + } else { + val deletedTopicIds = ZkMetadataCache.maybeInjectDeletedPartitionsFromFullMetadataRequest( + metadataSnapshot, updateMetadataRequest.controllerEpoch(), updateMetadataRequest.topicStates()) + if (deletedTopicIds.isEmpty) { + stateChangeLogger.trace(s"Received UpdateMetadataRequest with Type=FULL (2), " + + s"but no deleted topics were detected.") + } else { + stateChangeLogger.debug(s"Received UpdateMetadataRequest with Type=FULL (2), " + + s"found ${deletedTopicIds.size} deleted topic ID(s): $deletedTopicIds.") + } + } + } val aliveBrokers = new mutable.LongMap[Broker](metadataSnapshot.aliveBrokers.size) val aliveNodes = new mutable.LongMap[collection.Map[ListenerName, Node]](metadataSnapshot.aliveNodes.size) @@ -477,14 +552,6 @@ class ZkMetadataCache( } } - case class MetadataSnapshot(partitionStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]], - topicIds: Map[String, Uuid], - controllerId: Option[CachedControllerId], - aliveBrokers: mutable.LongMap[Broker], - aliveNodes: mutable.LongMap[collection.Map[ListenerName, Node]]) { - val topicNames: Map[Uuid, String] = topicIds.map { case (topicName, topicId) => (topicId, topicName) } - } - override def metadataVersion(): MetadataVersion = metadataVersion override def features(): Features = _features match { diff --git a/core/src/main/scala/kafka/tools/StorageTool.scala b/core/src/main/scala/kafka/tools/StorageTool.scala index b2db8b6e8a1e9..30d836370a9ac 100644 --- a/core/src/main/scala/kafka/tools/StorageTool.scala +++ b/core/src/main/scala/kafka/tools/StorageTool.scala @@ -59,10 +59,18 @@ object StorageTool extends Logging { case "format" => val directories = configToLogDirectories(config.get) val clusterId = namespace.getString("cluster_id") - val metadataVersion = getMetadataVersion(namespace, Option(config.get.interBrokerProtocolVersionString)) + val metadataVersion = getMetadataVersion(namespace, + Option(config.get.originals.get(KafkaConfig.InterBrokerProtocolVersionProp)).map(_.toString)) if (!metadataVersion.isKRaftSupported) { throw new TerseFailure(s"Must specify a valid KRaft metadata version of at least 3.0.") } + if (!metadataVersion.isProduction()) { + if (config.get.unstableMetadataVersionsEnabled) { + System.out.println(s"WARNING: using pre-production metadata version ${metadataVersion}.") + } else { + throw new TerseFailure(s"Metadata version ${metadataVersion} is not ready for production use yet.") + } + } val metaProperties = new MetaProperties.Builder(). setVersion(MetaPropertiesVersion.V1). setClusterId(clusterId). @@ -131,7 +139,7 @@ object StorageTool extends Logging { action(storeTrue()) formatParser.addArgument("--release-version", "-r"). action(store()). - help(s"A KRaft release version to use for the initial metadata version. The minimum is 3.0, the default is ${MetadataVersion.latest().version()}") + help(s"A KRaft release version to use for the initial metadata version. The minimum is 3.0, the default is ${MetadataVersion.LATEST_PRODUCTION.version()}") parser.parseArgsOrFail(args) } @@ -151,7 +159,7 @@ object StorageTool extends Logging { ): MetadataVersion = { val defaultValue = defaultVersionString match { case Some(versionString) => MetadataVersion.fromVersionString(versionString) - case None => MetadataVersion.latest() + case None => MetadataVersion.LATEST_PRODUCTION } Option(namespace.getString("release_version")) diff --git a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java index 0ebe08de3ca6d..de1245f55699d 100644 --- a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java +++ b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java @@ -40,6 +40,7 @@ import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.server.common.OffsetAndEpoch; import org.apache.kafka.server.log.remote.storage.ClassLoaderAwareRemoteStorageManager; import org.apache.kafka.server.log.remote.storage.LogSegmentData; import org.apache.kafka.server.log.remote.storage.NoOpRemoteLogMetadataManager; @@ -183,7 +184,7 @@ public class RemoteLogManagerTest { List epochs = Collections.emptyList(); @Override - public void write(Collection epochs) { + public void write(Collection epochs, boolean ignored) { this.epochs = new ArrayList<>(epochs); } @@ -237,18 +238,63 @@ void testGetLeaderEpochCheckpoint() { assertEquals(epochEntry1, epochEntries.get(0)); } + @Test + void testFindHighestRemoteOffsetOnEmptyRemoteStorage() throws RemoteStorageException { + List totalEpochEntries = Arrays.asList( + new EpochEntry(0, 0), + new EpochEntry(1, 500) + ); + checkpoint.write(totalEpochEntries); + LeaderEpochFileCache cache = new LeaderEpochFileCache(tp, checkpoint); + when(mockLog.leaderEpochCache()).thenReturn(Option.apply(cache)); + TopicIdPartition tpId = new TopicIdPartition(Uuid.randomUuid(), tp); + OffsetAndEpoch offsetAndEpoch = remoteLogManager.findHighestRemoteOffset(tpId, mockLog); + assertEquals(new OffsetAndEpoch(-1L, -1), offsetAndEpoch); + } + @Test void testFindHighestRemoteOffset() throws RemoteStorageException { + List totalEpochEntries = Arrays.asList( + new EpochEntry(0, 0), + new EpochEntry(1, 500) + ); checkpoint.write(totalEpochEntries); LeaderEpochFileCache cache = new LeaderEpochFileCache(tp, checkpoint); when(mockLog.leaderEpochCache()).thenReturn(Option.apply(cache)); TopicIdPartition tpId = new TopicIdPartition(Uuid.randomUuid(), tp); - long offset = remoteLogManager.findHighestRemoteOffset(tpId, mockLog); - assertEquals(-1, offset); + when(remoteLogMetadataManager.highestOffsetForEpoch(eq(tpId), anyInt())).thenAnswer(ans -> { + Integer epoch = ans.getArgument(1, Integer.class); + if (epoch == 0) { + return Optional.of(200L); + } else { + return Optional.empty(); + } + }); + OffsetAndEpoch offsetAndEpoch = remoteLogManager.findHighestRemoteOffset(tpId, mockLog); + assertEquals(new OffsetAndEpoch(200L, 0), offsetAndEpoch); + } - when(remoteLogMetadataManager.highestOffsetForEpoch(tpId, 2)).thenReturn(Optional.of(200L)); - long offset2 = remoteLogManager.findHighestRemoteOffset(tpId, mockLog); - assertEquals(200, offset2); + @Test + void testFindHighestRemoteOffsetWithUncleanLeaderElection() throws RemoteStorageException { + List totalEpochEntries = Arrays.asList( + new EpochEntry(0, 0), + new EpochEntry(1, 150), + new EpochEntry(2, 300) + ); + checkpoint.write(totalEpochEntries); + LeaderEpochFileCache cache = new LeaderEpochFileCache(tp, checkpoint); + when(mockLog.leaderEpochCache()).thenReturn(Option.apply(cache)); + TopicIdPartition tpId = new TopicIdPartition(Uuid.randomUuid(), tp); + when(remoteLogMetadataManager.highestOffsetForEpoch(eq(tpId), anyInt())).thenAnswer(ans -> { + Integer epoch = ans.getArgument(1, Integer.class); + if (epoch == 0) { + return Optional.of(200L); + } else { + return Optional.empty(); + } + }); + OffsetAndEpoch offsetAndEpoch = remoteLogManager.findHighestRemoteOffset(tpId, mockLog); + assertEquals(new OffsetAndEpoch(149L, 0), offsetAndEpoch); } @Test @@ -484,7 +530,7 @@ private void assertCopyExpectedLogSegmentsToRemote(long oldSegmentStartOffset, // verify the highest remote offset is updated to the expected value ArgumentCaptor argument = ArgumentCaptor.forClass(Long.class); - verify(mockLog, times(1)).updateHighestOffsetInRemoteStorage(argument.capture()); + verify(mockLog, times(2)).updateHighestOffsetInRemoteStorage(argument.capture()); assertEquals(oldSegmentEndOffset, argument.getValue()); // Verify the metric for remote writes is updated correctly @@ -735,7 +781,7 @@ void testMetricsUpdateOnCopyLogSegmentsFailure() throws Exception { verify(remoteStorageManager, times(1)).copyLogSegmentData(any(RemoteLogSegmentMetadata.class), any(LogSegmentData.class)); // Verify we should not have updated the highest offset because of write failure - verify(mockLog, times(0)).updateHighestOffsetInRemoteStorage(anyLong()); + verify(mockLog).updateHighestOffsetInRemoteStorage(anyLong()); // Verify the metric for remote write requests/failures was updated. assertEquals(1, brokerTopicStats.topicStats(leaderTopicIdPartition.topic()).remoteCopyRequestRate().count()); assertEquals(1, brokerTopicStats.topicStats(leaderTopicIdPartition.topic()).failedRemoteCopyRequestRate().count()); @@ -775,7 +821,7 @@ void testCopyLogSegmentsToRemoteShouldNotCopySegmentForFollower() throws Excepti verify(remoteLogMetadataManager, never()).addRemoteLogSegmentMetadata(any(RemoteLogSegmentMetadata.class)); verify(remoteStorageManager, never()).copyLogSegmentData(any(RemoteLogSegmentMetadata.class), any(LogSegmentData.class)); verify(remoteLogMetadataManager, never()).updateRemoteLogSegmentMetadata(any(RemoteLogSegmentMetadataUpdate.class)); - verify(mockLog, never()).updateHighestOffsetInRemoteStorage(anyLong()); + verify(mockLog).updateHighestOffsetInRemoteStorage(anyLong()); } @Test @@ -992,7 +1038,7 @@ void testFindOffsetByTimestamp() throws IOException, RemoteStorageException { leaderEpochFileCache.assign(targetLeaderEpoch, startOffset); leaderEpochFileCache.assign(12, 500L); - doTestFindOffsetByTimestamp(ts, startOffset, targetLeaderEpoch, validSegmentEpochs); + doTestFindOffsetByTimestamp(ts, startOffset, targetLeaderEpoch, validSegmentEpochs, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); // Fetching message for timestamp `ts` will return the message with startOffset+1, and `ts+1` as there are no // messages starting with the startOffset and with `ts`. @@ -1026,7 +1072,7 @@ void testFindOffsetByTimestampWithInvalidEpochSegments() throws IOException, Rem leaderEpochFileCache.assign(targetLeaderEpoch, startOffset); leaderEpochFileCache.assign(12, 500L); - doTestFindOffsetByTimestamp(ts, startOffset, targetLeaderEpoch, validSegmentEpochs); + doTestFindOffsetByTimestamp(ts, startOffset, targetLeaderEpoch, validSegmentEpochs, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); // Fetch offsets for this segment returns empty as the segment epochs are not with in the leader epoch cache. Optional maybeTimestampAndOffset1 = remoteLogManager.findOffsetByTimestamp(tp, ts, startOffset, leaderEpochFileCache); @@ -1039,8 +1085,32 @@ void testFindOffsetByTimestampWithInvalidEpochSegments() throws IOException, Rem assertEquals(Optional.empty(), maybeTimestampAndOffset3); } + @Test + void testFindOffsetByTimestampWithSegmentNotReady() throws IOException, RemoteStorageException { + TopicPartition tp = leaderTopicIdPartition.topicPartition(); + + long ts = time.milliseconds(); + long startOffset = 120; + int targetLeaderEpoch = 10; + + TreeMap validSegmentEpochs = new TreeMap<>(); + validSegmentEpochs.put(targetLeaderEpoch, startOffset); + + LeaderEpochFileCache leaderEpochFileCache = new LeaderEpochFileCache(tp, checkpoint); + leaderEpochFileCache.assign(4, 99L); + leaderEpochFileCache.assign(5, 99L); + leaderEpochFileCache.assign(targetLeaderEpoch, startOffset); + leaderEpochFileCache.assign(12, 500L); + + doTestFindOffsetByTimestamp(ts, startOffset, targetLeaderEpoch, validSegmentEpochs, RemoteLogSegmentState.COPY_SEGMENT_STARTED); + + Optional maybeTimestampAndOffset = remoteLogManager.findOffsetByTimestamp(tp, ts, startOffset, leaderEpochFileCache); + assertEquals(Optional.empty(), maybeTimestampAndOffset); + } + private void doTestFindOffsetByTimestamp(long ts, long startOffset, int targetLeaderEpoch, - TreeMap validSegmentEpochs) throws IOException, RemoteStorageException { + TreeMap validSegmentEpochs, + RemoteLogSegmentState state) throws IOException, RemoteStorageException { TopicPartition tp = leaderTopicIdPartition.topicPartition(); RemoteLogSegmentId remoteLogSegmentId = new RemoteLogSegmentId(leaderTopicIdPartition, Uuid.randomUuid()); @@ -1050,6 +1120,7 @@ private void doTestFindOffsetByTimestamp(long ts, long startOffset, int targetLe when(segmentMetadata.startOffset()).thenReturn(startOffset); when(segmentMetadata.endOffset()).thenReturn(startOffset + 2); when(segmentMetadata.segmentLeaderEpochs()).thenReturn(validSegmentEpochs); + when(segmentMetadata.state()).thenReturn(state); File tpDir = new File(logDir, tp.toString()); Files.createDirectory(tpDir.toPath()); @@ -1404,9 +1475,9 @@ public void testStopPartitionsWithDeletion() throws RemoteStorageException { assertNotNull(remoteLogManager.task(followerTopicIdPartition)); when(remoteLogMetadataManager.listRemoteLogSegments(eq(leaderTopicIdPartition))) - .thenReturn(listRemoteLogSegmentMetadata(leaderTopicIdPartition, 5, 100, 1024).iterator()); + .thenReturn(listRemoteLogSegmentMetadata(leaderTopicIdPartition, 5, 100, 1024, RemoteLogSegmentState.DELETE_SEGMENT_FINISHED).iterator()); when(remoteLogMetadataManager.listRemoteLogSegments(eq(followerTopicIdPartition))) - .thenReturn(listRemoteLogSegmentMetadata(followerTopicIdPartition, 3, 100, 1024).iterator()); + .thenReturn(listRemoteLogSegmentMetadata(followerTopicIdPartition, 3, 100, 1024, RemoteLogSegmentState.DELETE_SEGMENT_FINISHED).iterator()); CompletableFuture dummyFuture = new CompletableFuture<>(); dummyFuture.complete(null); when(remoteLogMetadataManager.updateRemoteLogSegmentMetadata(any())) @@ -1546,7 +1617,7 @@ public void testDeletionOnRetentionBreachedSegments(long retentionSize, when(mockLog.logEndOffset()).thenReturn(200L); List metadataList = - listRemoteLogSegmentMetadata(leaderTopicIdPartition, 2, 100, 1024, epochEntries); + listRemoteLogSegmentMetadata(leaderTopicIdPartition, 2, 100, 1024, epochEntries, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition)) .thenReturn(metadataList.iterator()); when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition, 0)) @@ -1574,7 +1645,7 @@ public void testDeleteRetentionMsBeingCancelledBeforeSecondDelete() throws Remot List epochEntries = Collections.singletonList(epochEntry0); List metadataList = - listRemoteLogSegmentMetadata(leaderTopicIdPartition, 2, 100, 1024, epochEntries); + listRemoteLogSegmentMetadata(leaderTopicIdPartition, 2, 100, 1024, epochEntries, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition)) .thenReturn(metadataList.iterator()); when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition, 0)) @@ -1664,7 +1735,7 @@ public void testDeleteLogSegmentDueToRetentionSizeBreach(int segmentCount, when(mockLog.onlyLocalLogSegmentsSize()).thenReturn(localLogSegmentsSize); List segmentMetadataList = listRemoteLogSegmentMetadata( - leaderTopicIdPartition, segmentCount, recordsPerSegment, segmentSize, epochEntries); + leaderTopicIdPartition, segmentCount, recordsPerSegment, segmentSize, epochEntries, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); verifyDeleteLogSegment(segmentMetadataList, deletableSegmentCount, currentLeaderEpoch); } @@ -1701,7 +1772,7 @@ public void testDeleteLogSegmentDueToRetentionTimeBreach(int segmentCount, when(mockLog.onlyLocalLogSegmentsSize()).thenReturn(localLogSegmentsSize); List segmentMetadataList = listRemoteLogSegmentMetadataByTime( - leaderTopicIdPartition, segmentCount, deletableSegmentCount, recordsPerSegment, segmentSize, epochEntries); + leaderTopicIdPartition, segmentCount, deletableSegmentCount, recordsPerSegment, segmentSize, epochEntries, RemoteLogSegmentState.COPY_SEGMENT_FINISHED); verifyDeleteLogSegment(segmentMetadataList, deletableSegmentCount, currentLeaderEpoch); } @@ -1735,20 +1806,76 @@ private void verifyDeleteLogSegment(List segmentMetada } } + + @Test + public void testDeleteRetentionMsOnExpiredSegment() throws RemoteStorageException, IOException { + AtomicLong logStartOffset = new AtomicLong(0); + try (RemoteLogManager remoteLogManager = new RemoteLogManager(remoteLogManagerConfig, brokerId, logDir, clusterId, time, + tp -> Optional.of(mockLog), + (topicPartition, offset) -> logStartOffset.set(offset), + brokerTopicStats) { + public RemoteStorageManager createRemoteStorageManager() { + return remoteStorageManager; + } + public RemoteLogMetadataManager createRemoteLogMetadataManager() { + return remoteLogMetadataManager; + } + }) { + RemoteLogManager.RLMTask task = remoteLogManager.new RLMTask(leaderTopicIdPartition, 128); + task.convertToLeader(0); + + when(mockLog.topicPartition()).thenReturn(leaderTopicIdPartition.topicPartition()); + when(mockLog.logEndOffset()).thenReturn(200L); + + List epochEntries = Collections.singletonList(epochEntry0); + + List remoteLogSegmentMetadatas = listRemoteLogSegmentMetadata( + leaderTopicIdPartition, 2, 100, 1024, epochEntries, RemoteLogSegmentState.DELETE_SEGMENT_FINISHED); + + when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition)) + .thenReturn(remoteLogSegmentMetadatas.iterator()); + when(remoteLogMetadataManager.listRemoteLogSegments(leaderTopicIdPartition, 0)) + .thenReturn(remoteLogSegmentMetadatas.iterator()) + .thenReturn(remoteLogSegmentMetadatas.iterator()); + + checkpoint.write(epochEntries); + LeaderEpochFileCache cache = new LeaderEpochFileCache(tp, checkpoint); + when(mockLog.leaderEpochCache()).thenReturn(Option.apply(cache)); + + Map logProps = new HashMap<>(); + logProps.put("retention.bytes", -1L); + logProps.put("retention.ms", 0L); + LogConfig mockLogConfig = new LogConfig(logProps); + when(mockLog.config()).thenReturn(mockLogConfig); + + when(remoteLogMetadataManager.updateRemoteLogSegmentMetadata(any(RemoteLogSegmentMetadataUpdate.class))) + .thenAnswer(answer -> CompletableFuture.runAsync(() -> { })); + + task.cleanupExpiredRemoteLogSegments(); + + verifyNoMoreInteractions(remoteStorageManager); + assertEquals(0L, logStartOffset.get()); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + private List listRemoteLogSegmentMetadata(TopicIdPartition topicIdPartition, int segmentCount, int recordsPerSegment, - int segmentSize) { - return listRemoteLogSegmentMetadata(topicIdPartition, segmentCount, recordsPerSegment, segmentSize, Collections.emptyList()); + int segmentSize, + RemoteLogSegmentState state) { + return listRemoteLogSegmentMetadata(topicIdPartition, segmentCount, recordsPerSegment, segmentSize, Collections.emptyList(), state); } private List listRemoteLogSegmentMetadata(TopicIdPartition topicIdPartition, int segmentCount, int recordsPerSegment, int segmentSize, - List epochEntries) { + List epochEntries, + RemoteLogSegmentState state) { return listRemoteLogSegmentMetadataByTime( - topicIdPartition, segmentCount, 0, recordsPerSegment, segmentSize, epochEntries); + topicIdPartition, segmentCount, 0, recordsPerSegment, segmentSize, epochEntries, state); } private List listRemoteLogSegmentMetadataByTime(TopicIdPartition topicIdPartition, @@ -1756,7 +1883,8 @@ private List listRemoteLogSegmentMetadataByTime(TopicI int deletableSegmentCount, int recordsPerSegment, int segmentSize, - List epochEntries) { + List epochEntries, + RemoteLogSegmentState state) { List segmentMetadataList = new ArrayList<>(); for (int idx = 0; idx < segmentCount; idx++) { long timestamp = time.milliseconds(); @@ -1767,9 +1895,18 @@ private List listRemoteLogSegmentMetadataByTime(TopicI long endOffset = startOffset + recordsPerSegment - 1; List localTotalEpochEntries = epochEntries.isEmpty() ? totalEpochEntries : epochEntries; Map segmentLeaderEpochs = truncateAndGetLeaderEpochs(localTotalEpochEntries, startOffset, endOffset); - segmentMetadataList.add(new RemoteLogSegmentMetadata(new RemoteLogSegmentId(topicIdPartition, - Uuid.randomUuid()), startOffset, endOffset, timestamp, brokerId, timestamp, segmentSize, - segmentLeaderEpochs)); + RemoteLogSegmentMetadata metadata = new RemoteLogSegmentMetadata( + new RemoteLogSegmentId(topicIdPartition, Uuid.randomUuid()), + startOffset, + endOffset, + timestamp, + brokerId, + timestamp, + segmentSize, + Optional.empty(), + state, + segmentLeaderEpochs); + segmentMetadataList.add(metadata); } return segmentMetadataList; } diff --git a/core/src/test/java/kafka/metrics/ClientMetricsTestUtils.java b/core/src/test/java/kafka/metrics/ClientMetricsTestUtils.java deleted file mode 100644 index 3697ba7ba55dc..0000000000000 --- a/core/src/test/java/kafka/metrics/ClientMetricsTestUtils.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.metrics; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -public class ClientMetricsTestUtils { - - public static final String DEFAULT_METRICS = - "org.apache.kafka.client.producer.partition.queue.,org.apache.kafka.client.producer.partition.latency"; - public static final int DEFAULT_PUSH_INTERVAL_MS = 30 * 1000; // 30 seconds - public static final List DEFAULT_CLIENT_MATCH_PATTERNS = Collections.unmodifiableList(Arrays.asList( - ClientMetricsConfigs.CLIENT_SOFTWARE_NAME + "=apache-kafka-java", - ClientMetricsConfigs.CLIENT_SOFTWARE_VERSION + "=3.5.*" - )); - - public static Properties defaultProperties() { - Properties props = new Properties(); - props.put(ClientMetricsConfigs.SUBSCRIPTION_METRICS, DEFAULT_METRICS); - props.put(ClientMetricsConfigs.PUSH_INTERVAL_MS, Integer.toString(DEFAULT_PUSH_INTERVAL_MS)); - props.put(ClientMetricsConfigs.CLIENT_MATCH_PATTERN, String.join(",", DEFAULT_CLIENT_MATCH_PATTERNS)); - return props; - } -} diff --git a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java index 3e09dc8fbe89c..0857e4ded30c5 100644 --- a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java +++ b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java @@ -117,6 +117,6 @@ public void testNoAutoStart() { @ClusterTest public void testDefaults(ClusterConfig config) { - Assertions.assertEquals(MetadataVersion.IBP_3_7_IV1, config.metadataVersion()); + Assertions.assertEquals(MetadataVersion.IBP_3_7_IV3, config.metadataVersion()); } } diff --git a/core/src/test/java/kafka/test/annotation/ClusterTest.java b/core/src/test/java/kafka/test/annotation/ClusterTest.java index 150aa7e7d71df..1511e28a3d4f7 100644 --- a/core/src/test/java/kafka/test/annotation/ClusterTest.java +++ b/core/src/test/java/kafka/test/annotation/ClusterTest.java @@ -41,6 +41,6 @@ String name() default ""; SecurityProtocol securityProtocol() default SecurityProtocol.PLAINTEXT; String listener() default ""; - MetadataVersion metadataVersion() default MetadataVersion.IBP_3_7_IV1; + MetadataVersion metadataVersion() default MetadataVersion.IBP_3_7_IV3; ClusterConfigProperty[] serverProperties() default {}; } diff --git a/core/src/test/java/kafka/test/annotation/Type.java b/core/src/test/java/kafka/test/annotation/Type.java index 933ca5011341b..feb9a2784893d 100644 --- a/core/src/test/java/kafka/test/annotation/Type.java +++ b/core/src/test/java/kafka/test/annotation/Type.java @@ -30,36 +30,36 @@ public enum Type { KRAFT { @Override - public void invocationContexts(ClusterConfig config, Consumer invocationConsumer) { - invocationConsumer.accept(new RaftClusterInvocationContext(config.copyOf(), false)); + public void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer) { + invocationConsumer.accept(new RaftClusterInvocationContext(baseDisplayName, config.copyOf(), false)); } }, CO_KRAFT { @Override - public void invocationContexts(ClusterConfig config, Consumer invocationConsumer) { - invocationConsumer.accept(new RaftClusterInvocationContext(config.copyOf(), true)); + public void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer) { + invocationConsumer.accept(new RaftClusterInvocationContext(baseDisplayName, config.copyOf(), true)); } }, ZK { @Override - public void invocationContexts(ClusterConfig config, Consumer invocationConsumer) { - invocationConsumer.accept(new ZkClusterInvocationContext(config.copyOf())); + public void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer) { + invocationConsumer.accept(new ZkClusterInvocationContext(baseDisplayName, config.copyOf())); } }, ALL { @Override - public void invocationContexts(ClusterConfig config, Consumer invocationConsumer) { - invocationConsumer.accept(new RaftClusterInvocationContext(config.copyOf(), false)); - invocationConsumer.accept(new RaftClusterInvocationContext(config.copyOf(), true)); - invocationConsumer.accept(new ZkClusterInvocationContext(config.copyOf())); + public void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer) { + invocationConsumer.accept(new RaftClusterInvocationContext(baseDisplayName, config.copyOf(), false)); + invocationConsumer.accept(new RaftClusterInvocationContext(baseDisplayName, config.copyOf(), true)); + invocationConsumer.accept(new ZkClusterInvocationContext(baseDisplayName, config.copyOf())); } }, DEFAULT { @Override - public void invocationContexts(ClusterConfig config, Consumer invocationConsumer) { + public void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer) { throw new UnsupportedOperationException("Cannot create invocation contexts for DEFAULT type"); } }; - public abstract void invocationContexts(ClusterConfig config, Consumer invocationConsumer); + public abstract void invocationContexts(String baseDisplayName, ClusterConfig config, Consumer invocationConsumer); } diff --git a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java index 50e2a063649dc..8c838b279bdc8 100644 --- a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java +++ b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java @@ -128,7 +128,8 @@ private void processClusterTemplate(ExtensionContext context, ClusterTemplate an generatedClusterConfigs.add(ClusterConfig.defaultClusterBuilder().build()); } - generatedClusterConfigs.forEach(config -> config.clusterType().invocationContexts(config, testInvocations)); + String baseDisplayName = context.getRequiredTestMethod().getName(); + generatedClusterConfigs.forEach(config -> config.clusterType().invocationContexts(baseDisplayName, config, testInvocations)); } private void generateClusterConfigurations(ExtensionContext context, String generateClustersMethods, ClusterGenerator generator) { @@ -183,8 +184,6 @@ private void processClusterTest(ExtensionContext context, ClusterTest annot, Clu annot.securityProtocol(), annot.metadataVersion()); if (!annot.name().isEmpty()) { builder.name(annot.name()); - } else { - builder.name(context.getRequiredTestMethod().getName()); } if (!annot.listener().isEmpty()) { builder.listenerName(annot.listener()); @@ -197,7 +196,7 @@ private void processClusterTest(ExtensionContext context, ClusterTest annot, Clu ClusterConfig config = builder.build(); config.serverProperties().putAll(properties); - type.invocationContexts(config, testInvocations); + type.invocationContexts(context.getRequiredTestMethod().getName(), config, testInvocations); } private ClusterTestDefaults getClusterTestDefaults(Class testClass) { diff --git a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java index 98f8ab55ce767..4aa152055af20 100644 --- a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java +++ b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java @@ -64,12 +64,14 @@ */ public class RaftClusterInvocationContext implements TestTemplateInvocationContext { + private final String baseDisplayName; private final ClusterConfig clusterConfig; private final AtomicReference clusterReference; private final AtomicReference zkReference; private final boolean isCombined; - public RaftClusterInvocationContext(ClusterConfig clusterConfig, boolean isCombined) { + public RaftClusterInvocationContext(String baseDisplayName, ClusterConfig clusterConfig, boolean isCombined) { + this.baseDisplayName = baseDisplayName; this.clusterConfig = clusterConfig; this.clusterReference = new AtomicReference<>(); this.zkReference = new AtomicReference<>(); @@ -81,7 +83,7 @@ public String getDisplayName(int invocationIndex) { String clusterDesc = clusterConfig.nameTags().entrySet().stream() .map(Object::toString) .collect(Collectors.joining(", ")); - return String.format("[%d] Type=Raft-%s, %s", invocationIndex, isCombined ? "Combined" : "Isolated", clusterDesc); + return String.format("%s [%d] Type=Raft-%s, %s", baseDisplayName, invocationIndex, isCombined ? "Combined" : "Isolated", clusterDesc); } @Override diff --git a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java index 5eb342d8a5354..dd0098f7868ca 100644 --- a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java +++ b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java @@ -66,10 +66,12 @@ */ public class ZkClusterInvocationContext implements TestTemplateInvocationContext { + private final String baseDisplayName; private final ClusterConfig clusterConfig; private final AtomicReference clusterReference; - public ZkClusterInvocationContext(ClusterConfig clusterConfig) { + public ZkClusterInvocationContext(String baseDisplayName, ClusterConfig clusterConfig) { + this.baseDisplayName = baseDisplayName; this.clusterConfig = clusterConfig; this.clusterReference = new AtomicReference<>(); } @@ -79,7 +81,7 @@ public String getDisplayName(int invocationIndex) { String clusterDesc = clusterConfig.nameTags().entrySet().stream() .map(Object::toString) .collect(Collectors.joining(", ")); - return String.format("[%d] Type=ZK, %s", invocationIndex, clusterDesc); + return String.format("%s [%d] Type=ZK, %s", baseDisplayName, invocationIndex, clusterDesc); } @Override diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 77f1832bbe323..2cef03f67af22 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -198,6 +198,7 @@ private KafkaConfig createNodeConfig(TestKitNode node) { if (brokerNode != null) { props.putAll(brokerNode.propertyOverrides()); } + props.putIfAbsent(KafkaConfig$.MODULE$.UnstableMetadataVersionsEnableProp(), "true"); return new KafkaConfig(props, false, Option.empty()); } diff --git a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala index c3eca854ccc97..06ee92ff968c4 100644 --- a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala +++ b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala @@ -87,7 +87,9 @@ class BrokerApiVersionsCommandTest extends KafkaServerTestHarness { else s"${apiVersion.minVersion} to ${apiVersion.maxVersion}" val usableVersion = nodeApiVersions.latestUsableVersion(apiKey) - val line = s"\t${apiKey.name}(${apiKey.id}): $versionRangeStr [usable: $usableVersion]$terminator" + val line = + if (apiKey == ApiKeys.GET_TELEMETRY_SUBSCRIPTIONS || apiKey == ApiKeys.PUSH_TELEMETRY) s"\t${apiKey.name}(${apiKey.id}): UNSUPPORTED$terminator" + else s"\t${apiKey.name}(${apiKey.id}): $versionRangeStr [usable: $usableVersion]$terminator" assertTrue(lineIter.hasNext) assertEquals(line, lineIter.next()) } else { diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala index 1927077cfcfb2..ee9d8426eeb96 100644 --- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala @@ -79,7 +79,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest { super.setUp(testInfo) // create the test topic with all the brokers as replicas - createTopic(topic, 2, brokerCount) + createTopic(topic, 2, brokerCount, adminClientConfig = this.adminClientConfig) } protected class TestConsumerReassignmentListener extends ConsumerRebalanceListener { diff --git a/core/src/test/scala/integration/kafka/api/BaseAsyncConsumerTest.scala b/core/src/test/scala/integration/kafka/api/BaseAsyncConsumerTest.scala deleted file mode 100644 index 6d09575a46347..0000000000000 --- a/core/src/test/scala/integration/kafka/api/BaseAsyncConsumerTest.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.api - -import kafka.utils.TestUtils.waitUntilTrue -import org.junit.jupiter.api.Assertions.{assertNotNull, assertNull, assertTrue} -import org.junit.jupiter.api.Test - -import java.time.Duration -import scala.jdk.CollectionConverters._ - -class BaseAsyncConsumerTest extends AbstractConsumerTest { - val defaultBlockingAPITimeoutMs = 1000 - - @Test - def testCommitAsync(): Unit = { - val consumer = createAsyncConsumer() - val producer = createProducer() - val numRecords = 10000 - val startingTimestamp = System.currentTimeMillis() - val cb = new CountConsumerCommitCallback - sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) - consumer.assign(List(tp).asJava) - consumer.commitAsync(cb) - waitUntilTrue(() => { - cb.successCount == 1 - }, "wait until commit is completed successfully", defaultBlockingAPITimeoutMs) - val committedOffset = consumer.committed(Set(tp).asJava, Duration.ofMillis(defaultBlockingAPITimeoutMs)) - assertNotNull(committedOffset) - // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to - // tp. The committed offset should be null. This is intentional. - assertNull(committedOffset.get(tp)) - assertTrue(consumer.assignment.contains(tp)) - } - - @Test - def testCommitSync(): Unit = { - val consumer = createAsyncConsumer() - val producer = createProducer() - val numRecords = 10000 - val startingTimestamp = System.currentTimeMillis() - sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) - consumer.assign(List(tp).asJava) - consumer.commitSync() - val committedOffset = consumer.committed(Set(tp).asJava, Duration.ofMillis(defaultBlockingAPITimeoutMs)) - assertNotNull(committedOffset) - // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to - // tp. The committed offset should be null. This is intentional. - assertNull(committedOffset.get(tp)) - assertTrue(consumer.assignment.contains(tp)) - } -} diff --git a/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala b/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala index 02d6265edc9f3..eb2be928c42b3 100644 --- a/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/BaseConsumerTest.scala @@ -16,13 +16,15 @@ */ package kafka.api +import kafka.utils.TestInfoUtils import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig} import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig} import org.apache.kafka.common.{ClusterResource, ClusterResourceListener, PartitionInfo} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.serialization.{Deserializer, Serializer} -import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource} import java.util.Properties import java.util.concurrent.atomic.AtomicInteger @@ -34,8 +36,9 @@ import scala.collection.Seq */ abstract class BaseConsumerTest extends AbstractConsumerTest { - @Test - def testSimpleConsumption(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testSimpleConsumption(quorum: String, groupProtocol: String): Unit = { val numRecords = 10000 val producer = createProducer() val startingTimestamp = System.currentTimeMillis() @@ -53,8 +56,9 @@ abstract class BaseConsumerTest extends AbstractConsumerTest { sendAndAwaitAsyncCommit(consumer) } - @Test - def testClusterResourceListener(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testClusterResourceListener(quorum: String, groupProtocol: String): Unit = { val numRecords = 100 val producerProps = new Properties() producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[BaseConsumerTest.TestClusterResourceListenerSerializer]) @@ -74,8 +78,10 @@ abstract class BaseConsumerTest extends AbstractConsumerTest { assertNotEquals(0, BaseConsumerTest.updateConsumerCount.get()) } - @Test - def testCoordinatorFailover(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testCoordinatorFailover(quorum: String, groupProtocol: String): Unit = { val listener = new TestConsumerReassignmentListener() this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "5001") this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "1000") @@ -98,7 +104,7 @@ abstract class BaseConsumerTest extends AbstractConsumerTest { // shutdown the coordinator val coordinator = parts.head.leader().id() - this.servers(coordinator).shutdown() + this.brokers(coordinator).shutdown() // the failover should not cause a rebalance ensureNoRebalance(consumer, listener) @@ -106,6 +112,39 @@ abstract class BaseConsumerTest extends AbstractConsumerTest { } object BaseConsumerTest { + // We want to test the following combinations: + // * ZooKeeper and the generic group protocol + // * KRaft and the generic group protocol + // * KRaft with the new group coordinator enabled and the generic group protocol + // * KRaft with the new group coordinator enabled and the consumer group protocol (temporarily disabled) + def getTestQuorumAndGroupProtocolParametersAll() : java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.of("zk", "generic"), + Arguments.of("kraft", "generic"), + Arguments.of("kraft+kip848", "generic")) +// Arguments.of("kraft+kip848", "consumer")) + } + + // In Scala 2.12, it is necessary to disambiguate the java.util.stream.Stream.of() method call + // in the case where there's only a single Arguments in the list. The following commented-out + // method works in Scala 2.13, but not 2.12. For this reason, tests which run against just a + // single combination are written using @CsvSource rather than the more elegant @MethodSource. + // def getTestQuorumAndGroupProtocolParametersZkOnly() : java.util.stream.Stream[Arguments] = { + // java.util.stream.Stream.of( + // Arguments.of("zk", "generic")) + // } + + // For tests that only work with the generic group protocol, we want to test the following combinations: + // * ZooKeeper and the generic group protocol + // * KRaft and the generic group protocol + // * KRaft with the new group coordinator enabled and the generic group protocol + def getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly() : java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.of("zk", "generic"), + Arguments.of("kraft", "generic"), + Arguments.of("kraft+kip848", "generic")) + } + val updateProducerCount = new AtomicInteger() val updateConsumerCount = new AtomicInteger() diff --git a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala index 1a861b2b2f129..2381976f378f9 100644 --- a/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala +++ b/core/src/test/scala/integration/kafka/api/IntegrationTestHarness.scala @@ -27,7 +27,6 @@ import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig} import kafka.server.KafkaConfig import kafka.integration.KafkaServerTestHarness import org.apache.kafka.clients.admin.{Admin, AdminClientConfig} -import org.apache.kafka.clients.consumer.internals.PrototypeAsyncConsumer import org.apache.kafka.common.network.{ListenerName, Mode} import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, Serializer} import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo} @@ -68,6 +67,9 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness { if (isZkMigrationTest()) { cfgs.foreach(_.setProperty(KafkaConfig.MigrationEnabledProp, "true")) } + if (isNewGroupCoordinatorEnabled()) { + cfgs.foreach(_.setProperty(KafkaConfig.NewGroupCoordinatorEnableProp, "true")) + } insertControllerListenersIfNeeded(cfgs) cfgs.map(KafkaConfig.fromProps) } @@ -141,6 +143,7 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness { consumerConfig.putIfAbsent(ConsumerConfig.GROUP_ID_CONFIG, "group") consumerConfig.putIfAbsent(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName) consumerConfig.putIfAbsent(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[ByteArrayDeserializer].getName) + maybeGroupProtocolSpecified(testInfo).map(groupProtocol => consumerConfig.putIfAbsent(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol.name)) adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()) @@ -171,19 +174,6 @@ abstract class IntegrationTestHarness extends KafkaServerTestHarness { producer } - def createAsyncConsumer[K, V](keyDeserializer: Deserializer[K] = new ByteArrayDeserializer, - valueDeserializer: Deserializer[V] = new ByteArrayDeserializer, - configOverrides: Properties = new Properties, - configsToRemove: List[String] = List()): PrototypeAsyncConsumer[K, V] = { - val props = new Properties - props ++= consumerConfig - props ++= configOverrides - configsToRemove.foreach(props.remove(_)) - val consumer = new PrototypeAsyncConsumer[K, V](props, keyDeserializer, valueDeserializer) - consumers += consumer - consumer - } - def createConsumer[K, V](keyDeserializer: Deserializer[K] = new ByteArrayDeserializer, valueDeserializer: Deserializer[V] = new ByteArrayDeserializer, configOverrides: Properties = new Properties, diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala index c9dff86ccd4b8..add60e81c35e5 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala @@ -19,13 +19,12 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock import java.util.regex.Pattern import java.util.{Locale, Optional, Properties} - -import kafka.server.{KafkaServer, QuotaType} -import kafka.utils.TestUtils +import kafka.server.{KafkaBroker, QuotaType} +import kafka.utils.{TestInfoUtils, TestUtils} import org.apache.kafka.clients.admin.{NewPartitions, NewTopic} import org.apache.kafka.clients.consumer._ import org.apache.kafka.clients.producer.{ProducerConfig, ProducerRecord} -import org.apache.kafka.common.{MetricName, TopicPartition} +import org.apache.kafka.common.{KafkaException, MetricName, PartitionInfo, TopicPartition} import org.apache.kafka.common.config.TopicConfig import org.apache.kafka.common.errors.{InvalidGroupIdException, InvalidTopicException} import org.apache.kafka.common.header.Headers @@ -34,19 +33,20 @@ import org.apache.kafka.common.serialization._ import org.apache.kafka.common.utils.Utils import org.apache.kafka.test.{MockConsumerInterceptor, MockProducerInterceptor} import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ValueSource +import org.junit.jupiter.params.provider.{CsvSource, MethodSource} import scala.collection.mutable import scala.collection.mutable.Buffer import scala.jdk.CollectionConverters._ -/* We have some tests in this class instead of `BaseConsumerTest` in order to keep the build time under control. */ +@Timeout(600) class PlaintextConsumerTest extends BaseConsumerTest { - @Test - def testHeaders(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testHeaders(quorum: String, groupProtocol: String): Unit = { val numRecords = 1 val record = new ProducerRecord(tp.topic, tp.partition, null, "key".getBytes, "value".getBytes) @@ -131,17 +131,20 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(numRecords, records.size) } + // Deprecated poll(timeout) not supported for consumer group protocol @deprecated("poll(Duration) is the replacement", since = "2.0") - @Test - def testDeprecatedPollBlocksForAssignment(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testDeprecatedPollBlocksForAssignment(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() consumer.subscribe(Set(topic).asJava) consumer.poll(0) assertEquals(Set(tp, tp2), consumer.assignment().asScala) } - @Test - def testHeadersSerializerDeserializer(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testHeadersSerializerDeserializer(quorum: String, groupProtocol: String): Unit = { val extendedSerializer = new Serializer[Array[Byte]] with SerializerImpl val extendedDeserializer = new Deserializer[Array[Byte]] with DeserializerImpl @@ -149,8 +152,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { testHeadersSerializeDeserialize(extendedSerializer, extendedDeserializer) } - @Test - def testMaxPollRecords(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testMaxPollRecords(quorum: String, groupProtocol: String): Unit = { val maxPollRecords = 2 val numRecords = 10000 @@ -165,8 +169,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { startingTimestamp = startingTimestamp) } - @Test - def testMaxPollIntervalMs(): Unit = { + // ConsumerRebalanceListener temporarily not supported for the consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMaxPollIntervalMs(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 1000.toString) this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString) this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 2000.toString) @@ -190,8 +196,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(1, listener.callsToRevoked) } - @Test - def testMaxPollIntervalMsDelayInRevocation(): Unit = { + // ConsumerRebalanceListener temporarily not supported for the consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMaxPollIntervalMsDelayInRevocation(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 5000.toString) this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString) this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000.toString) @@ -230,8 +238,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertTrue(commitCompleted) } - @Test - def testMaxPollIntervalMsDelayInAssignment(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMaxPollIntervalMsDelayInAssignment(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 5000.toString) this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 500.toString) this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 1000.toString) @@ -254,8 +264,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { ensureNoRebalance(consumer, listener) } - @Test - def testAutoCommitOnClose(): Unit = { + // Consumer group protocol temporarily does not commit offsets on consumer close + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testAutoCommitOnClose(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true") val consumer = createConsumer() @@ -277,8 +289,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(500, anotherConsumer.committed(Set(tp2).asJava).get(tp2).offset) } - @Test - def testAutoCommitOnCloseAfterWakeup(): Unit = { + // Consumer group protocol temporarily does not commit offsets on consumer close + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testAutoCommitOnCloseAfterWakeup(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true") val consumer = createConsumer() @@ -304,8 +318,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(500, anotherConsumer.committed(Set(tp2).asJava).get(tp2).offset) } - @Test - def testAutoOffsetReset(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAutoOffsetReset(quorum: String, groupProtocol: String): Unit = { val producer = createProducer() val startingTimestamp = System.currentTimeMillis() sendRecords(producer, numRecords = 1, tp, startingTimestamp = startingTimestamp) @@ -315,8 +330,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumeAndVerifyRecords(consumer = consumer, numRecords = 1, startingOffset = 0, startingTimestamp = startingTimestamp) } - @Test - def testGroupConsumption(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testGroupConsumption(quorum: String, groupProtocol: String): Unit = { val producer = createProducer() val startingTimestamp = System.currentTimeMillis() sendRecords(producer, numRecords = 10, tp, startingTimestamp = startingTimestamp) @@ -335,8 +351,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * metadata refresh the consumer becomes subscribed to this new topic and all partitions * of that topic are assigned to it. */ - @Test - def testPatternSubscription(): Unit = { + // Pattern subscriptions temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPatternSubscription(quorum: String, groupProtocol: String): Unit = { val numRecords = 10000 val producer = createProducer() sendRecords(producer, numRecords, tp) @@ -392,8 +410,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * The metadata refresh interval is intentionally increased to a large enough value to guarantee * that it is the subscription call that triggers a metadata refresh, and not the timeout. */ - @Test - def testSubsequentPatternSubscription(): Unit = { + // Pattern subscriptions temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testSubsequentPatternSubscription(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.METADATA_MAX_AGE_CONFIG, "30000") val consumer = createConsumer() @@ -443,8 +463,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * When consumer unsubscribes from all its subscriptions, it is expected that its * assignments are cleared right away. */ - @Test - def testPatternUnsubscription(): Unit = { + // Pattern subscriptions temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPatternUnsubscription(quorum: String, groupProtocol: String): Unit = { val numRecords = 10000 val producer = createProducer() sendRecords(producer, numRecords, tp) @@ -469,8 +491,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(0, consumer.assignment().size) } - @Test - def testCommitMetadata(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testCommitMetadata(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() consumer.assign(List(tp).asJava) @@ -490,8 +513,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(nullMetadata, consumer.committed(Set(tp).asJava).get(tp)) } - @Test - def testAsyncCommit(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAsyncCommit(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() consumer.assign(List(tp).asJava) @@ -509,8 +533,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(new OffsetAndMetadata(count), consumer.committed(Set(tp).asJava).get(tp)) } - @Test - def testExpandingTopicSubscriptions(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testExpandingTopicSubscriptions(quorum: String, groupProtocol: String): Unit = { val otherTopic = "other" val initialAssignment = Set(new TopicPartition(topic, 0), new TopicPartition(topic, 1)) val consumer = createConsumer() @@ -523,8 +548,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { awaitAssignment(consumer, expandedAssignment) } - @Test - def testShrinkingTopicSubscriptions(): Unit = { + // Consumer group protocol temporarily does not properly handle assignment change + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testShrinkingTopicSubscriptions(quorum: String, groupProtocol: String): Unit = { val otherTopic = "other" createTopic(otherTopic, 2, brokerCount) val initialAssignment = Set(new TopicPartition(topic, 0), new TopicPartition(topic, 1), new TopicPartition(otherTopic, 0), new TopicPartition(otherTopic, 1)) @@ -537,8 +564,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { awaitAssignment(consumer, shrunkenAssignment) } - @Test - def testPartitionsFor(): Unit = { + // partitionsFor not implemented in consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPartitionsFor(quorum: String, groupProtocol: String): Unit = { val numParts = 2 createTopic("part-test", numParts, 1) val consumer = createConsumer() @@ -547,23 +576,30 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(2, parts.size) } - @Test - def testPartitionsForAutoCreate(): Unit = { + // partitionsFor not implemented in consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPartitionsForAutoCreate(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() // First call would create the topic consumer.partitionsFor("non-exist-topic") - val partitions = consumer.partitionsFor("non-exist-topic") - assertFalse(partitions.isEmpty) + TestUtils.waitUntilTrue(() => { + !consumer.partitionsFor("non-exist-topic").isEmpty + }, s"Timed out while awaiting non empty partitions.") } - @Test - def testPartitionsForInvalidTopic(): Unit = { + // partitionsFor not implemented in consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPartitionsForInvalidTopic(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() assertThrows(classOf[InvalidTopicException], () => consumer.partitionsFor(";3# ads,{234")) } - @Test - def testSeek(): Unit = { + // Temporarily do not run flaky test for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testSeek(quorum: String, groupProtocol: String): Unit = { val consumer = createConsumer() val totalRecords = 50L val mid = totalRecords / 2 @@ -617,8 +653,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { producer.close() } - @Test - def testPositionAndCommit(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPositionAndCommit(quorum: String, groupProtocol: String): Unit = { val producer = createProducer() var startingTimestamp = System.currentTimeMillis() sendRecords(producer, numRecords = 5, tp, startingTimestamp = startingTimestamp) @@ -649,8 +686,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumeAndVerifyRecords(consumer = otherConsumer, numRecords = 1, startingOffset = 5, startingTimestamp = startingTimestamp) } - @Test - def testPartitionPauseAndResume(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPartitionPauseAndResume(quorum: String, groupProtocol: String): Unit = { val partitions = List(tp).asJava val producer = createProducer() var startingTimestamp = System.currentTimeMillis() @@ -667,8 +705,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumeAndVerifyRecords(consumer = consumer, numRecords = 5, startingOffset = 5, startingTimestamp = startingTimestamp) } - @Test - def testFetchInvalidOffset(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchInvalidOffset(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none") val consumer = createConsumer(configOverrides = this.consumerConfig) @@ -692,8 +731,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(outOfRangePos.toLong, outOfRangePartitions.get(tp)) } - @Test - def testFetchOutOfRangeOffsetResetConfigEarliest(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchOutOfRangeOffsetResetConfigEarliest(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") // ensure no in-flight fetch request so that the offset can be reset immediately this.consumerConfig.setProperty(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, "0") @@ -713,8 +753,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { } - @Test - def testFetchOutOfRangeOffsetResetConfigLatest(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchOutOfRangeOffsetResetConfigLatest(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest") // ensure no in-flight fetch request so that the offset can be reset immediately this.consumerConfig.setProperty(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, "0") @@ -739,8 +780,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(totalRecords, nextRecord.offset()) } - @Test - def testFetchRecordLargerThanFetchMaxBytes(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchRecordLargerThanFetchMaxBytes(quorum: String, groupProtocol: String): Unit = { val maxFetchBytes = 10 * 1024 this.consumerConfig.setProperty(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, maxFetchBytes.toString) checkLargeRecord(maxFetchBytes + 1) @@ -768,8 +810,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { } /** We should only return a large record if it's the first record in the first non-empty partition of the fetch request */ - @Test - def testFetchHonoursFetchSizeIfLargeRecordNotFirst(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchHonoursFetchSizeIfLargeRecordNotFirst(quorum: String, groupProtocol: String): Unit = { val maxFetchBytes = 10 * 1024 this.consumerConfig.setProperty(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, maxFetchBytes.toString) checkFetchHonoursSizeIfLargeRecordNotFirst(maxFetchBytes) @@ -800,23 +843,26 @@ class PlaintextConsumerTest extends BaseConsumerTest { } /** We should only return a large record if it's the first record in the first partition of the fetch request */ - @Test - def testFetchHonoursMaxPartitionFetchBytesIfLargeRecordNotFirst(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchHonoursMaxPartitionFetchBytesIfLargeRecordNotFirst(quorum: String, groupProtocol: String): Unit = { val maxPartitionFetchBytes = 10 * 1024 this.consumerConfig.setProperty(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes.toString) checkFetchHonoursSizeIfLargeRecordNotFirst(maxPartitionFetchBytes) } - @Test - def testFetchRecordLargerThanMaxPartitionFetchBytes(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testFetchRecordLargerThanMaxPartitionFetchBytes(quorum: String, groupProtocol: String): Unit = { val maxPartitionFetchBytes = 10 * 1024 this.consumerConfig.setProperty(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes.toString) checkLargeRecord(maxPartitionFetchBytes + 1) } /** Test that we consume all partitions if fetch max bytes and max.partition.fetch.bytes are low */ - @Test - def testLowMaxFetchSizeForRequestAndPartition(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testLowMaxFetchSizeForRequestAndPartition(quorum: String, groupProtocol: String): Unit = { // one of the effects of this is that there will be some log reads where `0 > remaining limit bytes < message size` // and we don't return the message because it's not the first message in the first non-empty partition of the fetch // this behaves a little different than when remaining limit bytes is 0 and it's important to test it @@ -863,8 +909,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(expected, actual) } - @Test - def testRoundRobinAssignment(): Unit = { + // Only the generic group protocol supports client-side assignors + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testRoundRobinAssignment(quorum: String, groupProtocol: String): Unit = { // 1 consumer using round-robin assignment this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "roundrobin-group") this.consumerConfig.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classOf[RoundRobinAssignor].getName) @@ -899,8 +947,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(0, consumer.assignment().size) } - @Test - def testMultiConsumerRoundRobinAssignor(): Unit = { + // Only the generic group protocol supports client-side assignors + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerRoundRobinAssignor(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "roundrobin-group") this.consumerConfig.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, classOf[RoundRobinAssignor].getName) @@ -936,8 +986,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * - (#par / 10) partition per consumer, where one partition from each of the early (#par mod 9) consumers * will move to consumer #10, leading to a total of (#par mod 9) partition movement */ - @Test - def testMultiConsumerStickyAssignor(): Unit = { + // Only the generic group protocol supports client-side assignors + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerStickyAssignor(quorum: String, groupProtocol: String): Unit = { def reverse(m: Map[Long, Set[TopicPartition]]) = m.values.toSet.flatten.map(v => (v, m.keys.filter(m(_).contains(v)).head)).toMap @@ -982,8 +1034,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * This test re-uses BaseConsumerTest's consumers. * As a result, it is testing the default assignment strategy set by BaseConsumerTest */ - @Test - def testMultiConsumerDefaultAssignor(): Unit = { + // Only the generic group protocol supports client-side assignors + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerDefaultAssignor(quorum: String, groupProtocol: String): Unit = { // use consumers and topics defined in this class + one more topic val producer = createProducer() sendRecords(producer, numRecords = 100, tp) @@ -1017,12 +1071,17 @@ class PlaintextConsumerTest extends BaseConsumerTest { } } + // Only the generic group protocol supports client-side assignors @ParameterizedTest - @ValueSource(strings = Array( - "org.apache.kafka.clients.consumer.CooperativeStickyAssignor", - "org.apache.kafka.clients.consumer.RangeAssignor")) - def testRebalanceAndRejoin(assignmentStrategy: String): Unit = { + @CsvSource(Array( + "org.apache.kafka.clients.consumer.CooperativeStickyAssignor, zk", + "org.apache.kafka.clients.consumer.RangeAssignor, zk", + "org.apache.kafka.clients.consumer.CooperativeStickyAssignor, kraft", + "org.apache.kafka.clients.consumer.RangeAssignor, kraft" + )) + def testRebalanceAndRejoin(assignmentStrategy: String, quorum: String): Unit = { // create 2 consumers + this.consumerConfig.setProperty(ConsumerConfig.GROUP_PROTOCOL_CONFIG, "generic") this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "rebalance-and-rejoin-group") this.consumerConfig.setProperty(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, assignmentStrategy) this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true") @@ -1105,8 +1164,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { * As a result, it is testing the default assignment strategy set by BaseConsumerTest * It tests the assignment results is expected using default assignor (i.e. Range assignor) */ - @Test - def testMultiConsumerDefaultAssignorAndVerifyAssignment(): Unit = { + // Only the generic group protocol supports client-side assignors + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerDefaultAssignorAndVerifyAssignment(quorum: String, groupProtocol: String): Unit = { // create two new topics, each having 3 partitions val topic1 = "topic1" val topic2 = "topic2" @@ -1137,18 +1198,24 @@ class PlaintextConsumerTest extends BaseConsumerTest { } } - @Test - def testMultiConsumerSessionTimeoutOnStopPolling(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerSessionTimeoutOnStopPolling(quorum: String, groupProtocol: String): Unit = { runMultiConsumerSessionTimeoutTest(false) } - @Test - def testMultiConsumerSessionTimeoutOnClose(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testMultiConsumerSessionTimeoutOnClose(quorum: String, groupProtocol: String): Unit = { runMultiConsumerSessionTimeoutTest(true) } - @Test - def testInterceptors(): Unit = { + // Consumer interceptors temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testInterceptors(quorum: String, groupProtocol: String): Unit = { val appendStr = "mock" MockConsumerInterceptor.resetCounters() MockProducerInterceptor.resetCounters() @@ -1206,8 +1273,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { MockProducerInterceptor.resetCounters() } - @Test - def testAutoCommitIntercept(): Unit = { + // Consumer interceptors temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testAutoCommitIntercept(quorum: String, groupProtocol: String): Unit = { val topic2 = "topic2" createTopic(topic2, 2, brokerCount) @@ -1256,8 +1325,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { MockConsumerInterceptor.resetCounters() } - @Test - def testInterceptorsWithWrongKeyValue(): Unit = { + // Consumer interceptors temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testInterceptorsWithWrongKeyValue(quorum: String, groupProtocol: String): Unit = { val appendStr = "mock" // create producer with interceptor that has different key and value types from the producer val producerProps = new Properties() @@ -1282,8 +1353,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(s"value will not be modified", new String(record.value())) } - @Test - def testConsumeMessagesWithCreateTime(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testConsumeMessagesWithCreateTime(quorum: String, groupProtocol: String): Unit = { val numRecords = 50 // Test non-compressed messages val producer = createProducer() @@ -1299,8 +1371,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumeAndVerifyRecords(consumer = consumer, numRecords = numRecords, tp = tp2, startingOffset = 0) } - @Test - def testConsumeMessagesWithLogAppendTime(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testConsumeMessagesWithLogAppendTime(quorum: String, groupProtocol: String): Unit = { val topicName = "testConsumeMessagesWithLogAppendTime" val topicProps = new Properties() topicProps.setProperty(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "LogAppendTime") @@ -1327,8 +1400,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { startingTimestamp = startTime, timestampType = TimestampType.LOG_APPEND_TIME) } - @Test - def testListTopics(): Unit = { + // listTopics temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testListTopics(quorum: String, groupProtocol: String): Unit = { val numParts = 2 val topic1 = "part-test-topic-1" val topic2 = "part-test-topic-2" @@ -1347,8 +1422,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(2, topics.get(topic3).size) } - @Test - def testUnsubscribeTopic(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testUnsubscribeTopic(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "100") // timeout quickly to avoid slow test this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "30") val consumer = createConsumer() @@ -1363,8 +1440,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(0, consumer.assignment.size()) } - @Test - def testPauseStateNotPreservedByRebalance(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPauseStateNotPreservedByRebalance(quorum: String, groupProtocol: String): Unit = { this.consumerConfig.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "100") // timeout quickly to avoid slow test this.consumerConfig.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "30") val consumer = createConsumer() @@ -1384,8 +1462,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumeAndVerifyRecords(consumer = consumer, numRecords = 0, startingOffset = 5, startingTimestamp = startingTimestamp) } - @Test - def testCommitSpecifiedOffsets(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testCommitSpecifiedOffsets(quorum: String, groupProtocol: String): Unit = { val producer = createProducer() sendRecords(producer, numRecords = 5, tp) sendRecords(producer, numRecords = 7, tp2) @@ -1411,8 +1490,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(7, consumer.committed(Set(tp2).asJava).get(tp2).offset) } - @Test - def testAutoCommitOnRebalance(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testAutoCommitOnRebalance(quorum: String, groupProtocol: String): Unit = { val topic2 = "topic2" createTopic(topic2, 2, brokerCount) @@ -1450,8 +1531,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(500, consumer.committed(Set(tp2).asJava).get(tp2).offset) } - @Test - def testPerPartitionLeadMetricsCleanUpWithSubscribe(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPerPartitionLeadMetricsCleanUpWithSubscribe(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 val topic2 = "topic2" createTopic(topic2, 2, brokerCount) @@ -1489,8 +1572,10 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertNull(consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags2))) } - @Test - def testPerPartitionLagMetricsCleanUpWithSubscribe(): Unit = { + // ConsumerRebalanceListener temporarily not supported for consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testPerPartitionLagMetricsCleanUpWithSubscribe(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 val topic2 = "topic2" createTopic(topic2, 2, brokerCount) @@ -1529,8 +1614,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertNull(consumer.metrics.get(new MetricName("records-lag", "consumer-fetch-manager-metrics", "", tags2))) } - @Test - def testPerPartitionLeadMetricsCleanUpWithAssign(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPerPartitionLeadMetricsCleanUpWithAssign(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 // Test assign // send some messages. @@ -1558,8 +1644,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertNull(consumer.metrics.get(new MetricName("records-lead", "consumer-fetch-manager-metrics", "", tags))) } - @Test - def testPerPartitionLagMetricsCleanUpWithAssign(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPerPartitionLagMetricsCleanUpWithAssign(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 // Test assign // send some messages. @@ -1589,8 +1676,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertNull(consumer.metrics.get(new MetricName("records-lag", "consumer-fetch-manager-metrics", "", tags))) } - @Test - def testPerPartitionLagMetricsWhenReadCommitted(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPerPartitionLagMetricsWhenReadCommitted(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 // send some messages. val producer = createProducer() @@ -1612,8 +1700,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertNotNull(fetchLag) } - @Test - def testPerPartitionLeadWithMaxPollRecords(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPerPartitionLeadWithMaxPollRecords(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 val maxPollRecords = 10 val producer = createProducer() @@ -1634,8 +1723,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(maxPollRecords, lead.metricValue().asInstanceOf[Double], s"The lead should be $maxPollRecords") } - @Test - def testPerPartitionLagWithMaxPollRecords(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testPerPartitionLagWithMaxPollRecords(quorum: String, groupProtocol: String): Unit = { val numMessages = 1000 val maxPollRecords = 10 val producer = createProducer() @@ -1657,8 +1747,9 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(numMessages - records.count, lag.metricValue.asInstanceOf[Double], epsilon, s"The lag should be ${numMessages - records.count}") } - @Test - def testQuotaMetricsNotCreatedIfNoQuotasConfigured(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testQuotaMetricsNotCreatedIfNoQuotasConfigured(quorum: String, groupProtocol: String): Unit = { val numRecords = 1000 val producer = createProducer() val startingTimestamp = System.currentTimeMillis() @@ -1669,7 +1760,7 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer.seek(tp, 0) consumeAndVerifyRecords(consumer = consumer, numRecords = numRecords, startingOffset = 0, startingTimestamp = startingTimestamp) - def assertNoMetric(broker: KafkaServer, name: String, quotaType: QuotaType, clientId: String): Unit = { + def assertNoMetric(broker: KafkaBroker, name: String, quotaType: QuotaType, clientId: String): Unit = { val metricName = broker.metrics.metricName("throttle-time", quotaType.toString, "", @@ -1677,15 +1768,15 @@ class PlaintextConsumerTest extends BaseConsumerTest { "client-id", clientId) assertNull(broker.metrics.metric(metricName), "Metric should not have been created " + metricName) } - servers.foreach(assertNoMetric(_, "byte-rate", QuotaType.Produce, producerClientId)) - servers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Produce, producerClientId)) - servers.foreach(assertNoMetric(_, "byte-rate", QuotaType.Fetch, consumerClientId)) - servers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Fetch, consumerClientId)) + brokers.foreach(assertNoMetric(_, "byte-rate", QuotaType.Produce, producerClientId)) + brokers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Produce, producerClientId)) + brokers.foreach(assertNoMetric(_, "byte-rate", QuotaType.Fetch, consumerClientId)) + brokers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Fetch, consumerClientId)) - servers.foreach(assertNoMetric(_, "request-time", QuotaType.Request, producerClientId)) - servers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Request, producerClientId)) - servers.foreach(assertNoMetric(_, "request-time", QuotaType.Request, consumerClientId)) - servers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Request, consumerClientId)) + brokers.foreach(assertNoMetric(_, "request-time", QuotaType.Request, producerClientId)) + brokers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Request, producerClientId)) + brokers.foreach(assertNoMetric(_, "request-time", QuotaType.Request, consumerClientId)) + brokers.foreach(assertNoMetric(_, "throttle-time", QuotaType.Request, consumerClientId)) } def runMultiConsumerSessionTimeoutTest(closeConsumer: Boolean): Unit = { @@ -1805,17 +1896,14 @@ class PlaintextConsumerTest extends BaseConsumerTest { s"The current assignment is ${consumer.assignment()}") } - @Test - def testConsumingWithNullGroupId(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testConsumingWithNullGroupId(quorum: String, groupProtocol: String): Unit = { val topic = "test_topic" - val partition = 0; + val partition = 0 val tp = new TopicPartition(topic, partition) createTopic(topic, 1, 1) - TestUtils.waitUntilTrue(() => { - this.zkClient.topicExists(topic) - }, "Failed to create topic") - val producer = createProducer() producer.send(new ProducerRecord(topic, partition, "k1".getBytes, "v1".getBytes)).get() producer.send(new ProducerRecord(topic, partition, "k2".getBytes, "v2".getBytes)).get() @@ -1870,17 +1958,29 @@ class PlaintextConsumerTest extends BaseConsumerTest { assertEquals(2, numRecords3, "Expected consumer3 to consume from offset 1") } - @Test - def testConsumingWithEmptyGroupId(): Unit = { + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testNullGroupIdNotSupportedIfCommitting(quorum: String, groupProtocol: String): Unit = { + val consumer1Config = new Properties(consumerConfig) + consumer1Config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + consumer1Config.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer1") + val consumer1 = createConsumer( + configOverrides = consumer1Config, + configsToRemove = List(ConsumerConfig.GROUP_ID_CONFIG)) + + consumer1.assign(List(tp).asJava) + assertThrows(classOf[InvalidGroupIdException], () => consumer1.commitSync()) + } + + // Empty group ID only supported for generic group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testConsumingWithEmptyGroupId(quorum: String, groupProtocol: String): Unit = { val topic = "test_topic" - val partition = 0; + val partition = 0 val tp = new TopicPartition(topic, partition) createTopic(topic, 1, 1) - TestUtils.waitUntilTrue(() => { - this.zkClient.topicExists(topic) - }, "Failed to create topic") - val producer = createProducer() producer.send(new ProducerRecord(topic, partition, "k1".getBytes, "v1".getBytes)).get() producer.send(new ProducerRecord(topic, partition, "k2".getBytes, "v2".getBytes)).get() @@ -1919,8 +2019,24 @@ class PlaintextConsumerTest extends BaseConsumerTest { "Expected consumer2 to consume one message from offset 1, which is the committed offset of consumer1") } - @Test - def testStaticConsumerDetectsNewPartitionCreatedAfterRestart(): Unit = { + // Empty group ID not supported with consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @CsvSource(Array( + "kraft+kip848, consumer" + )) + def testEmptyGroupIdNotSupported(quorum:String, groupProtocol: String): Unit = { + val consumer1Config = new Properties(consumerConfig) + consumer1Config.put(ConsumerConfig.GROUP_ID_CONFIG, "") + consumer1Config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + consumer1Config.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer1") + + assertThrows(classOf[KafkaException], () => createConsumer(configOverrides = consumer1Config)) + } + + // Static membership temporarily not supported in consumer group protocol + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersGenericGroupProtocolOnly")) + def testStaticConsumerDetectsNewPartitionCreatedAfterRestart(quorum:String, groupProtocol: String): Unit = { val foo = "foo" val foo0 = new TopicPartition(foo, 0) val foo1 = new TopicPartition(foo, 1) @@ -1947,4 +2063,209 @@ class PlaintextConsumerTest extends BaseConsumerTest { consumer2.close() } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndCommitAsyncNotCommitted(quorum:String, groupProtocol: String): Unit = { + val props = new Properties() + val consumer = createConsumer(configOverrides = props) + val producer = createProducer() + val numRecords = 10000 + val startingTimestamp = System.currentTimeMillis() + val cb = new CountConsumerCommitCallback + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + consumer.assign(List(tp).asJava) + consumer.commitAsync(cb) + TestUtils.pollUntilTrue(consumer, () => cb.successCount >= 1 || cb.lastError.isDefined, + "Failed to observe commit callback before timeout", waitTimeMs = 10000) + val committedOffset = consumer.committed(Set(tp).asJava) + assertNotNull(committedOffset) + // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to + // tp. The committed offset should be null. This is intentional. + assertNull(committedOffset.get(tp)) + assertTrue(consumer.assignment.contains(tp)) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndCommitSyncNotCommitted(quorum:String, groupProtocol: String): Unit = { + val props = new Properties() + val consumer = createConsumer(configOverrides = props) + val producer = createProducer() + val numRecords = 10000 + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + consumer.assign(List(tp).asJava) + consumer.commitSync() + val committedOffset = consumer.committed(Set(tp).asJava) + assertNotNull(committedOffset) + // No valid fetch position due to the absence of consumer.poll; and therefore no offset was committed to + // tp. The committed offset should be null. This is intentional. + assertNull(committedOffset.get(tp)) + assertTrue(consumer.assignment.contains(tp)) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndCommitSyncAllConsumed(quorum:String, groupProtocol: String): Unit = { + val numRecords = 10000 + + val producer = createProducer() + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + + val props = new Properties() + val consumer = createConsumer(configOverrides = props) + consumer.assign(List(tp).asJava) + consumer.seek(tp, 0) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 0, startingTimestamp = startingTimestamp) + + consumer.commitSync() + val committedOffset = consumer.committed(Set(tp).asJava) + assertNotNull(committedOffset) + assertNotNull(committedOffset.get(tp)) + assertEquals(numRecords, committedOffset.get(tp).offset()) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndConsume(quorum:String, groupProtocol: String): Unit = { + val numRecords = 10 + + val producer = createProducer() + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + + val props = new Properties() + val consumer = createConsumer(configOverrides = props, + configsToRemove = List(ConsumerConfig.GROUP_ID_CONFIG)) + consumer.assign(List(tp).asJava) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 0, startingTimestamp = startingTimestamp) + + assertEquals(numRecords, consumer.position(tp)) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndConsumeSkippingPosition(quorum:String, groupProtocol: String): Unit = { + val numRecords = 10 + + val producer = createProducer() + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + + val props = new Properties() + val consumer = createConsumer(configOverrides = props, + configsToRemove = List(ConsumerConfig.GROUP_ID_CONFIG)) + consumer.assign(List(tp).asJava) + val offset = 1 + consumer.seek(tp, offset) + consumeAndVerifyRecords(consumer = consumer, numRecords - offset, startingOffset = offset, + startingKeyAndValueIndex = offset, startingTimestamp = startingTimestamp + offset) + + assertEquals(numRecords, consumer.position(tp)) + } + + // partitionsFor not implemented in consumer group protocol and this test requires ZK also + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @CsvSource(Array( + "zk, generic" + )) + def testAssignAndConsumeWithLeaderChangeValidatingPositions(quorum:String, groupProtocol: String): Unit = { + val numRecords = 10 + val producer = createProducer() + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + val props = new Properties() + val consumer = createConsumer(configOverrides = props, + configsToRemove = List(ConsumerConfig.GROUP_ID_CONFIG)) + consumer.assign(List(tp).asJava) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 0, startingTimestamp = startingTimestamp) + + // Force leader epoch change to trigger position validation + var parts: mutable.Buffer[PartitionInfo] = null + while (parts == null) + parts = consumer.partitionsFor(tp.topic()).asScala + val leader = parts.head.leader().id() + this.servers(leader).shutdown() + this.servers(leader).startup() + + // Consume after leader change + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 10, + startingTimestamp = startingTimestamp) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndFetchCommittedOffsets(quorum:String, groupProtocol: String): Unit = { + val numRecords = 100 + val startingTimestamp = System.currentTimeMillis() + val producer = createProducer() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + val props = new Properties() + val consumer = createConsumer(configOverrides = props) + consumer.assign(List(tp).asJava) + // First consumer consumes and commits offsets + consumer.seek(tp, 0) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 0, + startingTimestamp = startingTimestamp) + consumer.commitSync() + assertEquals(numRecords, consumer.committed(Set(tp).asJava).get(tp).offset) + // We should see the committed offsets from another consumer + val anotherConsumer = createConsumer(configOverrides = props) + anotherConsumer.assign(List(tp).asJava) + assertEquals(numRecords, anotherConsumer.committed(Set(tp).asJava).get(tp).offset) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndConsumeFromCommittedOffsets(quorum:String, groupProtocol: String): Unit = { + val producer = createProducer() + val numRecords = 100 + val startingTimestamp = System.currentTimeMillis() + sendRecords(producer, numRecords = numRecords, tp, startingTimestamp = startingTimestamp) + + // Commit offset with first consumer + val props = new Properties() + props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "group1") + val consumer = createConsumer(configOverrides = props) + consumer.assign(List(tp).asJava) + val offset = 10 + consumer.commitSync(Map[TopicPartition, OffsetAndMetadata]((tp, new OffsetAndMetadata(offset))) + .asJava) + assertEquals(offset, consumer.committed(Set(tp).asJava).get(tp).offset) + consumer.close() + + // Consume from committed offsets with another consumer in same group + val anotherConsumer = createConsumer(configOverrides = props) + assertEquals(offset, anotherConsumer.committed(Set(tp).asJava).get(tp).offset) + anotherConsumer.assign(List(tp).asJava) + consumeAndVerifyRecords(consumer = anotherConsumer, numRecords - offset, + startingOffset = offset, startingKeyAndValueIndex = offset, + startingTimestamp = startingTimestamp + offset) + } + + @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumAndGroupProtocolNames) + @MethodSource(Array("getTestQuorumAndGroupProtocolParametersAll")) + def testAssignAndRetrievingCommittedOffsetsMultipleTimes(quorum:String, groupProtocol: String): Unit = { + val numRecords = 100 + val startingTimestamp = System.currentTimeMillis() + val producer = createProducer() + sendRecords(producer, numRecords, tp, startingTimestamp = startingTimestamp) + + val props = new Properties() + val consumer = createConsumer(configOverrides = props) + consumer.assign(List(tp).asJava) + + // Consume and commit offsets + consumer.seek(tp, 0) + consumeAndVerifyRecords(consumer = consumer, numRecords, startingOffset = 0, + startingTimestamp = startingTimestamp) + consumer.commitSync() + + // Check committed offsets twice with same consumer + assertEquals(numRecords, consumer.committed(Set(tp).asJava).get(tp).offset) + assertEquals(numRecords, consumer.committed(Set(tp).asJava).get(tp).offset) + } } diff --git a/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala index 16f31c406f917..b2937c5366f44 100644 --- a/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslMultiMechanismConsumerTest.scala @@ -13,12 +13,13 @@ package kafka.api import kafka.server.KafkaConfig -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo, Timeout} import kafka.utils.{JaasTestUtils, TestUtils} import org.apache.kafka.common.security.auth.SecurityProtocol import scala.jdk.CollectionConverters._ +@Timeout(600) class SaslMultiMechanismConsumerTest extends BaseConsumerTest with SaslSetup { private val kafkaClientSaslMechanism = "PLAIN" private val kafkaServerSaslMechanisms = List("GSSAPI", "PLAIN") diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala index 0ccb5f2ffc146..6a03d51aa3680 100644 --- a/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslPlainPlaintextConsumerTest.scala @@ -13,13 +13,13 @@ package kafka.api import java.util.Locale - import kafka.server.KafkaConfig import kafka.utils.{JaasTestUtils, TestUtils} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.security.auth.SecurityProtocol -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo, Timeout} +@Timeout(600) class SaslPlainPlaintextConsumerTest extends BaseConsumerTest with SaslSetup { override protected def listenerName = new ListenerName("CLIENT") private val kafkaClientSaslMechanism = "PLAIN" diff --git a/core/src/test/scala/integration/kafka/api/SaslPlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlaintextConsumerTest.scala index 0933818830c90..cfe245fabe04f 100644 --- a/core/src/test/scala/integration/kafka/api/SaslPlaintextConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslPlaintextConsumerTest.scala @@ -14,8 +14,9 @@ package kafka.api import kafka.utils.JaasTestUtils import org.apache.kafka.common.security.auth.SecurityProtocol -import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo} +import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo, Timeout} +@Timeout(600) class SaslPlaintextConsumerTest extends BaseConsumerTest with SaslSetup { override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT diff --git a/core/src/test/scala/integration/kafka/api/SaslSslConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SaslSslConsumerTest.scala index c53db2368bdb6..0ea458e25aa0a 100644 --- a/core/src/test/scala/integration/kafka/api/SaslSslConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslSslConsumerTest.scala @@ -15,8 +15,9 @@ package kafka.api import kafka.server.KafkaConfig import kafka.utils.{JaasTestUtils, TestUtils} import org.apache.kafka.common.security.auth.SecurityProtocol -import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo} +import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo, Timeout} +@Timeout(600) class SaslSslConsumerTest extends BaseConsumerTest with SaslSetup { this.serverConfig.setProperty(KafkaConfig.ZkEnableSecureAclsProp, "true") override protected def securityProtocol = SecurityProtocol.SASL_SSL diff --git a/core/src/test/scala/integration/kafka/api/SslConsumerTest.scala b/core/src/test/scala/integration/kafka/api/SslConsumerTest.scala index d6984da5b327c..39d5b0716d61d 100644 --- a/core/src/test/scala/integration/kafka/api/SslConsumerTest.scala +++ b/core/src/test/scala/integration/kafka/api/SslConsumerTest.scala @@ -13,9 +13,10 @@ package kafka.api import kafka.utils.TestUtils - import org.apache.kafka.common.security.auth.SecurityProtocol +import org.junit.jupiter.api.Timeout +@Timeout(600) class SslConsumerTest extends BaseConsumerTest { override protected def securityProtocol = SecurityProtocol.SSL override protected lazy val trustStoreFile = Some(TestUtils.tempFile("truststore", ".jks")) diff --git a/core/src/test/scala/integration/kafka/server/FetchFromFollowerIntegrationTest.scala b/core/src/test/scala/integration/kafka/server/FetchFromFollowerIntegrationTest.scala index d15d0147608d4..b5495dc2bd8f0 100644 --- a/core/src/test/scala/integration/kafka/server/FetchFromFollowerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/server/FetchFromFollowerIntegrationTest.scala @@ -26,7 +26,7 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.FetchResponse import org.apache.kafka.common.serialization.ByteArrayDeserializer import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} -import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.{Disabled, Timeout} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource @@ -179,6 +179,7 @@ class FetchFromFollowerIntegrationTest extends BaseFetchRequestTest { } } + @Disabled @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) @ValueSource(strings = Array("zk", "kraft")) def testRackAwareRangeAssignor(quorum: String): Unit = { diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala index 836c1b1cfdc96..e25ae662a337a 100644 --- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala +++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala @@ -984,6 +984,11 @@ class KRaftClusterTest { val countAfterTenIntervals = snapshotCounter(metaLog) assertTrue(countAfterTenIntervals > 1, s"Expected to see at least one more snapshot, saw $countAfterTenIntervals") assertTrue(countAfterTenIntervals < 20, s"Did not expect to see more than twice as many snapshots as snapshot intervals, saw $countAfterTenIntervals") + TestUtils.waitUntilTrue(() => { + val emitterMetrics = cluster.controllers().values().iterator().next(). + sharedServer.snapshotEmitter.metrics() + emitterMetrics.latestSnapshotGeneratedBytes() > 0 + }, "Failed to see latestSnapshotGeneratedBytes > 0") } finally { cluster.close() } diff --git a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala index 19836ac299ca5..c670a7e9db022 100755 --- a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala +++ b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala @@ -17,17 +17,17 @@ package kafka.server -import java.io.{ByteArrayOutputStream, File, PrintStream} +import java.io.File import java.net.InetSocketAddress import java.util import java.util.{Collections, Optional, OptionalInt, Properties} import java.util.concurrent.{CompletableFuture, TimeUnit} import javax.security.auth.login.Configuration -import kafka.tools.StorageTool import kafka.utils.{CoreUtils, Logging, TestInfoUtils, TestUtils} import kafka.zk.{AdminZkClient, EmbeddedZookeeper, KafkaZkClient} +import org.apache.kafka.clients.consumer.GroupProtocol import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.Uuid +import org.apache.kafka.common.{DirectoryId, Uuid} import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{Exit, Time} @@ -110,6 +110,7 @@ class KRaftQuorumImplementation( setVersion(MetaPropertiesVersion.V1). setClusterId(clusterId). setNodeId(config.nodeId). + setDirectoryId(DirectoryId.random()). build()) }) copier.setPreWriteHandler((logDir, _, _) => { @@ -189,6 +190,14 @@ abstract class QuorumTestHarness extends Logging { TestInfoUtils.isZkMigrationTest(testInfo) } + def isNewGroupCoordinatorEnabled(): Boolean = { + TestInfoUtils.isNewGroupCoordinatorEnabled(testInfo) + } + + def maybeGroupProtocolSpecified(testInfo: TestInfo): Option[GroupProtocol] = { + TestInfoUtils.maybeGroupProtocolSpecified(testInfo) + } + def checkIsZKTest(): Unit = { if (isKRaftTest()) { throw new RuntimeException("This function can't be accessed when running the test " + @@ -300,25 +309,6 @@ abstract class QuorumTestHarness extends Logging { def optionalMetadataRecords: Option[ArrayBuffer[ApiMessageAndVersion]] = None - private def formatDirectories(directories: immutable.Seq[String], - metaProperties: MetaProperties): Unit = { - val stream = new ByteArrayOutputStream() - var out: PrintStream = null - try { - out = new PrintStream(stream) - val bootstrapMetadata = StorageTool.buildBootstrapMetadata(metadataVersion, - optionalMetadataRecords, "format command") - if (StorageTool.formatCommand(out, directories, metaProperties, bootstrapMetadata, metadataVersion, - ignoreFormatted = false) != 0) { - throw new RuntimeException(stream.toString()) - } - debug(s"Formatted storage directory(ies) ${directories}") - } finally { - if (out != null) out.close() - stream.close() - } - } - private def newKRaftQuorum(testInfo: TestInfo): KRaftQuorumImplementation = { val propsList = kraftControllerConfigs() if (propsList.size != 1) { @@ -327,6 +317,7 @@ abstract class QuorumTestHarness extends Logging { val props = propsList(0) props.setProperty(KafkaConfig.ServerMaxStartupTimeMsProp, TimeUnit.MINUTES.toMillis(10).toString) props.setProperty(KafkaConfig.ProcessRolesProp, "controller") + props.setProperty(KafkaConfig.UnstableMetadataVersionsEnableProp, "true") if (props.getProperty(KafkaConfig.NodeIdProp) == null) { props.setProperty(KafkaConfig.NodeIdProp, "1000") } @@ -337,7 +328,7 @@ abstract class QuorumTestHarness extends Logging { setClusterId(Uuid.randomUuid().toString). setNodeId(nodeId). build() - formatDirectories(immutable.Seq(metadataDir.getAbsolutePath), metaProperties) + TestUtils.formatDirectories(immutable.Seq(metadataDir.getAbsolutePath), metaProperties, metadataVersion, optionalMetadataRecords) val metadataRecords = new util.ArrayList[ApiMessageAndVersion] metadataRecords.add(new ApiMessageAndVersion(new FeatureLevelRecord(). diff --git a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala index e8417f6a87295..6be7f6b422d79 100644 --- a/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/zk/ZkMigrationIntegrationTest.scala @@ -17,7 +17,7 @@ package kafka.zk import kafka.security.authorizer.AclEntry.{WildcardHost, WildcardPrincipalString} -import kafka.server.{ConfigType, ControllerRequestCompletionHandler, KafkaConfig} +import kafka.server.{ConfigType, KafkaConfig} import kafka.test.{ClusterConfig, ClusterGenerator, ClusterInstance} import kafka.test.annotation.{AutoStart, ClusterConfigProperty, ClusterTemplate, ClusterTest, Type} import kafka.test.junit.ClusterTestExtensions @@ -31,7 +31,7 @@ import org.apache.kafka.common.acl.AclOperation.{DESCRIBE, READ, WRITE} import org.apache.kafka.common.acl.AclPermissionType.ALLOW import org.apache.kafka.common.acl.{AccessControlEntry, AclBinding} import org.apache.kafka.common.config.{ConfigResource, TopicConfig} -import org.apache.kafka.common.errors.TimeoutException +import org.apache.kafka.common.errors.{TimeoutException, UnknownTopicOrPartitionException} import org.apache.kafka.common.message.AllocateProducerIdsRequestData import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity} import org.apache.kafka.common.requests.{AllocateProducerIdsRequest, AllocateProducerIdsResponse} @@ -45,14 +45,15 @@ import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.metadata.authorizer.StandardAcl import org.apache.kafka.metadata.migration.ZkMigrationLeadershipState import org.apache.kafka.raft.RaftConfig +import org.apache.kafka.server.ControllerRequestCompletionHandler import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion, ProducerIdsBlock} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotEquals, assertNotNull, assertTrue} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertNotEquals, assertNotNull, assertTrue, fail} import org.junit.jupiter.api.{Assumptions, Timeout} import org.junit.jupiter.api.extension.ExtendWith import org.slf4j.LoggerFactory import java.util -import java.util.concurrent.{CompletableFuture, TimeUnit} +import java.util.concurrent.{CompletableFuture, ExecutionException, TimeUnit} import java.util.{Properties, UUID} import scala.collection.Seq import scala.jdk.CollectionConverters._ @@ -71,7 +72,9 @@ object ZkMigrationIntegrationTest { MetadataVersion.IBP_3_5_IV2, MetadataVersion.IBP_3_6_IV2, MetadataVersion.IBP_3_7_IV0, - MetadataVersion.IBP_3_7_IV1 + MetadataVersion.IBP_3_7_IV1, + MetadataVersion.IBP_3_7_IV2, + MetadataVersion.IBP_3_7_IV3 ).foreach { mv => val clusterConfig = ClusterConfig.defaultClusterBuilder() .metadataVersion(mv) @@ -279,8 +282,6 @@ class ZkMigrationIntegrationTest { newTopics.add(new NewTopic("test-topic-1", 10, 3.toShort)) newTopics.add(new NewTopic("test-topic-2", 10, 3.toShort)) newTopics.add(new NewTopic("test-topic-3", 10, 3.toShort)) - newTopics.add(new NewTopic("test-topic-4", 10, 3.toShort)) - newTopics.add(new NewTopic("test-topic-5", 10, 3.toShort)) val createTopicResult = admin.createTopics(newTopics) createTopicResult.all().get(300, TimeUnit.SECONDS) admin.close() @@ -302,11 +303,6 @@ class ZkMigrationIntegrationTest { kraftCluster.startup() val readyFuture = kraftCluster.controllers().values().asScala.head.controller.waitForReadyBrokers(3) - // Start a deletion that will take some time, but don't wait for it - admin = zkCluster.createAdminClient() - admin.deleteTopics(Seq("test-topic-1", "test-topic-2", "test-topic-3", "test-topic-4", "test-topic-5").asJava) - admin.close() - // Enable migration configs and restart brokers log.info("Restart brokers in migration mode") zkCluster.config().serverProperties().put(KafkaConfig.MigrationEnabledProp, "true") @@ -315,12 +311,17 @@ class ZkMigrationIntegrationTest { zkCluster.config().serverProperties().put(KafkaConfig.ListenerSecurityProtocolMapProp, "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT") zkCluster.rollingBrokerRestart() + // Emulate a ZK topic deletion + zkClient.createDeleteTopicPath("test-topic-1") + zkClient.createDeleteTopicPath("test-topic-2") + zkClient.createDeleteTopicPath("test-topic-3") + zkCluster.waitForReadyBrokers() readyFuture.get(60, TimeUnit.SECONDS) // Only continue with the test if there are some pending deletions to verify. If there are not any pending // deletions, this will mark the test as "skipped" instead of failed. - val topicDeletions = zkCluster.asInstanceOf[ZkClusterInstance].getUnderlying.zkClient.getTopicDeletions + val topicDeletions = zkClient.getTopicDeletions Assumptions.assumeTrue(topicDeletions.nonEmpty, "This test needs pending topic deletions after a migration in order to verify the behavior") @@ -333,11 +334,21 @@ class ZkMigrationIntegrationTest { // At this point, some of the topics may have been deleted by ZK controller and the rest will be // implicitly deleted by the KRaft controller and remove from the ZK brokers as stray partitions + def topicsAllDeleted(admin: Admin): Boolean = { + val topics = admin.listTopics().names().get(60, TimeUnit.SECONDS) + topics.retainAll(util.Arrays.asList( + "test-topic-1", "test-topic-2", "test-topic-3" + )) + topics.isEmpty + } + admin = zkCluster.createAdminClient() + log.info("Waiting for topics to be deleted") TestUtils.waitUntilTrue( - () => admin.listTopics().names().get(60, TimeUnit.SECONDS).isEmpty, + () => topicsAllDeleted(admin), "Timed out waiting for topics to be deleted", - 300000) + 30000, + 1000) val newTopics = new util.ArrayList[NewTopic]() newTopics.add(new NewTopic("test-topic-1", 2, 3.toShort)) @@ -346,18 +357,33 @@ class ZkMigrationIntegrationTest { val createTopicResult = admin.createTopics(newTopics) createTopicResult.all().get(60, TimeUnit.SECONDS) - val expectedNewTopics = Seq("test-topic-1", "test-topic-2", "test-topic-3") + def topicsAllRecreated(admin: Admin): Boolean = { + val topics = admin.listTopics().names().get(60, TimeUnit.SECONDS) + topics.retainAll(util.Arrays.asList( + "test-topic-1", "test-topic-2", "test-topic-3" + )) + topics.size() == 3 + } + + log.info("Waiting for topics to be re-created") TestUtils.waitUntilTrue( - () => admin.listTopics().names().get(60, TimeUnit.SECONDS).equals(expectedNewTopics.toSet.asJava), + () => topicsAllRecreated(admin), "Timed out waiting for topics to be created", - 300000) + 30000, + 1000) TestUtils.retry(300000) { // Need a retry here since topic metadata may be inconsistent between brokers - val topicDescriptions = admin.describeTopics(expectedNewTopics.asJavaCollection) - .topicNameValues().asScala.map { case (name, description) => + val topicDescriptions = try { + admin.describeTopics(util.Arrays.asList( + "test-topic-1", "test-topic-2", "test-topic-3" + )).topicNameValues().asScala.map { case (name, description) => name -> description.get(60, TimeUnit.SECONDS) - }.toMap + }.toMap + } catch { + case e: ExecutionException if e.getCause.isInstanceOf[UnknownTopicOrPartitionException] => Map.empty[String, TopicDescription] + case t: Throwable => fail("Error describing topics", t.getCause) + } assertEquals(2, topicDescriptions("test-topic-1").partitions().size()) assertEquals(1, topicDescriptions("test-topic-2").partitions().size()) @@ -373,8 +399,6 @@ class ZkMigrationIntegrationTest { assertTrue(absentTopics.contains("test-topic-1")) assertTrue(absentTopics.contains("test-topic-2")) assertTrue(absentTopics.contains("test-topic-3")) - assertFalse(absentTopics.contains("test-topic-4")) - assertFalse(absentTopics.contains("test-topic-5")) } admin.close() diff --git a/core/src/test/scala/kafka/server/NodeToControllerRequestThreadTest.scala b/core/src/test/scala/kafka/server/NodeToControllerRequestThreadTest.scala index 01432bf6f4727..5b98fe4d9c853 100644 --- a/core/src/test/scala/kafka/server/NodeToControllerRequestThreadTest.scala +++ b/core/src/test/scala/kafka/server/NodeToControllerRequestThreadTest.scala @@ -31,6 +31,7 @@ import org.apache.kafka.common.requests.{AbstractRequest, EnvelopeRequest, Envel import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder import org.apache.kafka.common.utils.MockTime +import org.apache.kafka.server.ControllerRequestCompletionHandler import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import org.mockito.Mockito._ diff --git a/core/src/test/scala/kafka/utils/TestInfoUtils.scala b/core/src/test/scala/kafka/utils/TestInfoUtils.scala index ba93fa36b9944..c82a654b228bb 100644 --- a/core/src/test/scala/kafka/utils/TestInfoUtils.scala +++ b/core/src/test/scala/kafka/utils/TestInfoUtils.scala @@ -21,6 +21,7 @@ import java.util import java.util.{Collections, Optional} import org.junit.jupiter.api.TestInfo +import org.apache.kafka.clients.consumer.GroupProtocol class EmptyTestInfo extends TestInfo { override def getDisplayName: String = "" @@ -51,5 +52,20 @@ object TestInfoUtils { testInfo.getDisplayName().contains("quorum=zkMigration") } } - final val TestWithParameterizedQuorumName = "{displayName}.quorum={0}" + final val TestWithParameterizedQuorumName = "{displayName}.{argumentsWithNames}" + + final val TestWithParameterizedQuorumAndGroupProtocolNames = "{displayName}.quorum={0}.groupProtocol={1}" + + def isNewGroupCoordinatorEnabled(testInfo: TestInfo): Boolean = { + testInfo.getDisplayName().contains("kraft+kip848") + } + + def maybeGroupProtocolSpecified(testInfo: TestInfo): Option[GroupProtocol] = { + if (testInfo.getDisplayName().contains("groupProtocol=generic")) + Some(GroupProtocol.GENERIC) + else if (testInfo.getDisplayName().contains("groupProtocol=consumer")) + Some(GroupProtocol.CONSUMER) + else + None + } } diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index 12e460a0bd3a0..069b79a5e0dad 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -1628,6 +1628,123 @@ class ConfigCommandTest extends Logging { Seq("/clients/client-3", sanitizedPrincipal + "/clients/client-2")) } + @Test + def shouldAlterClientMetricsConfig(): Unit = { + val node = new Node(1, "localhost", 9092) + verifyAlterClientMetricsConfig(node, "1", List("--entity-name", "1")) + } + + private def verifyAlterClientMetricsConfig(node: Node, resourceName: String, resourceOpts: List[String]): Unit = { + val optsList = List("--bootstrap-server", "localhost:9092", + "--entity-type", "client-metrics", + "--alter", + "--delete-config", "interval.ms", + "--add-config", "metrics=org.apache.kafka.consumer.," + + "match=[client_software_name=kafka.python,client_software_version=1\\.2\\..*]") ++ resourceOpts + val alterOpts = new ConfigCommandOptions(optsList.toArray) + + val resource = new ConfigResource(ConfigResource.Type.CLIENT_METRICS, resourceName) + val configEntries = util.Collections.singletonList(new ConfigEntry("interval.ms", "1000", + ConfigEntry.ConfigSource.DYNAMIC_CLIENT_METRICS_CONFIG, false, false, util.Collections.emptyList[ConfigEntry.ConfigSynonym], + ConfigEntry.ConfigType.UNKNOWN, null)) + val future = new KafkaFutureImpl[util.Map[ConfigResource, Config]] + future.complete(util.Collections.singletonMap(resource, new Config(configEntries))) + val describeResult: DescribeConfigsResult = mock(classOf[DescribeConfigsResult]) + when(describeResult.all()).thenReturn(future) + + val alterFuture = new KafkaFutureImpl[Void] + alterFuture.complete(null) + val alterResult: AlterConfigsResult = mock(classOf[AlterConfigsResult]) + when(alterResult.all()).thenReturn(alterFuture) + + val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) { + override def describeConfigs(resources: util.Collection[ConfigResource], options: DescribeConfigsOptions): DescribeConfigsResult = { + assertFalse(options.includeSynonyms(), "Config synonyms requested unnecessarily") + assertEquals(1, resources.size) + val resource = resources.iterator.next + assertEquals(ConfigResource.Type.CLIENT_METRICS, resource.`type`) + assertEquals(resourceName, resource.name) + describeResult + } + + override def incrementalAlterConfigs(configs: util.Map[ConfigResource, util.Collection[AlterConfigOp]], options: AlterConfigsOptions): AlterConfigsResult = { + assertEquals(1, configs.size) + val entry = configs.entrySet.iterator.next + val resource = entry.getKey + val alterConfigOps = entry.getValue + assertEquals(ConfigResource.Type.CLIENT_METRICS, resource.`type`) + assertEquals(3, alterConfigOps.size) + + val expectedConfigOps = List( + new AlterConfigOp(new ConfigEntry("match", "client_software_name=kafka.python,client_software_version=1\\.2\\..*"), AlterConfigOp.OpType.SET), + new AlterConfigOp(new ConfigEntry("metrics", "org.apache.kafka.consumer."), AlterConfigOp.OpType.SET), + new AlterConfigOp(new ConfigEntry("interval.ms", ""), AlterConfigOp.OpType.DELETE) + ) + assertEquals(expectedConfigOps, alterConfigOps.asScala.toList) + alterResult + } + } + ConfigCommand.alterConfig(mockAdminClient, alterOpts) + verify(describeResult).all() + verify(alterResult).all() + } + + @Test + def shouldNotDescribeClientMetricsConfigWithoutEntityName(): Unit = { + val describeOpts = new ConfigCommandOptions(Array("--bootstrap-server", "localhost:9092", + "--entity-type", "client-metrics", + "--describe")) + + val exception = assertThrows(classOf[IllegalArgumentException], () => describeOpts.checkArgs()) + assertEquals("an entity name must be specified with --describe of client-metrics", exception.getMessage) + } + + @Test + def shouldNotAlterClientMetricsConfigWithoutEntityName(): Unit = { + val alterOpts = new ConfigCommandOptions(Array("--bootstrap-server", "localhost:9092", + "--entity-type", "client-metrics", + "--alter", + "--add-config", "interval.ms=1000")) + + val exception = assertThrows(classOf[IllegalArgumentException], () => alterOpts.checkArgs()) + assertEquals("an entity name must be specified with --alter of client-metrics", exception.getMessage) + } + + @Test + def shouldNotSupportAlterClientMetricsWithZookeeperArg(): Unit = { + val alterOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, + "--entity-name", "sub", + "--entity-type", "client-metrics", + "--alter", + "--add-config", "interval.ms=1000")) + + val exception = assertThrows(classOf[IllegalArgumentException], () => alterOpts.checkArgs()) + assertEquals("Invalid entity type client-metrics, the entity type must be one of users, brokers with a --zookeeper argument", exception.getMessage) + } + + @Test + def shouldNotSupportDescribeClientMetricsWithZookeeperArg(): Unit = { + val describeOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, + "--entity-name", "sub", + "--entity-type", "client-metrics", + "--describe")) + + val exception = assertThrows(classOf[IllegalArgumentException], () => describeOpts.checkArgs()) + assertEquals("Invalid entity type client-metrics, the entity type must be one of users, brokers with a --zookeeper argument", exception.getMessage) + } + + @Test + def shouldNotSupportAlterClientMetricsWithZookeeper(): Unit = { + val alterOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, + "--entity-name", "sub", + "--entity-type", "client-metrics", + "--alter", + "--add-config", "interval.ms=1000")) + + val exception = assertThrows(classOf[IllegalArgumentException], () => ConfigCommand.alterConfigWithZk(null, alterOpts, dummyAdminZkClient)) + assertEquals("client-metrics is not a known entityType. Should be one of List(topics, clients, users, brokers, ips)", exception.getMessage) + } + class DummyAdminZkClient(zkClient: KafkaZkClient) extends AdminZkClient(zkClient) { override def changeBrokerConfig(brokerIds: Seq[Int], configs: Properties): Unit = {} override def fetchEntityConfig(entityType: String, entityName: String): Properties = {new Properties} diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala index afa9b8b6e0bff..b908fcc5e59d7 100644 --- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala +++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala @@ -51,6 +51,7 @@ import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.replica.ClientMetadata import org.apache.kafka.common.replica.ClientMetadata.DefaultClientMetadata import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.common.MetadataVersion.IBP_2_6_IV0 import org.apache.kafka.server.metrics.KafkaYammerMetrics @@ -2190,7 +2191,7 @@ class PartitionTest extends AbstractPartitionTest { val partition = new Partition( topicPartition, replicaLagTimeMaxMs = Defaults.ReplicaLagTimeMaxMs, - interBrokerProtocolVersion = MetadataVersion.IBP_3_7_IV1, + interBrokerProtocolVersion = MetadataVersion.IBP_3_7_IV2, localBrokerId = brokerId, () => defaultBrokerEpoch(brokerId), time, diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 9b8c02249aa27..f4ef0d4a9fd22 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -28,7 +28,7 @@ import kafka.utils._ import kafka.zk.KafkaZkClient import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{MemoryRecords, RecordBatch, RecordConversionStats} +import org.apache.kafka.common.record.{MemoryRecords, RecordBatch, RecordValidationStats} import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.server.util.timer.MockTimer import org.apache.kafka.server.util.{MockScheduler, MockTime} @@ -177,7 +177,7 @@ object AbstractCoordinatorConcurrencyTest { entriesPerPartition: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock] = None, - processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), + processingStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, transactionalId: String = null, actionQueue: ActionQueue = null): Unit = { diff --git a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorLoaderImplTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorLoaderImplTest.scala index ef19d732c3436..5bb4eb5b756c9 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorLoaderImplTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorLoaderImplTest.scala @@ -21,7 +21,7 @@ import kafka.server.ReplicaManager import kafka.utils.TestUtils import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.NotLeaderOrFollowerException -import org.apache.kafka.common.record.{CompressionType, FileRecords, MemoryRecords, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, FileRecords, MemoryRecords, RecordBatch, SimpleRecord} import org.apache.kafka.common.utils.{MockTime, Time} import org.apache.kafka.coordinator.group.runtime.CoordinatorLoader.UnknownRecordTypeException import org.apache.kafka.coordinator.group.runtime.{CoordinatorLoader, CoordinatorPlayback} @@ -104,7 +104,7 @@ class CoordinatorLoaderImplTest { )) { loader => when(replicaManager.getLog(tp)).thenReturn(Some(log)) when(log.logStartOffset).thenReturn(0L) - when(replicaManager.getLogEndOffset(tp)).thenReturn(Some(5L)) + when(replicaManager.getLogEndOffset(tp)).thenReturn(Some(7L)) val readResult1 = logReadResult(startOffset = 0, records = Seq( new SimpleRecord("k1".getBytes, "v1".getBytes), @@ -131,13 +131,27 @@ class CoordinatorLoaderImplTest { minOneMessage = true )).thenReturn(readResult2) + val readResult3 = logReadResult(startOffset = 5, producerId = 100L, producerEpoch = 5, records = Seq( + new SimpleRecord("k6".getBytes, "v6".getBytes), + new SimpleRecord("k7".getBytes, "v7".getBytes) + )) + + when(log.read( + startOffset = 5L, + maxLength = 1000, + isolation = FetchIsolation.LOG_END, + minOneMessage = true + )).thenReturn(readResult3) + assertNotNull(loader.load(tp, coordinator).get(10, TimeUnit.SECONDS)) - verify(coordinator).replay(("k1", "v1")) - verify(coordinator).replay(("k2", "v2")) - verify(coordinator).replay(("k3", "v3")) - verify(coordinator).replay(("k4", "v4")) - verify(coordinator).replay(("k5", "v5")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k1", "v1")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k2", "v2")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k3", "v3")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k4", "v4")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k5", "v5")) + verify(coordinator).replay(100L, 5.toShort, ("k6", "v6")) + verify(coordinator).replay(100L, 5.toShort, ("k7", "v7")) } } @@ -220,7 +234,7 @@ class CoordinatorLoaderImplTest { loader.load(tp, coordinator).get(10, TimeUnit.SECONDS) - verify(coordinator).replay(("k2", "v2")) + verify(coordinator).replay(RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, ("k2", "v2")) } } @@ -354,14 +368,28 @@ class CoordinatorLoaderImplTest { private def logReadResult( startOffset: Long, + producerId: Long = RecordBatch.NO_PRODUCER_ID, + producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH, records: Seq[SimpleRecord] ): FetchDataInfo = { val fileRecords = mock(classOf[FileRecords]) - val memoryRecords = MemoryRecords.withRecords( - startOffset, - CompressionType.NONE, - records: _* - ) + val memoryRecords = if (producerId == RecordBatch.NO_PRODUCER_ID) { + MemoryRecords.withRecords( + startOffset, + CompressionType.NONE, + records: _* + ) + } else { + MemoryRecords.withTransactionalRecords( + startOffset, + CompressionType.NONE, + producerId, + producerEpoch, + 0, + RecordBatch.NO_PARTITION_LEADER_EPOCH, + records: _* + ) + } when(fileRecords.sizeInBytes).thenReturn(memoryRecords.sizeInBytes) diff --git a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala index badcb6f8cba58..121a1f119a1ce 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala @@ -27,7 +27,7 @@ import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.utils.{MockTime, Time} import org.apache.kafka.coordinator.group.runtime.PartitionWriter import org.apache.kafka.storage.internals.log.{AppendOrigin, LogConfig} -import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows} +import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue} import org.junit.jupiter.api.Test import org.mockito.{ArgumentCaptor, ArgumentMatchers} import org.mockito.Mockito.{mock, verify, when} @@ -133,7 +133,12 @@ class CoordinatorPartitionWriterTest { ("k2", "v2"), ) - assertEquals(11, partitionRecordWriter.append(tp, records.asJava)) + assertEquals(11, partitionRecordWriter.append( + tp, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + records.asJava + )) val batch = recordsCapture.getValue.getOrElse(tp, throw new AssertionError(s"No records for $tp")) @@ -149,6 +154,86 @@ class CoordinatorPartitionWriterTest { assertEquals(records, receivedRecords) } + @Test + def testTransactionalWriteRecords(): Unit = { + val tp = new TopicPartition("foo", 0) + val replicaManager = mock(classOf[ReplicaManager]) + val time = new MockTime() + val partitionRecordWriter = new CoordinatorPartitionWriter( + replicaManager, + new StringKeyValueSerializer(), + CompressionType.NONE, + time + ) + + when(replicaManager.getLogConfig(tp)).thenReturn(Some(LogConfig.fromProps( + Collections.emptyMap(), + new Properties() + ))) + + val recordsCapture: ArgumentCaptor[Map[TopicPartition, MemoryRecords]] = + ArgumentCaptor.forClass(classOf[Map[TopicPartition, MemoryRecords]]) + val callbackCapture: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = + ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + + when(replicaManager.appendRecords( + ArgumentMatchers.eq(0L), + ArgumentMatchers.eq(1.toShort), + ArgumentMatchers.eq(true), + ArgumentMatchers.eq(AppendOrigin.COORDINATOR), + recordsCapture.capture(), + callbackCapture.capture(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any() + )).thenAnswer(_ => { + callbackCapture.getValue.apply(Map( + tp -> new PartitionResponse( + Errors.NONE, + 5, + 10, + RecordBatch.NO_TIMESTAMP, + -1, + Collections.emptyList(), + "" + ) + )) + }) + + val records = List( + ("k0", "v0"), + ("k1", "v1"), + ("k2", "v2"), + ) + + assertEquals(11, partitionRecordWriter.append( + tp, + 100L, + 50.toShort, + records.asJava + )) + + val batch = recordsCapture.getValue.getOrElse(tp, + throw new AssertionError(s"No records for $tp")) + assertEquals(1, batch.batches().asScala.toList.size) + + val firstBatch = batch.batches.asScala.head + assertEquals(100L, firstBatch.producerId) + assertEquals(50.toShort, firstBatch.producerEpoch) + assertTrue(firstBatch.isTransactional) + + val receivedRecords = batch.records.asScala.map { record => + ( + Charset.defaultCharset().decode(record.key).toString, + Charset.defaultCharset().decode(record.value).toString, + ) + }.toList + + assertEquals(records, receivedRecords) + } + @Test def testWriteRecordsWithFailure(): Unit = { val tp = new TopicPartition("foo", 0) @@ -195,8 +280,12 @@ class CoordinatorPartitionWriterTest { ("k2", "v2"), ) - assertThrows(classOf[NotLeaderOrFollowerException], - () => partitionRecordWriter.append(tp, records.asJava)) + assertThrows(classOf[NotLeaderOrFollowerException], () => partitionRecordWriter.append( + tp, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + records.asJava) + ) } @Test @@ -224,8 +313,12 @@ class CoordinatorPartitionWriterTest { ("k1", new String(randomBytes)), ) - assertThrows(classOf[RecordTooLargeException], - () => partitionRecordWriter.append(tp, records.asJava)) + assertThrows(classOf[RecordTooLargeException], () => partitionRecordWriter.append( + tp, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + records.asJava) + ) } @Test @@ -244,8 +337,12 @@ class CoordinatorPartitionWriterTest { new Properties() ))) - assertThrows(classOf[IllegalStateException], - () => partitionRecordWriter.append(tp, List.empty.asJava)) + assertThrows(classOf[IllegalStateException], () => partitionRecordWriter.append( + tp, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + List.empty.asJava) + ) } @Test @@ -267,7 +364,11 @@ class CoordinatorPartitionWriterTest { ("k2", "v2"), ) - assertThrows(classOf[NotLeaderOrFollowerException], - () => partitionRecordWriter.append(tp, records.asJava)) + assertThrows(classOf[NotLeaderOrFollowerException], () => partitionRecordWriter.append( + tp, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + records.asJava) + ) } } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorAdapterTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorAdapterTest.scala index 75ecb1e101b58..022a8716d8373 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorAdapterTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorAdapterTest.scala @@ -881,4 +881,17 @@ class GroupCoordinatorAdapterTest { assertTrue(future.isCompletedExceptionally) assertFutureThrows(future, classOf[InvalidGroupIdException]) } + + @Test + def testConsumerGroupDescribe(): Unit = { + val groupCoordinator = mock(classOf[GroupCoordinator]) + val adapter = new GroupCoordinatorAdapter(groupCoordinator, Time.SYSTEM) + val context = makeContext(ApiKeys.CONSUMER_GROUP_DESCRIBE, ApiKeys.CONSUMER_GROUP_DESCRIBE.latestVersion) + val groupIds = List("group-id-1", "group-id-2").asJava + + val future = adapter.consumerGroupDescribe(context, groupIds) + assertTrue(future.isDone) + assertTrue(future.isCompletedExceptionally) + assertFutureThrows(future, classOf[UnsupportedVersionException]) + } } diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala index 5dd665deae990..8b5d44a2a364f 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/ProducerIdManagerTest.scala @@ -17,7 +17,6 @@ package kafka.coordinator.transaction import kafka.coordinator.transaction.ProducerIdManager.RetryBackoffMs -import kafka.server.NodeToControllerChannelManager import kafka.utils.TestUtils import kafka.zk.{KafkaZkClient, ProducerIdBlockZNode} import org.apache.kafka.common.KafkaException @@ -26,6 +25,7 @@ import org.apache.kafka.common.message.AllocateProducerIdsResponseData import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.AllocateProducerIdsResponse import org.apache.kafka.common.utils.{MockTime, Time} +import org.apache.kafka.server.NodeToControllerChannelManager import org.apache.kafka.server.common.ProducerIdsBlock import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test diff --git a/core/src/test/scala/unit/kafka/log/LocalLogTest.scala b/core/src/test/scala/unit/kafka/log/LocalLogTest.scala index 924b8920bee78..29b5fd34f9091 100644 --- a/core/src/test/scala/unit/kafka/log/LocalLogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LocalLogTest.scala @@ -330,74 +330,6 @@ class LocalLogTest { testDeleteSegmentFiles(asyncDelete = true) } - @Test - def testDeletableSegmentsFilter(): Unit = { - for (offset <- 0 to 8) { - val record = new SimpleRecord(mockTime.milliseconds, "a".getBytes) - appendRecords(List(record), initialOffset = offset) - log.roll() - } - - assertEquals(10, log.segments.numberOfSegments) - - { - val deletable = log.deletableSegments( - (segment: LogSegment, _: Option[LogSegment]) => segment.baseOffset <= 5) - val expected = log.segments.nonActiveLogSegmentsFrom(0L).asScala.filter(segment => segment.baseOffset <= 5).toList - assertEquals(6, expected.length) - assertEquals(expected, deletable.toList) - } - - { - val deletable = log.deletableSegments((_: LogSegment, _: Option[LogSegment]) => true) - val expected = log.segments.nonActiveLogSegmentsFrom(0L).asScala.toList - assertEquals(9, expected.length) - assertEquals(expected, deletable.toList) - } - - { - val record = new SimpleRecord(mockTime.milliseconds, "a".getBytes) - appendRecords(List(record), initialOffset = 9L) - val deletable = log.deletableSegments((_: LogSegment, _: Option[LogSegment]) => true) - val expected = log.segments.values.asScala.toList - assertEquals(10, expected.length) - assertEquals(expected, deletable.toList) - } - } - - @Test - def testDeletableSegmentsIteration(): Unit = { - for (offset <- 0 to 8) { - val record = new SimpleRecord(mockTime.milliseconds, "a".getBytes) - appendRecords(List(record), initialOffset = offset) - log.roll() - } - - assertEquals(10L, log.segments.numberOfSegments) - - var offset = 0 - val deletableSegments = log.deletableSegments( - (segment: LogSegment, nextSegmentOpt: Option[LogSegment]) => { - assertEquals(offset, segment.baseOffset) - val floorSegmentOpt = log.segments.floorSegment(offset) - assertTrue(floorSegmentOpt.isPresent) - assertEquals(floorSegmentOpt.get, segment) - if (offset == log.logEndOffset) { - assertFalse(nextSegmentOpt.isDefined) - } else { - assertTrue(nextSegmentOpt.isDefined) - val higherSegmentOpt = log.segments.higherSegment(segment.baseOffset) - assertTrue(higherSegmentOpt.isPresent) - assertEquals(segment.baseOffset + 1, higherSegmentOpt.get.baseOffset) - assertEquals(higherSegmentOpt.get, nextSegmentOpt.get) - } - offset += 1 - true - }) - assertEquals(10L, log.segments.numberOfSegments) - assertEquals(log.segments.nonActiveLogSegmentsFrom(0L).asScala.toSeq, deletableSegments.toSeq) - } - @Test def testCreateAndDeleteSegment(): Unit = { val record = new SimpleRecord(mockTime.milliseconds, "a".getBytes) diff --git a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala index aacab5b624ef5..11fff517b430d 100644 --- a/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogSegmentTest.scala @@ -404,7 +404,7 @@ class LogSegmentTest { val checkpoint: LeaderEpochCheckpoint = new LeaderEpochCheckpoint { private var epochs = Seq.empty[EpochEntry] - override def write(epochs: util.Collection[EpochEntry]): Unit = { + override def write(epochs: util.Collection[EpochEntry], ignored: Boolean): Unit = { this.epochs = epochs.asScala.toSeq } diff --git a/core/src/test/scala/unit/kafka/log/LogTestUtils.scala b/core/src/test/scala/unit/kafka/log/LogTestUtils.scala index ad47da05a005f..c3f630e7646c5 100644 --- a/core/src/test/scala/unit/kafka/log/LogTestUtils.scala +++ b/core/src/test/scala/unit/kafka/log/LogTestUtils.scala @@ -56,7 +56,9 @@ object LogTestUtils { def createLogConfig(segmentMs: Long = LogConfig.DEFAULT_SEGMENT_MS, segmentBytes: Int = LogConfig.DEFAULT_SEGMENT_BYTES, retentionMs: Long = LogConfig.DEFAULT_RETENTION_MS, + localRetentionMs: Long = LogConfig.DEFAULT_LOCAL_RETENTION_MS, retentionBytes: Long = LogConfig.DEFAULT_RETENTION_BYTES, + localRetentionBytes: Long = LogConfig.DEFAULT_LOCAL_RETENTION_BYTES, segmentJitterMs: Long = LogConfig.DEFAULT_SEGMENT_JITTER_MS, cleanupPolicy: String = LogConfig.DEFAULT_CLEANUP_POLICY, maxMessageBytes: Int = LogConfig.DEFAULT_MAX_MESSAGE_BYTES, @@ -68,7 +70,9 @@ object LogTestUtils { logProps.put(TopicConfig.SEGMENT_MS_CONFIG, segmentMs: java.lang.Long) logProps.put(TopicConfig.SEGMENT_BYTES_CONFIG, segmentBytes: Integer) logProps.put(TopicConfig.RETENTION_MS_CONFIG, retentionMs: java.lang.Long) + logProps.put(TopicConfig.LOCAL_LOG_RETENTION_MS_CONFIG, localRetentionMs: java.lang.Long) logProps.put(TopicConfig.RETENTION_BYTES_CONFIG, retentionBytes: java.lang.Long) + logProps.put(TopicConfig.LOCAL_LOG_RETENTION_BYTES_CONFIG, localRetentionBytes: java.lang.Long) logProps.put(TopicConfig.SEGMENT_JITTER_MS_CONFIG, segmentJitterMs: java.lang.Long) logProps.put(TopicConfig.CLEANUP_POLICY_CONFIG, cleanupPolicy) logProps.put(TopicConfig.MAX_MESSAGE_BYTES_CONFIG, maxMessageBytes: Integer) diff --git a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala index 6b781f6fa696a..5d900223a711d 100644 --- a/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogValidatorTest.scala @@ -177,7 +177,7 @@ class LogValidatorTest { val expectedMaxTimestampOffset = if (magic >= RecordBatch.MAGIC_VALUE_V2) 2 else 0 assertEquals(expectedMaxTimestampOffset, validatedResults.shallowOffsetOfMaxTimestampMs, s"The offset of max timestamp should be $expectedMaxTimestampOffset") - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 0, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 0, records, compressed = false) } @@ -224,8 +224,8 @@ class LogValidatorTest { assertTrue(validatedResults.messageSizeMaybeChanged, "Message size may have been changed") - val stats = validatedResults.recordConversionStats - verifyRecordConversionStats(stats, numConvertedRecords = 3, records, compressed = true) + val stats = validatedResults.recordValidationStats + verifyRecordValidationStats(stats, numConvertedRecords = 3, records, compressed = true) } @Test @@ -276,7 +276,7 @@ class LogValidatorTest { assertFalse(validatedResults.messageSizeMaybeChanged, "Message size should not have been changed") - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 0, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 0, records, compressed = true) } @@ -350,11 +350,14 @@ class LogValidatorTest { (RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, RecordBatch.NO_PARTITION_LEADER_EPOCH) - val records = MemoryRecords.withRecords(magic, 0L, CompressionType.GZIP, TimestampType.CREATE_TIME, producerId, - producerEpoch, baseSequence, partitionLeaderEpoch, isTransactional, + val recordList = List( new SimpleRecord(timestampSeq(0), "hello".getBytes), new SimpleRecord(timestampSeq(1), "there".getBytes), - new SimpleRecord(timestampSeq(2), "beautiful".getBytes)) + new SimpleRecord(timestampSeq(2), "beautiful".getBytes) + ) + + val records = MemoryRecords.withRecords(magic, 0L, CompressionType.GZIP, TimestampType.CREATE_TIME, producerId, + producerEpoch, baseSequence, partitionLeaderEpoch, isTransactional, recordList: _*) val offsetCounter = PrimitiveRef.ofLong(0); val validatingResults = new LogValidator(records, @@ -399,12 +402,20 @@ class LogValidatorTest { assertEquals(i, offsetCounter.value); assertEquals(now + 1, validatingResults.maxTimestampMs, s"Max timestamp should be ${now + 1}") - assertEquals(1, validatingResults.shallowOffsetOfMaxTimestampMs, + + val expectedShallowOffsetOfMaxTimestamp = if (magic >= RecordVersion.V2.value) { + // v2 records are always batched, even when not compressed. + // the shallow offset of max timestamp is the last offset of the batch + recordList.size - 1 + } else { + 1 + } + assertEquals(expectedShallowOffsetOfMaxTimestamp, validatingResults.shallowOffsetOfMaxTimestampMs, s"Offset of max timestamp should be 1") + assertFalse(validatingResults.messageSizeMaybeChanged, "Message size should not have been changed") - - verifyRecordConversionStats(validatingResults.recordConversionStats, numConvertedRecords = 0, records, + verifyRecordValidationStats(validatingResults.recordValidationStats, numConvertedRecords = 0, records, compressed = false) } @@ -478,7 +489,7 @@ class LogValidatorTest { assertTrue(validatingResults.messageSizeMaybeChanged, "Message size should have been changed") - verifyRecordConversionStats(validatingResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatingResults.recordValidationStats, numConvertedRecords = 3, records, compressed = true) } @@ -529,7 +540,7 @@ class LogValidatorTest { s"Offset of max timestamp should be ${validatedRecords.records.asScala.size - 1}") assertTrue(validatedResults.messageSizeMaybeChanged, "Message size should have been changed") - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = true) } @@ -576,7 +587,7 @@ class LogValidatorTest { s"Offset of max timestamp should be ${validatedRecords.records.asScala.size - 1}") assertTrue(validatedResults.messageSizeMaybeChanged, "Message size should have been changed") - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = true) } @@ -596,11 +607,14 @@ class LogValidatorTest { (RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, RecordBatch.NO_PARTITION_LEADER_EPOCH) - val records = MemoryRecords.withRecords(magic, 0L, CompressionType.GZIP, TimestampType.CREATE_TIME, producerId, - producerEpoch, baseSequence, partitionLeaderEpoch, isTransactional, + val recordList = List( new SimpleRecord(timestampSeq(0), "hello".getBytes), new SimpleRecord(timestampSeq(1), "there".getBytes), - new SimpleRecord(timestampSeq(2), "beautiful".getBytes)) + new SimpleRecord(timestampSeq(2), "beautiful".getBytes) + ) + + val records = MemoryRecords.withRecords(magic, 0L, CompressionType.GZIP, TimestampType.CREATE_TIME, producerId, + producerEpoch, baseSequence, partitionLeaderEpoch, isTransactional, recordList: _*) val validatedResults = new LogValidator(records, topicPartition, @@ -639,11 +653,15 @@ class LogValidatorTest { } } assertEquals(now + 1, validatedResults.maxTimestampMs, s"Max timestamp should be ${now + 1}") - assertEquals(validatedRecords.records.asScala.size - 1, validatedResults.shallowOffsetOfMaxTimestampMs, + + // All versions have an outer batch when compressed, so the shallow offset + // of max timestamp is always the offset of the last record in the batch. + val expectedShallowOffsetOfMaxTimestamp = recordList.size - 1 + assertEquals(expectedShallowOffsetOfMaxTimestamp, validatedResults.shallowOffsetOfMaxTimestampMs, s"Offset of max timestamp should be ${validatedRecords.records.asScala.size - 1}") assertFalse(validatedResults.messageSizeMaybeChanged, "Message size should not have been changed") - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 0, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 0, records, compressed = true) } @@ -926,7 +944,7 @@ class LogValidatorTest { PrimitiveRef.ofLong(offset), metricsRecorder, RequestLocal.withThreadConfinedCaching.bufferSupplier ) checkOffsets(validatedResults.validatedRecords, offset) - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = false) } @@ -952,7 +970,7 @@ class LogValidatorTest { PrimitiveRef.ofLong(offset), metricsRecorder, RequestLocal.withThreadConfinedCaching.bufferSupplier ) checkOffsets(validatedResults.validatedRecords, offset) - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = false) } @@ -978,7 +996,7 @@ class LogValidatorTest { PrimitiveRef.ofLong(offset), metricsRecorder, RequestLocal.withThreadConfinedCaching.bufferSupplier ) checkOffsets(validatedResults.validatedRecords, offset) - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = true) } @@ -1004,7 +1022,7 @@ class LogValidatorTest { PrimitiveRef.ofLong(offset), metricsRecorder, RequestLocal.withThreadConfinedCaching.bufferSupplier ) checkOffsets(validatedResults.validatedRecords, offset) - verifyRecordConversionStats(validatedResults.recordConversionStats, numConvertedRecords = 3, records, + verifyRecordValidationStats(validatedResults.recordValidationStats, numConvertedRecords = 3, records, compressed = true) } @@ -1641,7 +1659,7 @@ class LogValidatorTest { } } - def verifyRecordConversionStats(stats: RecordConversionStats, numConvertedRecords: Int, records: MemoryRecords, + def verifyRecordValidationStats(stats: RecordValidationStats, numConvertedRecords: Int, records: MemoryRecords, compressed: Boolean): Unit = { assertNotNull(stats, "Records processing info is null") assertEquals(numConvertedRecords, stats.numRecordsConverted) diff --git a/core/src/test/scala/unit/kafka/log/UnifiedLogTest.scala b/core/src/test/scala/unit/kafka/log/UnifiedLogTest.scala index 86bcf6c878b16..1a4ffaa57d05e 100755 --- a/core/src/test/scala/unit/kafka/log/UnifiedLogTest.scala +++ b/core/src/test/scala/unit/kafka/log/UnifiedLogTest.scala @@ -36,14 +36,14 @@ import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.util.{KafkaScheduler, MockTime, Scheduler} import org.apache.kafka.storage.internals.checkpoint.{LeaderEpochCheckpointFile, PartitionMetadataFile} import org.apache.kafka.storage.internals.epoch.LeaderEpochFileCache -import org.apache.kafka.storage.internals.log.{AbortedTxn, AppendOrigin, EpochEntry, FetchIsolation, LogConfig, LogFileUtils, LogOffsetMetadata, LogOffsetSnapshot, LogOffsetsListener, LogSegment, LogStartOffsetIncrementReason, ProducerStateManager, ProducerStateManagerConfig, RecordValidationException, VerificationGuard} +import org.apache.kafka.storage.internals.log.{AbortedTxn, AppendOrigin, EpochEntry, FetchIsolation, LogConfig, LogFileUtils, LogOffsetMetadata, LogOffsetSnapshot, LogOffsetsListener, LogSegment, LogSegments, LogStartOffsetIncrementReason, ProducerStateManager, ProducerStateManagerConfig, RecordValidationException, VerificationGuard} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource import org.mockito.ArgumentMatchers -import org.mockito.ArgumentMatchers.anyLong -import org.mockito.Mockito.{mock, when} +import org.mockito.ArgumentMatchers.{any, anyLong} +import org.mockito.Mockito.{doThrow, mock, spy, when} import java.io._ import java.nio.ByteBuffer @@ -3625,7 +3625,7 @@ class UnifiedLogTest { val records = TestUtils.singletonRecords(value = s"test$i".getBytes) log.appendAsLeader(records, leaderEpoch = 0) } - + log.updateHighWatermark(90L) log.maybeIncrementLogStartOffset(20L, LogStartOffsetIncrementReason.SegmentDeletion) assertEquals(20, log.logStartOffset) @@ -3911,6 +3911,139 @@ class UnifiedLogTest { log.appendAsLeader(transactionalRecords, leaderEpoch = 0, verificationGuard = verificationGuard) } + @Test + def testRecoveryPointNotIncrementedOnProducerStateSnapshotFlushFailure(): Unit = { + val logConfig = LogTestUtils.createLogConfig() + val log = spy(createLog(logDir, logConfig)) + + doThrow(new KafkaStorageException("Injected exception")).when(log).flushProducerStateSnapshot(any()) + + log.appendAsLeader(TestUtils.singletonRecords("a".getBytes), leaderEpoch = 0) + try { + log.roll(Some(1L)) + } catch { + case _: KafkaStorageException => // ignore + } + + // check that the recovery point isn't incremented + assertEquals(0L, log.recoveryPoint) + } + + @Test + def testDeletableSegmentsFilter(): Unit = { + val logConfig = LogTestUtils.createLogConfig(segmentBytes = 1024 * 1024) + val log = createLog(logDir, logConfig) + for (_ <- 0 to 8) { + val records = TestUtils.records(List( + new SimpleRecord(mockTime.milliseconds, "a".getBytes), + )) + log.appendAsLeader(records, leaderEpoch = 0) + log.roll() + } + log.maybeIncrementHighWatermark(log.logEndOffsetMetadata) + + assertEquals(10, log.logSegments.size()) + + { + val deletable = log.deletableSegments( + (segment: LogSegment, _: Option[LogSegment]) => segment.baseOffset <= 5) + val expected = log.nonActiveLogSegmentsFrom(0L).asScala.filter(segment => segment.baseOffset <= 5).toList + assertEquals(6, expected.length) + assertEquals(expected, deletable.toList) + } + + { + val deletable = log.deletableSegments((_: LogSegment, _: Option[LogSegment]) => true) + val expected = log.nonActiveLogSegmentsFrom(0L).asScala.toList + assertEquals(9, expected.length) + assertEquals(expected, deletable.toList) + } + + { + val records = TestUtils.records(List( + new SimpleRecord(mockTime.milliseconds, "a".getBytes), + )) + log.appendAsLeader(records, leaderEpoch = 0) + log.maybeIncrementHighWatermark(log.logEndOffsetMetadata) + val deletable = log.deletableSegments((_: LogSegment, _: Option[LogSegment]) => true) + val expected = log.logSegments.asScala.toList + assertEquals(10, expected.length) + assertEquals(expected, deletable.toList) + } + } + + @Test + def testDeletableSegmentsIteration(): Unit = { + val logConfig = LogTestUtils.createLogConfig(segmentBytes = 1024 * 1024) + val log = createLog(logDir, logConfig) + for (_ <- 0 to 8) { + val records = TestUtils.records(List( + new SimpleRecord(mockTime.milliseconds, "a".getBytes), + )) + log.appendAsLeader(records, leaderEpoch = 0) + log.roll() + } + log.maybeIncrementHighWatermark(log.logEndOffsetMetadata) + + assertEquals(10, log.logSegments.size()) + + var offset = 0 + val deletableSegments = log.deletableSegments( + (segment: LogSegment, nextSegmentOpt: Option[LogSegment]) => { + assertEquals(offset, segment.baseOffset) + val logSegments = new LogSegments(log.topicPartition) + log.logSegments.forEach(segment => logSegments.add(segment)) + val floorSegmentOpt = logSegments.floorSegment(offset) + assertTrue(floorSegmentOpt.isPresent) + assertEquals(floorSegmentOpt.get, segment) + if (offset == log.logEndOffset) { + assertFalse(nextSegmentOpt.isDefined) + } else { + assertTrue(nextSegmentOpt.isDefined) + val higherSegmentOpt = logSegments.higherSegment(segment.baseOffset) + assertTrue(higherSegmentOpt.isPresent) + assertEquals(segment.baseOffset + 1, higherSegmentOpt.get.baseOffset) + assertEquals(higherSegmentOpt.get, nextSegmentOpt.get) + } + offset += 1 + true + }) + assertEquals(10L, log.logSegments.size()) + assertEquals(log.nonActiveLogSegmentsFrom(0L).asScala.toSeq, deletableSegments.toSeq) + } + + @Test + def testActiveSegmentDeletionDueToRetentionTimeBreachWithRemoteStorage(): Unit = { + val logConfig = LogTestUtils.createLogConfig(indexIntervalBytes = 1, segmentIndexBytes = 12, + retentionMs = 3, localRetentionMs = 1, fileDeleteDelayMs = 0, remoteLogStorageEnable = true) + val log = createLog(logDir, logConfig, remoteStorageSystemEnable = true) + + // Append 1 message to the active segment + log.appendAsLeader(TestUtils.records(List(new SimpleRecord(mockTime.milliseconds(), "a".getBytes))), + leaderEpoch = 0) + // Update the highWatermark so that these segments will be eligible for deletion. + log.updateHighWatermark(log.logEndOffset) + assertEquals(1, log.logSegments.size) + assertEquals(0, log.activeSegment.baseOffset()) + + mockTime.sleep(2) + // It should have rolled the active segment as they are eligible for deletion + log.deleteOldSegments() + assertEquals(2, log.logSegments.size) + log.logSegments.asScala.zipWithIndex.foreach { + case (segment, idx) => assertEquals(idx, segment.baseOffset) + } + + // Once rolled, the segment should be uploaded to remote storage and eligible for deletion + log.updateHighestOffsetInRemoteStorage(1) + log.deleteOldSegments() + assertEquals(1, log.logSegments.size) + assertEquals(1, log.logSegments.asScala.head.baseOffset()) + assertEquals(1, log.localLogStartOffset()) + assertEquals(1, log.logEndOffset) + assertEquals(0, log.logStartOffset) + } + private def appendTransactionalToBuffer(buffer: ByteBuffer, producerId: Long, producerEpoch: Short, diff --git a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala index 5276d030cb8ac..5b9e0aafee2f4 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala @@ -74,6 +74,7 @@ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { apiVersionsResponse: ApiVersionsResponse, listenerName: ListenerName = cluster.clientListener(), enableUnstableLastVersion: Boolean = false, + clientTelemetryEnabled: Boolean = false, apiVersion: Short = ApiKeys.API_VERSIONS.latestVersion ): Unit = { if (cluster.isKRaftTest && apiVersion >= 3) { @@ -100,7 +101,8 @@ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { ApiMessageType.ListenerType.BROKER, RecordVersion.current, NodeApiVersions.create(ApiKeys.controllerApis().asScala.map(ApiVersionsResponse.toApiVersion).asJava).allSupportedApiVersions(), - enableUnstableLastVersion + enableUnstableLastVersion, + clientTelemetryEnabled ) } diff --git a/core/src/test/scala/unit/kafka/server/AlterPartitionManagerTest.scala b/core/src/test/scala/unit/kafka/server/AlterPartitionManagerTest.scala index ccf3909adcc06..64b920cbf3889 100644 --- a/core/src/test/scala/unit/kafka/server/AlterPartitionManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/AlterPartitionManagerTest.scala @@ -33,6 +33,7 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.RequestHeader import org.apache.kafka.common.requests.{AbstractRequest, AlterPartitionRequest, AlterPartitionResponse} import org.apache.kafka.metadata.LeaderRecoveryState +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.common.MetadataVersion.{IBP_2_7_IV2, IBP_3_2_IV0, IBP_3_5_IV1} import org.apache.kafka.server.util.{MockScheduler, MockTime} diff --git a/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala b/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala index 89b81035dc37a..a3c6deb488df4 100644 --- a/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/AutoTopicCreationManagerTest.scala @@ -21,7 +21,6 @@ import java.net.InetAddress import java.nio.ByteBuffer import java.util.concurrent.atomic.AtomicBoolean import java.util.{Collections, Optional, Properties} - import kafka.controller.KafkaController import kafka.coordinator.transaction.TransactionCoordinator import kafka.utils.TestUtils @@ -38,6 +37,7 @@ import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.{KafkaPrincipal, KafkaPrincipalSerde, SecurityProtocol} import org.apache.kafka.common.utils.{SecurityUtils, Utils} import org.apache.kafka.coordinator.group.GroupCoordinator +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue} import org.junit.jupiter.api.{BeforeEach, Test} import org.mockito.ArgumentMatchers.any @@ -327,7 +327,7 @@ class AutoTopicCreationManagerTest { .setMinVersion(0) .setMaxVersion(0) Mockito.when(brokerToController.controllerApiVersions()) - .thenReturn(Some(NodeApiVersions.create(Collections.singleton(createTopicApiVersion)))) + .thenReturn(Optional.of(NodeApiVersions.create(Collections.singleton(createTopicApiVersion)))) Mockito.when(controller.isActive).thenReturn(false) diff --git a/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala b/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala index 0bc993d55dfa7..113088af4d512 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerLifecycleManagerTest.scala @@ -201,40 +201,33 @@ class BrokerLifecycleManagerTest { while (!future.isDone || context.mockClient.hasInFlightRequests) { context.poll() manager.eventQueue.wakeup() - context.time.sleep(100) + context.time.sleep(5) } future.get } @Test - def testOfflineDirsSentUntilHeartbeatSuccess(): Unit = { + def testAlwaysSendsAccumulatedOfflineDirs(): Unit = { val ctx = new RegistrationTestContext(configProperties) val manager = new BrokerLifecycleManager(ctx.config, ctx.time, "offline-dirs-sent-in-heartbeat-", isZkBroker = false) val controllerNode = new Node(3000, "localhost", 8021) ctx.controllerNodeProvider.node.set(controllerNode) val registration = prepareResponse(ctx, new BrokerRegistrationResponse(new BrokerRegistrationResponseData().setBrokerEpoch(1000))) - val hb1 = prepareResponse[BrokerHeartbeatRequest](ctx, new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData() - .setErrorCode(Errors.NOT_CONTROLLER.code()))) - val hb2 = prepareResponse[BrokerHeartbeatRequest](ctx, new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData())) - val hb3 = prepareResponse[BrokerHeartbeatRequest](ctx, new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData())) - - val offlineDirs = Set(Uuid.fromString("h3sC4Yk-Q9-fd0ntJTocCA"), Uuid.fromString("ej8Q9_d2Ri6FXNiTxKFiow")) - offlineDirs.foreach(manager.propagateDirectoryFailure) - - // start the manager late to prevent a race, and force expectations on the first heartbeat manager.start(() => ctx.highestMetadataOffset.get(), ctx.mockChannelManager, ctx.clusterId, ctx.advertisedListeners, Collections.emptyMap(), OptionalLong.empty()) - poll(ctx, manager, registration) - val dirs1 = poll(ctx, manager, hb1).data().offlineLogDirs() - val dirs2 = poll(ctx, manager, hb2).data().offlineLogDirs() - val dirs3 = poll(ctx, manager, hb3).data().offlineLogDirs() - assertEquals(offlineDirs, dirs1.asScala.toSet) - assertEquals(offlineDirs, dirs2.asScala.toSet) - assertEquals(Set.empty, dirs3.asScala.toSet) + manager.propagateDirectoryFailure(Uuid.fromString("h3sC4Yk-Q9-fd0ntJTocCA")) + manager.propagateDirectoryFailure(Uuid.fromString("ej8Q9_d2Ri6FXNiTxKFiow")) + manager.propagateDirectoryFailure(Uuid.fromString("1iF76HVNRPqC7Y4r6647eg")) + val latestHeartbeat = Seq.fill(10)( + prepareResponse[BrokerHeartbeatRequest](ctx, new BrokerHeartbeatResponse(new BrokerHeartbeatResponseData())) + ).map(poll(ctx, manager, _)).last + assertEquals( + Set("h3sC4Yk-Q9-fd0ntJTocCA", "ej8Q9_d2Ri6FXNiTxKFiow", "1iF76HVNRPqC7Y4r6647eg").map(Uuid.fromString), + latestHeartbeat.data().offlineLogDirs().asScala.toSet) manager.close() } diff --git a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala index ae14fa7171b99..6c6199f71da9a 100644 --- a/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BrokerRegistrationRequestTest.scala @@ -31,6 +31,7 @@ import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.Time import org.apache.kafka.common.{Node, Uuid} +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.common.MetadataVersion import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows} import org.junit.jupiter.api.extension.ExtendWith @@ -47,7 +48,7 @@ import java.util.concurrent.{CompletableFuture, TimeUnit, TimeoutException} class BrokerRegistrationRequestTest { def brokerToControllerChannelManager(clusterInstance: ClusterInstance): NodeToControllerChannelManager = { - NodeToControllerChannelManager( + new NodeToControllerChannelManagerImpl( new ControllerNodeProvider() { def node: Option[Node] = Some(new Node( clusterInstance.anyControllerSocketServer().config.nodeId, diff --git a/core/src/test/scala/unit/kafka/server/ConsumerGroupHeartbeatRequestTest.scala b/core/src/test/scala/unit/kafka/server/ConsumerGroupHeartbeatRequestTest.scala index 42bfb97bb25b2..d0bb95cafe81c 100644 --- a/core/src/test/scala/unit/kafka/server/ConsumerGroupHeartbeatRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ConsumerGroupHeartbeatRequestTest.scala @@ -24,12 +24,11 @@ import kafka.utils.TestUtils import org.apache.kafka.common.message.{ConsumerGroupHeartbeatRequestData, ConsumerGroupHeartbeatResponseData} import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{ConsumerGroupHeartbeatRequest, ConsumerGroupHeartbeatResponse} -import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull, assertThrows} +import org.junit.jupiter.api.Assertions.{assertEquals, assertNotEquals, assertNotNull} import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Timeout import org.junit.jupiter.api.extension.ExtendWith -import java.io.EOFException import java.util.stream.Collectors import scala.jdk.CollectionConverters._ @@ -39,20 +38,10 @@ import scala.jdk.CollectionConverters._ @Tag("integration") class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { - @ClusterTest - def testConsumerGroupHeartbeatIsDisabledByDefault(): Unit = { - val consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( - new ConsumerGroupHeartbeatRequestData(), - true - ).build() - assertThrows(classOf[EOFException], () => connectAndReceive(consumerGroupHeartbeatRequest)) - } - - @ClusterTest(serverProperties = Array(new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"))) + @ClusterTest() def testConsumerGroupHeartbeatIsAccessibleWhenEnabled(): Unit = { val consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( - new ConsumerGroupHeartbeatRequestData(), - true + new ConsumerGroupHeartbeatRequestData() ).build() val consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) @@ -61,7 +50,6 @@ class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { } @ClusterTest(clusterType = Type.KRAFT, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -86,8 +74,7 @@ class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { .setMemberEpoch(0) .setRebalanceTimeoutMs(5 * 60 * 1000) .setSubscribedTopicNames(List("foo").asJava) - .setTopicPartitions(List.empty.asJava), - true + .setTopicPartitions(List.empty.asJava) ).build() // Send the request until receiving a successful response. There is a delay @@ -115,8 +102,7 @@ class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { new ConsumerGroupHeartbeatRequestData() .setGroupId("grp") .setMemberId(consumerGroupHeartbeatResponse.data.memberId) - .setMemberEpoch(consumerGroupHeartbeatResponse.data.memberEpoch), - true + .setMemberEpoch(consumerGroupHeartbeatResponse.data.memberEpoch) ).build() // This is the expected assignment. @@ -142,8 +128,7 @@ class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { new ConsumerGroupHeartbeatRequestData() .setGroupId("grp") .setMemberId(consumerGroupHeartbeatResponse.data.memberId) - .setMemberEpoch(-1), - true + .setMemberEpoch(-1) ).build() consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) @@ -152,6 +137,120 @@ class ConsumerGroupHeartbeatRequestTest(cluster: ClusterInstance) { assertEquals(-1, consumerGroupHeartbeatResponse.data.memberEpoch) } + @ClusterTest(clusterType = Type.KRAFT, serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testRejoiningStaticMemberGetsAssignmentsBackWhenNewGroupCoordinatorIsEnabled(): Unit = { + val raftCluster = cluster.asInstanceOf[RaftClusterInstance] + val admin = cluster.createAdminClient() + val instanceId = "instanceId" + + // Creates the __consumer_offsets topics because it won't be created automatically + // in this test because it does not use FindCoordinator API. + TestUtils.createOffsetsTopicWithAdmin( + admin = admin, + brokers = raftCluster.brokers.collect(Collectors.toList[BrokerServer]).asScala, + controllers = raftCluster.controllerServers().asScala.toSeq + ) + + // Heartbeat request so that a static member joins the group + var consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( + new ConsumerGroupHeartbeatRequestData() + .setGroupId("grp") + .setInstanceId(instanceId) + .setMemberEpoch(0) + .setRebalanceTimeoutMs(5 * 60 * 1000) + .setSubscribedTopicNames(List("foo").asJava) + .setTopicPartitions(List.empty.asJava) + ).build() + + // Send the request until receiving a successful response. There is a delay + // here because the group coordinator is loaded in the background. + var consumerGroupHeartbeatResponse: ConsumerGroupHeartbeatResponse = null + TestUtils.waitUntilTrue(() => { + consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) + consumerGroupHeartbeatResponse.data.errorCode == Errors.NONE.code + }, msg = s"Static member could not join the group successfully. Last response $consumerGroupHeartbeatResponse.") + + // Verify the response. + assertNotNull(consumerGroupHeartbeatResponse.data.memberId) + assertEquals(1, consumerGroupHeartbeatResponse.data.memberEpoch) + assertEquals(new ConsumerGroupHeartbeatResponseData.Assignment(), consumerGroupHeartbeatResponse.data.assignment) + + // Create the topic. + val topicId = TestUtils.createTopicWithAdminRaw( + admin = admin, + topic = "foo", + numPartitions = 3 + ) + + // Prepare the next heartbeat. + consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( + new ConsumerGroupHeartbeatRequestData() + .setGroupId("grp") + .setInstanceId(instanceId) + .setMemberId(consumerGroupHeartbeatResponse.data.memberId) + .setMemberEpoch(consumerGroupHeartbeatResponse.data.memberEpoch) + ).build() + + // This is the expected assignment. + val expectedAssignment = new ConsumerGroupHeartbeatResponseData.Assignment() + .setTopicPartitions(List(new ConsumerGroupHeartbeatResponseData.TopicPartitions() + .setTopicId(topicId) + .setPartitions(List[Integer](0, 1, 2).asJava)).asJava) + + // Heartbeats until the partitions are assigned. + consumerGroupHeartbeatResponse = null + TestUtils.waitUntilTrue(() => { + consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) + consumerGroupHeartbeatResponse.data.errorCode == Errors.NONE.code && + consumerGroupHeartbeatResponse.data.assignment == expectedAssignment + }, msg = s"Static member could not get partitions assigned. Last response $consumerGroupHeartbeatResponse.") + + // Verify the response. + assertNotNull(consumerGroupHeartbeatResponse.data.memberId) + assertEquals(2, consumerGroupHeartbeatResponse.data.memberEpoch) + assertEquals(expectedAssignment, consumerGroupHeartbeatResponse.data.assignment) + + val oldMemberId = consumerGroupHeartbeatResponse.data.memberId + + // Leave the group temporarily + consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( + new ConsumerGroupHeartbeatRequestData() + .setGroupId("grp") + .setInstanceId(instanceId) + .setMemberId(consumerGroupHeartbeatResponse.data.memberId) + .setMemberEpoch(-2) + ).build() + + consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) + + // Verify the response. + assertEquals(-2, consumerGroupHeartbeatResponse.data.memberEpoch) + + // Another static member replaces the above member. It gets the same assignments back + consumerGroupHeartbeatRequest = new ConsumerGroupHeartbeatRequest.Builder( + new ConsumerGroupHeartbeatRequestData() + .setGroupId("grp") + .setInstanceId(instanceId) + .setMemberEpoch(0) + .setRebalanceTimeoutMs(5 * 60 * 1000) + .setSubscribedTopicNames(List("foo").asJava) + .setTopicPartitions(List.empty.asJava) + ).build() + + consumerGroupHeartbeatResponse = connectAndReceive(consumerGroupHeartbeatRequest) + + // Verify the response. + assertNotNull(consumerGroupHeartbeatResponse.data.memberId) + assertEquals(2, consumerGroupHeartbeatResponse.data.memberEpoch) + assertEquals(expectedAssignment, consumerGroupHeartbeatResponse.data.assignment) + // The 2 member IDs should be different + assertNotEquals(oldMemberId, consumerGroupHeartbeatResponse.data.memberId) + } + private def connectAndReceive(request: ConsumerGroupHeartbeatRequest): ConsumerGroupHeartbeatResponse = { IntegrationTestUtils.connectAndReceive[ConsumerGroupHeartbeatResponse]( request, diff --git a/core/src/test/scala/unit/kafka/server/ControllerConfigurationValidatorTest.scala b/core/src/test/scala/unit/kafka/server/ControllerConfigurationValidatorTest.scala index 4ce400e1ed58c..c242194ef7637 100644 --- a/core/src/test/scala/unit/kafka/server/ControllerConfigurationValidatorTest.scala +++ b/core/src/test/scala/unit/kafka/server/ControllerConfigurationValidatorTest.scala @@ -17,12 +17,12 @@ package kafka.server -import kafka.metrics.ClientMetricsConfigs import kafka.utils.TestUtils import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, BROKER_LOGGER, CLIENT_METRICS, TOPIC} import org.apache.kafka.common.config.TopicConfig.{SEGMENT_BYTES_CONFIG, SEGMENT_JITTER_MS_CONFIG, SEGMENT_MS_CONFIG} import org.apache.kafka.common.errors.{InvalidConfigurationException, InvalidRequestException, InvalidTopicException} +import org.apache.kafka.server.metrics.ClientMetricsConfigs import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows} import org.junit.jupiter.api.Test @@ -153,4 +153,4 @@ class ControllerConfigurationValidatorTest { assertThrows(classOf[InvalidConfigurationException], () => validator.validate( new ConfigResource(CLIENT_METRICS, "subscription-1"), config)). getMessage()) } -} \ No newline at end of file +} diff --git a/core/src/test/scala/unit/kafka/server/DeleteGroupsRequestTest.scala b/core/src/test/scala/unit/kafka/server/DeleteGroupsRequestTest.scala index 19e7087c8b03e..27856a36be906 100644 --- a/core/src/test/scala/unit/kafka/server/DeleteGroupsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/DeleteGroupsRequestTest.scala @@ -32,7 +32,6 @@ import org.junit.jupiter.api.extension.ExtendWith @Tag("integration") class DeleteGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -44,7 +43,6 @@ class DeleteGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -54,7 +52,6 @@ class DeleteGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -98,7 +95,8 @@ class DeleteGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinator leaveGroup( groupId = "grp", memberId = memberId, - useNewProtocol = useNewProtocol + useNewProtocol = useNewProtocol, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) ) deleteGroups( @@ -124,10 +122,7 @@ class DeleteGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinator .setGroupId("grp") .setGroupState(GenericGroupState.DEAD.toString) ), - describeGroups( - groupIds = List("grp"), - version = ApiKeys.DESCRIBE_GROUPS.latestVersion(isUnstableApiEnabled) - ) + describeGroups(List("grp")) ) } } diff --git a/core/src/test/scala/unit/kafka/server/DescribeClusterRequestTest.scala b/core/src/test/scala/unit/kafka/server/DescribeClusterRequestTest.scala index 7c260dae8532c..acb0a215b19e5 100644 --- a/core/src/test/scala/unit/kafka/server/DescribeClusterRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/DescribeClusterRequestTest.scala @@ -83,6 +83,8 @@ class DescribeClusterRequestTest extends BaseRequestTest { Int.MinValue } + ensureConsistentKRaftMetadata() + for (version <- ApiKeys.DESCRIBE_CLUSTER.oldestVersion to ApiKeys.DESCRIBE_CLUSTER.latestVersion) { val describeClusterRequest = new DescribeClusterRequest.Builder(new DescribeClusterRequestData() .setIncludeClusterAuthorizedOperations(includeClusterAuthorizedOperations)) diff --git a/core/src/test/scala/unit/kafka/server/DescribeGroupsRequestTest.scala b/core/src/test/scala/unit/kafka/server/DescribeGroupsRequestTest.scala index 0ba3b31b94dd4..6a1c9b4f92be3 100644 --- a/core/src/test/scala/unit/kafka/server/DescribeGroupsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/DescribeGroupsRequestTest.scala @@ -63,13 +63,13 @@ class DescribeGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinat ) // Join the consumer group. Complete the rebalance so that grp-1 is in STABLE state. - val (memberId1, _) = joinConsumerGroupWithOldProtocol( + val (memberId1, _) = joinDynamicConsumerGroupWithOldProtocol( groupId = "grp-1", metadata = Array(1, 2, 3), assignment = Array(4, 5, 6) ) // Join the consumer group. Not complete the rebalance so that grp-2 is in COMPLETING_REBALANCE state. - val (memberId2, _) = joinConsumerGroupWithOldProtocol( + val (memberId2, _) = joinDynamicConsumerGroupWithOldProtocol( groupId = "grp-2", metadata = Array(1, 2, 3), completeRebalance = false diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala index 861bbfe36117d..e6537edb02560 100755 --- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala @@ -58,7 +58,7 @@ class DynamicBrokerConfigTest { props.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, oldKeystore) val config = KafkaConfig(props) val dynamicConfig = config.dynamicConfig - dynamicConfig.initialize(None) + dynamicConfig.initialize(None, None) assertEquals(config, dynamicConfig.currentKafkaConfig) assertEquals(oldKeystore, config.values.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) @@ -116,7 +116,7 @@ class DynamicBrokerConfigTest { Mockito.when(logManagerMock.reconfigureDefaultLogConfig(ArgumentMatchers.any(classOf[LogConfig]))) .thenAnswer(invocation => currentDefaultLogConfig.set(invocation.getArgument(0))) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(new DynamicLogConfig(logManagerMock, serverMock)) val props = new Properties() @@ -155,7 +155,7 @@ class DynamicBrokerConfigTest { Mockito.when(serverMock.logManager).thenReturn(logManagerMock) Mockito.when(serverMock.kafkaScheduler).thenReturn(schedulerMock) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(new BrokerDynamicThreadPool(serverMock)) config.dynamicConfig.addReconfigurable(acceptorMock) @@ -204,7 +204,7 @@ class DynamicBrokerConfigTest { val origProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181) origProps.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "JKS") val config = KafkaConfig(origProps) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val validProps = Map(s"listener.name.external.${SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG}" -> "ks.p12") @@ -226,7 +226,7 @@ class DynamicBrokerConfigTest { val origProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181) origProps.put(KafkaConfig.LogCleanerDedupeBufferSizeProp, "100000000") val config = KafkaConfig(origProps) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val validProps = Map.empty[String, String] val invalidProps = Map(KafkaConfig.LogCleanerThreadsProp -> "20") @@ -330,7 +330,7 @@ class DynamicBrokerConfigTest { val configProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181) configProps.put(KafkaConfig.PasswordEncoderSecretProp, "broker.secret") val config = KafkaConfig(configProps) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val props = new Properties props.put(name, value) @@ -402,7 +402,7 @@ class DynamicBrokerConfigTest { props.put(KafkaConfig.SaslJaasConfigProp, "staticLoginModule required;") props.put(KafkaConfig.PasswordEncoderSecretProp, "config-encoder-secret") val config = KafkaConfig(props) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val dynamicProps = new Properties dynamicProps.put(KafkaConfig.SaslJaasConfigProp, "dynamicLoginModule required;") @@ -414,7 +414,7 @@ class DynamicBrokerConfigTest { // New config with same secret should use the dynamic password config val newConfigWithSameSecret = KafkaConfig(props) - newConfigWithSameSecret.dynamicConfig.initialize(None) + newConfigWithSameSecret.dynamicConfig.initialize(None, None) newConfigWithSameSecret.dynamicConfig.updateBrokerConfig(0, persistedProps) assertEquals("dynamicLoginModule required;", newConfigWithSameSecret.values.get(KafkaConfig.SaslJaasConfigProp).asInstanceOf[Password].value) @@ -478,7 +478,7 @@ class DynamicBrokerConfigTest { def testAuthorizerConfig(): Unit = { val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 9092) val oldConfig = KafkaConfig.fromProps(props) - oldConfig.dynamicConfig.initialize(None) + oldConfig.dynamicConfig.initialize(None, None) val kafkaServer: KafkaServer = mock(classOf[kafka.server.KafkaServer]) when(kafkaServer.config).thenReturn(oldConfig) @@ -525,7 +525,7 @@ class DynamicBrokerConfigTest { def testCombinedControllerAuthorizerConfig(): Unit = { val props = createCombinedControllerConfig(0, 9092) val oldConfig = KafkaConfig.fromProps(props) - oldConfig.dynamicConfig.initialize(None) + oldConfig.dynamicConfig.initialize(None, None) val controllerServer: ControllerServer = mock(classOf[kafka.server.ControllerServer]) when(controllerServer.config).thenReturn(oldConfig) @@ -571,7 +571,7 @@ class DynamicBrokerConfigTest { def testIsolatedControllerAuthorizerConfig(): Unit = { val props = createIsolatedControllerConfig(0, port = 9092) val oldConfig = KafkaConfig.fromProps(props) - oldConfig.dynamicConfig.initialize(None) + oldConfig.dynamicConfig.initialize(None, None) val controllerServer: ControllerServer = mock(classOf[kafka.server.ControllerServer]) when(controllerServer.config).thenReturn(oldConfig) @@ -615,7 +615,7 @@ class DynamicBrokerConfigTest { initialProps.remove(KafkaConfig.BackgroundThreadsProp) val oldConfig = KafkaConfig.fromProps(initialProps) val dynamicBrokerConfig = new DynamicBrokerConfig(oldConfig) - dynamicBrokerConfig.initialize(Some(zkClient)) + dynamicBrokerConfig.initialize(Some(zkClient), None) dynamicBrokerConfig.addBrokerReconfigurable(new TestDynamicThreadPool) val newprops = new Properties() @@ -628,7 +628,7 @@ class DynamicBrokerConfigTest { def testImproperConfigsAreRemoved(): Unit = { val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect) val config = KafkaConfig(props) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) assertEquals(Defaults.MaxConnections, config.maxConnections) assertEquals(LogConfig.DEFAULT_MAX_MESSAGE_BYTES, config.messageMaxBytes) @@ -663,7 +663,7 @@ class DynamicBrokerConfigTest { Mockito.when(serverMock.config).thenReturn(config) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val m = new DynamicMetricsReporters(brokerId, config, metrics, "clusterId") config.dynamicConfig.addReconfigurable(m) assertEquals(1, m.currentReporters.size) @@ -689,7 +689,7 @@ class DynamicBrokerConfigTest { Mockito.when(serverMock.config).thenReturn(config) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val m = new DynamicMetricsReporters(brokerId, config, metrics, "clusterId") config.dynamicConfig.addReconfigurable(m) assertTrue(m.currentReporters.isEmpty) @@ -722,7 +722,7 @@ class DynamicBrokerConfigTest { props.put(KafkaConfig.LogRetentionTimeMillisProp, "2592000000") val config = KafkaConfig(props) val dynamicLogConfig = new DynamicLogConfig(mock(classOf[LogManager]), mock(classOf[KafkaServer])) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(dynamicLogConfig) val newProps = new Properties() @@ -745,7 +745,7 @@ class DynamicBrokerConfigTest { props.put(KafkaConfig.LogRetentionBytesProp, "4294967296") val config = KafkaConfig(props) val dynamicLogConfig = new DynamicLogConfig(mock(classOf[LogManager]), mock(classOf[KafkaServer])) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(dynamicLogConfig) val newProps = new Properties() @@ -768,7 +768,7 @@ class DynamicBrokerConfigTest { props.put(RemoteLogManagerConfig.LOG_LOCAL_RETENTION_MS_PROP, "1000") props.put(RemoteLogManagerConfig.LOG_LOCAL_RETENTION_BYTES_PROP, "1024") val config = KafkaConfig(props) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) // Check for invalid localRetentionMs < -2 verifyConfigUpdateWithInvalidConfig(config, props, Map.empty, Map(RemoteLogManagerConfig.LOG_LOCAL_RETENTION_MS_PROP -> "-3")) @@ -800,7 +800,7 @@ class DynamicBrokerConfigTest { Mockito.when(serverMock.config).thenReturn(config) Mockito.when(serverMock.remoteLogManagerOpt).thenReturn(remoteLogManagerMockOpt) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(new DynamicRemoteLogConfig(serverMock)) val props = new Properties() @@ -822,7 +822,7 @@ class DynamicBrokerConfigTest { props.put(KafkaConfig.LogRetentionBytesProp, retentionBytes.toString) val config = KafkaConfig(props) val dynamicLogConfig = new DynamicLogConfig(mock(classOf[LogManager]), mock(classOf[KafkaServer])) - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) config.dynamicConfig.addBrokerReconfigurable(dynamicLogConfig) val newProps = new Properties() diff --git a/core/src/test/scala/unit/kafka/server/GroupCoordinatorBaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/GroupCoordinatorBaseRequestTest.scala index 6600d6698772c..96cd3fd31fa85 100644 --- a/core/src/test/scala/unit/kafka/server/GroupCoordinatorBaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/GroupCoordinatorBaseRequestTest.scala @@ -25,9 +25,9 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupR import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse import org.apache.kafka.common.message.SyncGroupRequestData.SyncGroupRequestAssignment -import org.apache.kafka.common.message.{ConsumerGroupHeartbeatRequestData, ConsumerGroupHeartbeatResponseData, DeleteGroupsRequestData, DeleteGroupsResponseData, DescribeGroupsRequestData, DescribeGroupsResponseData, JoinGroupRequestData, LeaveGroupResponseData, ListGroupsRequestData, ListGroupsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteRequestData, OffsetDeleteResponseData, OffsetFetchResponseData, SyncGroupRequestData} -import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ConsumerGroupHeartbeatRequest, ConsumerGroupHeartbeatResponse, DeleteGroupsRequest, DeleteGroupsResponse, DescribeGroupsRequest, DescribeGroupsResponse, JoinGroupRequest, JoinGroupResponse, LeaveGroupRequest, LeaveGroupResponse, ListGroupsRequest, ListGroupsResponse, OffsetCommitRequest, OffsetCommitResponse, OffsetDeleteRequest, OffsetDeleteResponse, OffsetFetchRequest, OffsetFetchResponse, SyncGroupRequest, SyncGroupResponse} +import org.apache.kafka.common.message.{ConsumerGroupHeartbeatRequestData, ConsumerGroupHeartbeatResponseData, DeleteGroupsRequestData, DeleteGroupsResponseData, DescribeGroupsRequestData, DescribeGroupsResponseData, HeartbeatRequestData, HeartbeatResponseData, JoinGroupRequestData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsRequestData, ListGroupsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteRequestData, OffsetDeleteResponseData, OffsetFetchResponseData, SyncGroupRequestData, SyncGroupResponseData} +import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ConsumerGroupHeartbeatRequest, ConsumerGroupHeartbeatResponse, DeleteGroupsRequest, DeleteGroupsResponse, DescribeGroupsRequest, DescribeGroupsResponse, HeartbeatRequest, HeartbeatResponse, JoinGroupRequest, JoinGroupResponse, LeaveGroupRequest, LeaveGroupResponse, ListGroupsRequest, ListGroupsResponse, OffsetCommitRequest, OffsetCommitResponse, OffsetDeleteRequest, OffsetDeleteResponse, OffsetFetchRequest, OffsetFetchResponse, SyncGroupRequest, SyncGroupResponse} import org.junit.jupiter.api.Assertions.{assertEquals, fail} import java.util.Comparator @@ -253,70 +253,132 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { groupId: String, memberId: String, generationId: Int, + protocolType: String = "consumer", + protocolName: String = "consumer-range", assignments: List[SyncGroupRequestData.SyncGroupRequestAssignment] = List.empty, - expectedError: Errors = Errors.NONE - ): Unit = { + expectedProtocolType: String = "consumer", + expectedProtocolName: String = "consumer-range", + expectedAssignment: Array[Byte] = Array.empty, + expectedError: Errors = Errors.NONE, + version: Short = ApiKeys.SYNC_GROUP.latestVersion(isUnstableApiEnabled) + ): SyncGroupResponseData = { val syncGroupRequestData = new SyncGroupRequestData() .setGroupId(groupId) .setMemberId(memberId) .setGenerationId(generationId) - .setProtocolType("consumer") - .setProtocolName("consumer-range") + .setProtocolType(protocolType) + .setProtocolName(protocolName) .setAssignments(assignments.asJava) - val syncGroupRequest = new SyncGroupRequest.Builder(syncGroupRequestData).build() + val syncGroupRequest = new SyncGroupRequest.Builder(syncGroupRequestData).build(version) val syncGroupResponse = connectAndReceive[SyncGroupResponse](syncGroupRequest) - assertEquals(expectedError.code, syncGroupResponse.data.errorCode) + + assertEquals( + new SyncGroupResponseData() + .setErrorCode(expectedError.code) + .setProtocolType(if (version >= 5) expectedProtocolType else null) + .setProtocolName(if (version >= 5) expectedProtocolName else null) + .setAssignment(expectedAssignment), + syncGroupResponse.data + ) + + syncGroupResponse.data } - protected def joinConsumerGroupWithOldProtocol( + protected def sendJoinRequest( groupId: String, + memberId: String = "", + groupInstanceId: String = null, + protocolType: String = "consumer", + protocolName: String = "consumer-range", metadata: Array[Byte] = Array.empty, - assignment: Array[Byte] = Array.empty, - completeRebalance: Boolean = true - ): (String, Int) = { + version: Short = ApiKeys.JOIN_GROUP.latestVersion(isUnstableApiEnabled) + ): JoinGroupResponseData = { val joinGroupRequestData = new JoinGroupRequestData() .setGroupId(groupId) + .setMemberId(memberId) + .setGroupInstanceId(groupInstanceId) .setRebalanceTimeoutMs(5 * 50 * 1000) .setSessionTimeoutMs(600000) - .setProtocolType("consumer") + .setProtocolType(protocolType) .setProtocols(new JoinGroupRequestData.JoinGroupRequestProtocolCollection( List( new JoinGroupRequestData.JoinGroupRequestProtocol() - .setName("consumer-range") + .setName(protocolName) .setMetadata(metadata) ).asJava.iterator )) - // Join the group as a dynamic member. // Send the request until receiving a successful response. There is a delay // here because the group coordinator is loaded in the background. - var joinGroupRequest = new JoinGroupRequest.Builder(joinGroupRequestData).build() + val joinGroupRequest = new JoinGroupRequest.Builder(joinGroupRequestData).build(version) var joinGroupResponse: JoinGroupResponse = null TestUtils.waitUntilTrue(() => { joinGroupResponse = connectAndReceive[JoinGroupResponse](joinGroupRequest) - joinGroupResponse.data.errorCode == Errors.MEMBER_ID_REQUIRED.code + joinGroupResponse != null }, msg = s"Could not join the group successfully. Last response $joinGroupResponse.") + joinGroupResponse.data + } + + protected def joinDynamicConsumerGroupWithOldProtocol( + groupId: String, + metadata: Array[Byte] = Array.empty, + assignment: Array[Byte] = Array.empty, + completeRebalance: Boolean = true + ): (String, Int) = { + val joinGroupResponseData = sendJoinRequest( + groupId = groupId, + metadata = metadata + ) + assertEquals(Errors.MEMBER_ID_REQUIRED.code, joinGroupResponseData.errorCode) + // Rejoin the group with the member id. - joinGroupRequestData.setMemberId(joinGroupResponse.data.memberId) - joinGroupRequest = new JoinGroupRequest.Builder(joinGroupRequestData).build() - joinGroupResponse = connectAndReceive[JoinGroupResponse](joinGroupRequest) - assertEquals(Errors.NONE.code, joinGroupResponse.data.errorCode) + val rejoinGroupResponseData = sendJoinRequest( + groupId = groupId, + memberId = joinGroupResponseData.memberId, + metadata = metadata + ) + assertEquals(Errors.NONE.code, rejoinGroupResponseData.errorCode) if (completeRebalance) { // Send the sync group request to complete the rebalance. syncGroupWithOldProtocol( groupId = groupId, - memberId = joinGroupResponse.data.memberId(), - generationId = joinGroupResponse.data.generationId(), + memberId = rejoinGroupResponseData.memberId, + generationId = rejoinGroupResponseData.generationId, assignments = List(new SyncGroupRequestAssignment() - .setMemberId(joinGroupResponse.data.memberId) - .setAssignment(assignment)) + .setMemberId(rejoinGroupResponseData.memberId) + .setAssignment(assignment)), + expectedAssignment = assignment ) } - (joinGroupResponse.data.memberId, joinGroupResponse.data.generationId) + (rejoinGroupResponseData.memberId, rejoinGroupResponseData.generationId) + } + + protected def joinStaticConsumerGroupWithOldProtocol( + groupId: String, + groupInstanceId: String, + metadata: Array[Byte] = Array.empty, + completeRebalance: Boolean = true + ): (String, Int) = { + val joinGroupResponseData = sendJoinRequest( + groupId = groupId, + groupInstanceId = groupInstanceId, + metadata = metadata + ) + + if (completeRebalance) { + // Send the sync group request to complete the rebalance. + syncGroupWithOldProtocol( + groupId = groupId, + memberId = joinGroupResponseData.memberId, + generationId = joinGroupResponseData.generationId + ) + } + + (joinGroupResponseData.memberId, joinGroupResponseData.generationId) } protected def joinConsumerGroupWithNewProtocol(groupId: String): (String, Int) = { @@ -337,7 +399,7 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { } else { // Note that we don't heartbeat and assume that the test will // complete within the session timeout. - joinConsumerGroupWithOldProtocol(groupId) + joinDynamicConsumerGroupWithOldProtocol(groupId = groupId) } } @@ -357,11 +419,10 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { protected def describeGroups( groupIds: List[String], - version: Short + version: Short = ApiKeys.DESCRIBE_GROUPS.latestVersion(isUnstableApiEnabled) ): List[DescribeGroupsResponseData.DescribedGroup] = { val describeGroupsRequest = new DescribeGroupsRequest.Builder( - new DescribeGroupsRequestData() - .setGroups(groupIds.asJava) + new DescribeGroupsRequestData().setGroups(groupIds.asJava) ).build(version) val describeGroupsResponse = connectAndReceive[DescribeGroupsResponse](describeGroupsRequest) @@ -369,6 +430,28 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { describeGroupsResponse.data.groups.asScala.toList } + protected def heartbeat( + groupId: String, + generationId: Int, + memberId: String, + groupInstanceId: String = null, + expectedError: Errors = Errors.NONE, + version: Short + ): HeartbeatResponseData = { + val heartbeatRequest = new HeartbeatRequest.Builder( + new HeartbeatRequestData() + .setGroupId(groupId) + .setGenerationId(generationId) + .setMemberId(memberId) + .setGroupInstanceId(groupInstanceId) + ).build(version) + + val heartbeatResponse = connectAndReceive[HeartbeatResponse](heartbeatRequest) + assertEquals(expectedError.code, heartbeatResponse.data.errorCode) + + heartbeatResponse.data + } + protected def consumerGroupHeartbeat( groupId: String, memberId: String = "", @@ -421,26 +504,32 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { protected def leaveGroupWithOldProtocol( groupId: String, memberIds: List[String], + groupInstanceIds: List[String] = null, expectedLeaveGroupError: Errors, - expectedMemberErrors: List[Errors] + expectedMemberErrors: List[Errors], + version: Short ): Unit = { - if (memberIds.size != expectedMemberErrors.size) { - fail("genericGroupLeave: memberIds and expectedMemberErrors have unmatched sizes.") - } - val leaveGroupRequest = new LeaveGroupRequest.Builder( groupId, - memberIds.map(memberId => new MemberIdentity().setMemberId(memberId)).asJava - ).build() + List.tabulate(memberIds.length) { i => + new MemberIdentity() + .setMemberId(memberIds(i)) + .setGroupInstanceId(if (groupInstanceIds == null) null else groupInstanceIds(i)) + }.asJava + ).build(version) val expectedResponseData = new LeaveGroupResponseData() - .setErrorCode(expectedLeaveGroupError.code) - .setMembers(List.tabulate(memberIds.length) { i => - new MemberResponse() - .setMemberId(memberIds(i)) - .setGroupInstanceId(null) - .setErrorCode(expectedMemberErrors(i).code) - }.asJava) + if (expectedLeaveGroupError != Errors.NONE) { + expectedResponseData.setErrorCode(expectedLeaveGroupError.code) + } else { + expectedResponseData + .setMembers(List.tabulate(expectedMemberErrors.length) { i => + new MemberResponse() + .setMemberId(memberIds(i)) + .setGroupInstanceId(if (groupInstanceIds == null) null else groupInstanceIds(i)) + .setErrorCode(expectedMemberErrors(i).code) + }.asJava) + } val leaveGroupResponse = connectAndReceive[LeaveGroupResponse](leaveGroupRequest) assertEquals(expectedResponseData, leaveGroupResponse.data) @@ -449,12 +538,13 @@ class GroupCoordinatorBaseRequestTest(cluster: ClusterInstance) { protected def leaveGroup( groupId: String, memberId: String, - useNewProtocol: Boolean + useNewProtocol: Boolean, + version: Short ): Unit = { if (useNewProtocol) { leaveGroupWithNewProtocol(groupId, memberId) } else { - leaveGroupWithOldProtocol(groupId, List(memberId), Errors.NONE, List(Errors.NONE)) + leaveGroupWithOldProtocol(groupId, List(memberId), null, Errors.NONE, List(Errors.NONE), version) } } diff --git a/core/src/test/scala/unit/kafka/server/HeartbeatRequestTest.scala b/core/src/test/scala/unit/kafka/server/HeartbeatRequestTest.scala new file mode 100644 index 0000000000000..0d7ea8bd39ceb --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/HeartbeatRequestTest.scala @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.test.ClusterInstance +import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, ClusterTestDefaults, Type} +import kafka.test.junit.ClusterTestExtensions +import kafka.utils.TestUtils +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol +import org.apache.kafka.common.message.SyncGroupRequestData +import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.coordinator.group.generic.GenericGroupState +import org.junit.jupiter.api.{Tag, Timeout} +import org.junit.jupiter.api.extension.ExtendWith + +import java.util.Collections +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +@Timeout(120) +@ExtendWith(value = Array(classOf[ClusterTestExtensions])) +@ClusterTestDefaults(clusterType = Type.KRAFT, brokers = 1) +@Tag("integration") +class HeartbeatRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { + @ClusterTest(serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testHeartbeatWithOldConsumerGroupProtocolAndNewGroupCoordinator(): Unit = { + testHeartbeat() + } + + @ClusterTest(clusterType = Type.ALL, serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testHeartbeatWithOldConsumerGroupProtocolAndOldGroupCoordinator(): Unit = { + testHeartbeat() + } + + private def testHeartbeat(): Unit = { + // Creates the __consumer_offsets topics because it won't be created automatically + // in this test because it does not use FindCoordinator API. + createOffsetsTopic() + + // Create the topic. + createTopic( + topic = "foo", + numPartitions = 3 + ) + + for (version <- ApiKeys.HEARTBEAT.oldestVersion() to ApiKeys.HEARTBEAT.latestVersion(isUnstableApiEnabled)) { + val metadata = ConsumerProtocol.serializeSubscription( + new ConsumerPartitionAssignor.Subscription(Collections.singletonList("foo")) + ).array + + val (leaderMemberId, leaderEpoch) = joinDynamicConsumerGroupWithOldProtocol( + groupId = "grp", + metadata = metadata, + completeRebalance = false + ) + + // Heartbeat with unknown group id and unknown member id. + heartbeat( + groupId = "grp-unknown", + memberId = "member-id-unknown", + generationId = -1, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + + // Heartbeat with unknown group id. + heartbeat( + groupId = "grp-unknown", + memberId = leaderMemberId, + generationId = -1, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + + // Heartbeat with unknown member id. + heartbeat( + groupId = "grp", + memberId = "member-id-unknown", + generationId = -1, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + + // Heartbeat with unmatched generation id. + heartbeat( + groupId = "grp", + memberId = leaderMemberId, + generationId = -1, + expectedError = Errors.ILLEGAL_GENERATION, + version = version.toShort + ) + + // Heartbeat COMPLETING_REBALANCE group. + heartbeat( + groupId = "grp", + memberId = leaderMemberId, + generationId = leaderEpoch, + version = version.toShort + ) + + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = leaderEpoch, + assignments = List(new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)) + ), + expectedAssignment = Array[Byte](1) + ) + + // Heartbeat STABLE group. + heartbeat( + groupId = "grp", + memberId = leaderMemberId, + generationId = leaderEpoch, + version = version.toShort + ) + + // Join the second member. + val joinFollowerResponseData = sendJoinRequest( + groupId = "grp", + metadata = metadata + ) + + Future { + sendJoinRequest( + groupId = "grp", + memberId = joinFollowerResponseData.memberId, + metadata = metadata + ) + } + + TestUtils.waitUntilTrue(() => { + val described = describeGroups(groupIds = List("grp")) + GenericGroupState.PREPARING_REBALANCE.toString == described.head.groupState + }, msg = s"The group is not in PREPARING_REBALANCE state.") + + // Heartbeat PREPARING_REBALANCE group. + heartbeat( + groupId = "grp", + memberId = leaderMemberId, + generationId = leaderEpoch, + expectedError = Errors.REBALANCE_IN_PROGRESS, + version = version.toShort + ) + + sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata + ) + + leaveGroup( + groupId = "grp", + memberId = leaderMemberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + leaveGroup( + groupId = "grp", + memberId = joinFollowerResponseData.memberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + + // Heartbeat empty group. + heartbeat( + groupId = "grp", + memberId = leaderMemberId, + generationId = -1, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + } + } +} diff --git a/core/src/test/scala/unit/kafka/server/JoinGroupRequestTest.scala b/core/src/test/scala/unit/kafka/server/JoinGroupRequestTest.scala new file mode 100644 index 0000000000000..34e0431d15235 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/JoinGroupRequestTest.scala @@ -0,0 +1,401 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.test.ClusterInstance +import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, ClusterTestDefaults, Type} +import kafka.test.junit.ClusterTestExtensions +import kafka.utils.TestUtils +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol +import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember +import org.apache.kafka.common.message.{JoinGroupResponseData, SyncGroupRequestData} +import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.coordinator.group.generic.GenericGroupState +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{Tag, Timeout} +import org.junit.jupiter.api.extension.ExtendWith + +import java.util.Collections +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} +import scala.jdk.CollectionConverters._ + +@Timeout(120) +@ExtendWith(value = Array(classOf[ClusterTestExtensions])) +@ClusterTestDefaults(clusterType = Type.KRAFT, brokers = 1) +@Tag("integration") +class JoinGroupRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { + @ClusterTest(serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testJoinGroupWithOldConsumerGroupProtocolAndNewGroupCoordinator(): Unit = { + testJoinGroup() + } + + @ClusterTest(clusterType = Type.ALL, serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testJoinGroupWithOldConsumerGroupProtocolAndOldGroupCoordinator(): Unit = { + testJoinGroup() + } + + private def testJoinGroup(): Unit = { + // Creates the __consumer_offsets topics because it won't be created automatically + // in this test because it does not use FindCoordinator API. + createOffsetsTopic() + + // Create the topic. + createTopic( + topic = "foo", + numPartitions = 3 + ) + + for (version <- ApiKeys.JOIN_GROUP.oldestVersion to ApiKeys.JOIN_GROUP.latestVersion(isUnstableApiEnabled)) { + val metadata = ConsumerProtocol.serializeSubscription( + new ConsumerPartitionAssignor.Subscription(Collections.singletonList("foo")) + ).array + + // Join a dynamic member without member id. + // Prior to JoinGroup version 4, a new member is immediately added if it sends a join group request with UNKNOWN_MEMBER_ID. + val joinLeaderResponseData = sendJoinRequest( + groupId = "grp", + metadata = metadata, + version = version.toShort + ) + val leaderMemberId = joinLeaderResponseData.memberId + if (version >= 4) { + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setErrorCode(Errors.MEMBER_ID_REQUIRED.code) + .setMemberId(leaderMemberId) + .setProtocolName(if (version >= 7) null else ""), + joinLeaderResponseData + ) + } else { + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(1) + .setLeader(leaderMemberId) + .setMemberId(leaderMemberId) + .setProtocolName("consumer-range") + .setMembers(List(new JoinGroupResponseMember() + .setMemberId(leaderMemberId) + .setMetadata(metadata) + ).asJava), + joinLeaderResponseData + ) + } + + // Rejoin the group with the member id. + if (version >= 4) { + val rejoinLeaderResponseData = sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata, + version = version.toShort + ) + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(1) + .setMemberId(leaderMemberId) + .setProtocolName("consumer-range") + .setProtocolType(if (version >= 7) "consumer" else null) + .setLeader(leaderMemberId) + .setMembers(List(new JoinGroupResponseMember() + .setMemberId(leaderMemberId) + .setMetadata(metadata) + ).asJava), + rejoinLeaderResponseData + ) + } + + // Send a SyncGroup request. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 1, + assignments = List(new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)) + ), + expectedAssignment = Array[Byte](1) + ) + + // Join with an unknown member id. + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setMemberId("member-id-unknown") + .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code) + .setProtocolName(if (version >= 7) null else ""), + sendJoinRequest( + groupId = "grp", + memberId = "member-id-unknown", + version = version.toShort + ) + ) + + // Join with an inconsistent protocolType. + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setErrorCode(Errors.INCONSISTENT_GROUP_PROTOCOL.code) + .setProtocolName(if (version >= 7) null else ""), + sendJoinRequest( + groupId = "grp", + protocolType = "connect", + version = version.toShort + ) + ) + + // Join a second member. + // Non-null group instance id is not supported until JoinGroup version 5, + // so only version 4 needs to join a dynamic group (version < 5) and needs an extra join request to get the member id (version > 3). + var joinFollowerResponseData: JoinGroupResponseData = null + if (version == 4) { + joinFollowerResponseData = sendJoinRequest( + groupId = "grp", + metadata = metadata, + version = version.toShort + ) + } + + val joinFollowerFuture = Future { + sendJoinRequest( + groupId = "grp", + memberId = if (version != 4) "" else joinFollowerResponseData.memberId, + groupInstanceId = if (version >= 5) "group-instance-id" else null, + metadata = metadata, + version = version.toShort + ) + } + + TestUtils.waitUntilTrue(() => { + val described = describeGroups(groupIds = List("grp")) + GenericGroupState.PREPARING_REBALANCE.toString == described.head.groupState + }, msg = s"The group is not in PREPARING_REBALANCE state.") + + // The leader rejoins. + val rejoinLeaderResponseData = sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata, + version = version.toShort + ) + + val joinFollowerFutureResponseData = Await.result(joinFollowerFuture, Duration.Inf) + var followerMemberId = joinFollowerFutureResponseData.memberId + + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(2) + .setProtocolType(if (version >= 7) "consumer" else null) + .setProtocolName("consumer-range") + .setLeader(leaderMemberId) + .setMemberId(followerMemberId), + joinFollowerFutureResponseData + ) + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(2) + .setProtocolType(if (version >= 7) "consumer" else null) + .setProtocolName("consumer-range") + .setLeader(leaderMemberId) + .setMemberId(leaderMemberId) + .setMembers(List( + new JoinGroupResponseMember() + .setMemberId(leaderMemberId) + .setMetadata(metadata), + new JoinGroupResponseMember() + .setMemberId(followerMemberId) + .setGroupInstanceId(if (version >= 5) "group-instance-id" else null) + .setMetadata(metadata) + ).asJava), + rejoinLeaderResponseData + ) + + // Sync the leader. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = rejoinLeaderResponseData.generationId, + assignments = List( + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)), + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(followerMemberId) + .setAssignment(Array[Byte](2)) + ), + expectedAssignment = Array[Byte](1) + ) + + // Sync the follower. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = followerMemberId, + generationId = joinFollowerFutureResponseData.generationId, + expectedAssignment = Array[Byte](2) + ) + + // The follower rejoin doesn't trigger a rebalance if it's unchanged. + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(2) + .setProtocolType(if (version >= 7) "consumer" else null) + .setProtocolName("consumer-range") + .setLeader(leaderMemberId) + .setMemberId(followerMemberId), + sendJoinRequest( + groupId = "grp", + groupInstanceId = if (version >= 5) "group-instance-id" else null, + memberId = followerMemberId, + metadata = metadata, + version = version.toShort + ) + ) + + if (version >= 5) { + followerMemberId = testFencedStaticGroup(leaderMemberId, followerMemberId, metadata, version) + } + + leaveGroup( + groupId = "grp", + memberId = leaderMemberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + leaveGroup( + groupId = "grp", + memberId = followerMemberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + + deleteGroups( + groupIds = List("grp"), + expectedErrors = List(Errors.NONE), + version = ApiKeys.DELETE_GROUPS.latestVersion(isUnstableApiEnabled) + ) + } + } + + private def testFencedStaticGroup( + leaderMemberId: String, + followerMemberId: String, + metadata: Array[Byte], + version: Int, + ): String = { + // The leader rejoins and triggers a rebalance. + val rejoinLeaderFuture = Future { + sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata, + version = version.toShort + ) + } + + TestUtils.waitUntilTrue(() => { + val described = describeGroups(groupIds = List("grp")) + GenericGroupState.PREPARING_REBALANCE.toString == described.head.groupState + }, msg = s"The group is not in PREPARING_REBALANCE state.") + + // A new follower with duplicated group instance id joins. + val joinNewFollowerResponseData = sendJoinRequest( + groupId = "grp", + groupInstanceId = "group-instance-id", + metadata = metadata, + version = version.toShort + ) + + TestUtils.waitUntilTrue(() => { + val described = describeGroups(groupIds = List("grp")) + GenericGroupState.COMPLETING_REBALANCE.toString == described.head.groupState + }, msg = s"The group is not in COMPLETING_REBALANCE state.") + + // The old follower rejoin request should be fenced. + val rejoinFollowerResponseData = sendJoinRequest( + groupId = "grp", + memberId = followerMemberId, + groupInstanceId = "group-instance-id", + metadata = metadata, + version = version.toShort + ) + + val rejoinLeaderFutureResponseData = Await.result(rejoinLeaderFuture, Duration.Inf) + val newFollowerMemberId = joinNewFollowerResponseData.memberId + + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(3) + .setProtocolType(if (version >= 7) "consumer" else null) + .setProtocolName("consumer-range") + .setLeader(leaderMemberId) + .setMemberId(leaderMemberId) + .setMembers(List( + new JoinGroupResponseMember() + .setMemberId(leaderMemberId) + .setMetadata(metadata), + new JoinGroupResponseMember() + .setMemberId(newFollowerMemberId) + .setGroupInstanceId("group-instance-id") + .setMetadata(metadata) + ).asJava), + rejoinLeaderFutureResponseData + ) + + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setGenerationId(3) + .setProtocolType(if (version >= 7) "consumer" else null) + .setProtocolName("consumer-range") + .setLeader(leaderMemberId) + .setMemberId(newFollowerMemberId), + joinNewFollowerResponseData + ) + + verifyJoinGroupResponseDataEquals( + new JoinGroupResponseData() + .setProtocolName(if (version >= 7) null else "") + .setMemberId(followerMemberId) + .setErrorCode(Errors.FENCED_INSTANCE_ID.code), + rejoinFollowerResponseData + ) + + newFollowerMemberId + } + + private def normalize(responseData: JoinGroupResponseData): JoinGroupResponseData = { + val newResponseData = responseData.duplicate + Collections.sort(newResponseData.members, + (m1: JoinGroupResponseMember, m2: JoinGroupResponseMember) => m1.memberId.compareTo(m2.memberId) + ) + newResponseData + } + + private def verifyJoinGroupResponseDataEquals( + expected: JoinGroupResponseData, + actual: JoinGroupResponseData + ): Unit = { + assertEquals(normalize(expected), normalize(actual)) + } +} diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 17abdb0472d24..bf5ce8750d2d1 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -24,10 +24,10 @@ import java.util.Arrays.asList import java.util.concurrent.{CompletableFuture, TimeUnit} import java.util.{Collections, Optional, OptionalInt, OptionalLong, Properties} import kafka.api.LeaderAndIsr -import kafka.cluster.Broker +import kafka.cluster.{Broker, Partition} import kafka.controller.{ControllerContext, KafkaController} import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} -import kafka.metrics.ClientMetricsTestUtils +import kafka.log.UnifiedLog import kafka.network.{RequestChannel, RequestMetrics} import kafka.server.QuotaFactory.QuotaManagers import kafka.server.metadata.{ConfigRepository, KRaftMetadataCache, MockConfigRepository, ZkMetadataCache} @@ -95,10 +95,12 @@ import org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartiti import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult import org.apache.kafka.common.message.OffsetDeleteResponseData.{OffsetDeleteResponsePartition, OffsetDeleteResponsePartitionCollection, OffsetDeleteResponseTopic, OffsetDeleteResponseTopicCollection} import org.apache.kafka.coordinator.group.GroupCoordinator +import org.apache.kafka.server.ClientMetricsManager import org.apache.kafka.server.common.{Features, MetadataVersion} import org.apache.kafka.server.common.MetadataVersion.{IBP_0_10_2_IV0, IBP_2_2_IV1} +import org.apache.kafka.server.metrics.ClientMetricsTestUtils import org.apache.kafka.server.util.MockTime -import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData} +import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData, LogConfig} class KafkaApisTest { private val requestChannel: RequestChannel = mock(classOf[RequestChannel]) @@ -128,6 +130,7 @@ class KafkaApisTest { private val quotas = QuotaManagers(clientQuotaManager, clientQuotaManager, clientRequestQuotaManager, clientControllerQuotaManager, replicaQuotaManager, replicaQuotaManager, replicaQuotaManager, None) private val fetchManager: FetchManager = mock(classOf[FetchManager]) + private val clientMetricsManager: ClientMetricsManager = mock(classOf[ClientMetricsManager]) private val brokerTopicStats = new BrokerTopicStats private val clusterId = "clusterId" private val time = new MockTime @@ -195,6 +198,8 @@ class KafkaApisTest { false, () => new Features(MetadataVersion.latest(), Collections.emptyMap[String, java.lang.Short], 0, raftSupport)) + val clientMetricsManagerOpt = if (raftSupport) Some(clientMetricsManager) else None + new KafkaApis( requestChannel = requestChannel, metadataSupport = metadataSupport, @@ -215,7 +220,7 @@ class KafkaApisTest { time = time, tokenManager = null, apiVersionManager = apiVersionManager, - clientMetricsManager = null) + clientMetricsManager = clientMetricsManagerOpt) } @Test @@ -2475,6 +2480,204 @@ class KafkaApisTest { } } + @Test + def testProduceResponseContainsNewLeaderOnNotLeaderOrFollower(): Unit = { + val topic = "topic" + addTopicToMetadataCache(topic, numPartitions = 2, numBrokers = 3) + + for (version <- 10 to ApiKeys.PRODUCE.latestVersion) { + + reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) + + val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + + val tp = new TopicPartition(topic, 0) + val partition = mock(classOf[Partition]) + val newLeaderId = 2 + val newLeaderEpoch = 5 + + val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() + .setTopicData(new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(new ProduceRequestData.TopicProduceData() + .setName(tp.topic).setPartitionData(Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(tp.partition) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) + .iterator)) + .setAcks(1.toShort) + .setTimeoutMs(5000)) + .build(version.toShort) + val request = buildRequest(produceRequest) + + when(replicaManager.appendRecords(anyLong, + anyShort, + ArgumentMatchers.eq(false), + ArgumentMatchers.eq(AppendOrigin.CLIENT), + any(), + responseCallback.capture(), + any(), + any(), + any(), + any(), + any()) + ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) + + when(replicaManager.getPartitionOrError(tp)).thenAnswer(_ => Right(partition)) + when(partition.leaderReplicaIdOpt).thenAnswer(_ => Some(newLeaderId)) + when(partition.getLeaderEpoch).thenAnswer(_ => newLeaderEpoch) + + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + any[Long])).thenReturn(0) + when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( + any[RequestChannel.Request](), anyDouble, anyLong)).thenReturn(0) + + createKafkaApis().handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) + + val response = verifyNoThrottling[ProduceResponse](request) + + assertEquals(1, response.data.responses.size) + val topicProduceResponse = response.data.responses.asScala.head + assertEquals(1, topicProduceResponse.partitionResponses.size) + val partitionProduceResponse = topicProduceResponse.partitionResponses.asScala.head + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, Errors.forCode(partitionProduceResponse.errorCode)) + assertEquals(newLeaderId, partitionProduceResponse.currentLeader.leaderId()) + assertEquals(newLeaderEpoch, partitionProduceResponse.currentLeader.leaderEpoch()) + assertEquals(1, response.data.nodeEndpoints.size) + val node = response.data.nodeEndpoints.asScala.head + assertEquals(2, node.nodeId) + assertEquals("broker2", node.host) + } + } + + @Test + def testProduceResponseReplicaManagerLookupErrorOnNotLeaderOrFollower(): Unit = { + val topic = "topic" + addTopicToMetadataCache(topic, numPartitions = 2, numBrokers = 3) + + for (version <- 10 to ApiKeys.PRODUCE.latestVersion) { + + reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) + + val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + + val tp = new TopicPartition(topic, 0) + + val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() + .setTopicData(new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(new ProduceRequestData.TopicProduceData() + .setName(tp.topic).setPartitionData(Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(tp.partition) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) + .iterator)) + .setAcks(1.toShort) + .setTimeoutMs(5000)) + .build(version.toShort) + val request = buildRequest(produceRequest) + + when(replicaManager.appendRecords(anyLong, + anyShort, + ArgumentMatchers.eq(false), + ArgumentMatchers.eq(AppendOrigin.CLIENT), + any(), + responseCallback.capture(), + any(), + any(), + any(), + any(), + any()) + ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) + + when(replicaManager.getPartitionOrError(tp)).thenAnswer(_ => Left(Errors.UNKNOWN_TOPIC_OR_PARTITION)) + + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + any[Long])).thenReturn(0) + when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( + any[RequestChannel.Request](), anyDouble, anyLong)).thenReturn(0) + + createKafkaApis().handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) + + val response = verifyNoThrottling[ProduceResponse](request) + + assertEquals(1, response.data.responses.size) + val topicProduceResponse = response.data.responses.asScala.head + assertEquals(1, topicProduceResponse.partitionResponses.size) + val partitionProduceResponse = topicProduceResponse.partitionResponses.asScala.head + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, Errors.forCode(partitionProduceResponse.errorCode)) + // LeaderId and epoch should be the same values inserted into the metadata cache + assertEquals(0, partitionProduceResponse.currentLeader.leaderId()) + assertEquals(1, partitionProduceResponse.currentLeader.leaderEpoch()) + assertEquals(1, response.data.nodeEndpoints.size) + val node = response.data.nodeEndpoints.asScala.head + assertEquals(0, node.nodeId) + assertEquals("broker0", node.host) + } + } + + @Test + def testProduceResponseMetadataLookupErrorOnNotLeaderOrFollower(): Unit = { + val topic = "topic" + metadataCache = mock(classOf[ZkMetadataCache]) + + for (version <- 10 to ApiKeys.PRODUCE.latestVersion) { + + reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) + + val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + + val tp = new TopicPartition(topic, 0) + + val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() + .setTopicData(new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(new ProduceRequestData.TopicProduceData() + .setName(tp.topic).setPartitionData(Collections.singletonList( + new ProduceRequestData.PartitionProduceData() + .setIndex(tp.partition) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) + .iterator)) + .setAcks(1.toShort) + .setTimeoutMs(5000)) + .build(version.toShort) + val request = buildRequest(produceRequest) + + when(replicaManager.appendRecords(anyLong, + anyShort, + ArgumentMatchers.eq(false), + ArgumentMatchers.eq(AppendOrigin.CLIENT), + any(), + responseCallback.capture(), + any(), + any(), + any(), + any(), + any()) + ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) + + when(replicaManager.getPartitionOrError(tp)).thenAnswer(_ => Left(Errors.UNKNOWN_TOPIC_OR_PARTITION)) + + when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), + any[Long])).thenReturn(0) + when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( + any[RequestChannel.Request](), anyDouble, anyLong)).thenReturn(0) + when(metadataCache.contains(tp)).thenAnswer(_ => true) + when(metadataCache.getPartitionInfo(tp.topic(), tp.partition())).thenAnswer(_ => Option.empty) + when(metadataCache.getAliveBrokerNode(any(), any())).thenReturn(Option.empty) + + createKafkaApis().handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) + + val response = verifyNoThrottling[ProduceResponse](request) + + assertEquals(1, response.data.responses.size) + val topicProduceResponse = response.data.responses.asScala.head + assertEquals(1, topicProduceResponse.partitionResponses.size) + val partitionProduceResponse = topicProduceResponse.partitionResponses.asScala.head + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, Errors.forCode(partitionProduceResponse.errorCode)) + assertEquals(-1, partitionProduceResponse.currentLeader.leaderId()) + assertEquals(-1, partitionProduceResponse.currentLeader.leaderEpoch()) + assertEquals(0, response.data.nodeEndpoints.size) + } + } + @Test def testTransactionalParametersSetCorrectly(): Unit = { val topic = "topic" @@ -3786,6 +3989,73 @@ class KafkaApisTest { assertEquals(MemoryRecords.EMPTY, FetchResponse.recordsOrFail(partitionData)) } + @Test + def testFetchResponseContainsNewLeaderOnNotLeaderOrFollower(): Unit = { + val topicId = Uuid.randomUuid() + val tidp = new TopicIdPartition(topicId, new TopicPartition("foo", 0)) + val tp = tidp.topicPartition + addTopicToMetadataCache(tp.topic, numPartitions = 1, numBrokers = 3, topicId) + + when(replicaManager.getLogConfig(ArgumentMatchers.eq(tp))).thenReturn(Some(LogConfig.fromProps( + Collections.emptyMap(), + new Properties() + ))) + + val partition = mock(classOf[Partition]) + val newLeaderId = 2 + val newLeaderEpoch = 5 + + when(replicaManager.getPartitionOrError(tp)).thenAnswer(_ => Right(partition)) + when(partition.leaderReplicaIdOpt).thenAnswer(_ => Some(newLeaderId)) + when(partition.getLeaderEpoch).thenAnswer(_ => newLeaderEpoch) + + when(replicaManager.fetchMessages( + any[FetchParams], + any[Seq[(TopicIdPartition, FetchRequest.PartitionData)]], + any[ReplicaQuota], + any[Seq[(TopicIdPartition, FetchPartitionData)] => Unit]() + )).thenAnswer(invocation => { + val callback = invocation.getArgument(3).asInstanceOf[Seq[(TopicIdPartition, FetchPartitionData)] => Unit] + callback(Seq(tidp -> new FetchPartitionData(Errors.NOT_LEADER_OR_FOLLOWER, UnifiedLog.UnknownOffset, UnifiedLog.UnknownOffset, MemoryRecords.EMPTY, + Optional.empty(), OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), false))) + }) + + val fetchData = Map(tidp -> new FetchRequest.PartitionData(Uuid.ZERO_UUID, 0, 0, 1000, + Optional.empty())).asJava + val fetchDataBuilder = Map(tp -> new FetchRequest.PartitionData(Uuid.ZERO_UUID, 0, 0, 1000, + Optional.empty())).asJava + val fetchMetadata = new JFetchMetadata(0, 0) + val fetchContext = new FullFetchContext(time, new FetchSessionCache(1000, 100), + fetchMetadata, fetchData, false, false) + when(fetchManager.newContext( + any[Short], + any[JFetchMetadata], + any[Boolean], + any[util.Map[TopicIdPartition, FetchRequest.PartitionData]], + any[util.List[TopicIdPartition]], + any[util.Map[Uuid, String]])).thenReturn(fetchContext) + + when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs( + any[RequestChannel.Request](), anyDouble, anyLong)).thenReturn(0) + + val fetchRequest = new FetchRequest.Builder(16, 16, -1, -1, 100, 0, fetchDataBuilder) + .build() + val request = buildRequest(fetchRequest) + + createKafkaApis().handleFetchRequest(request) + + val response = verifyNoThrottling[FetchResponse](request) + val responseData = response.responseData(metadataCache.topicIdsToNames(), 16) + + val partitionData = responseData.get(tp) + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER.code, partitionData.errorCode) + assertEquals(newLeaderId, partitionData.currentLeader.leaderId()) + assertEquals(newLeaderEpoch, partitionData.currentLeader.leaderEpoch()) + val node = response.data.nodeEndpoints.asScala.head + assertEquals(2, node.nodeId) + assertEquals("broker2", node.host) + } + @ParameterizedTest @ApiKeyVersionsSource(apiKey = ApiKeys.JOIN_GROUP) def testHandleJoinGroupRequest(version: Short): Unit = { @@ -6465,12 +6735,45 @@ class KafkaApisTest { assertEquals(Errors.GROUP_AUTHORIZATION_FAILED.code, response.data.errorCode) } + @Test + def testConsumerGroupDescribe(): Unit = { + val groupIds = List("group-id-0", "group-id-1", "group-id-2").asJava + val consumerGroupDescribeRequestData = new ConsumerGroupDescribeRequestData() + consumerGroupDescribeRequestData.groupIds.addAll(groupIds) + val requestChannelRequest = buildRequest(new ConsumerGroupDescribeRequest.Builder(consumerGroupDescribeRequestData, true).build()) + + val future = new CompletableFuture[util.List[ConsumerGroupDescribeResponseData.DescribedGroup]]() + when(groupCoordinator.consumerGroupDescribe( + any[RequestContext], + any[util.List[String]] + )).thenReturn(future) + + createKafkaApis( + overrideProperties = Map(KafkaConfig.NewGroupCoordinatorEnableProp -> "true") + ).handle(requestChannelRequest, RequestLocal.NoCaching) + + val describedGroups = List( + new DescribedGroup().setGroupId(groupIds.get(0)), + new DescribedGroup().setGroupId(groupIds.get(1)), + new DescribedGroup().setGroupId(groupIds.get(2)) + ).asJava + + future.complete(describedGroups) + val expectedConsumerGroupDescribeResponseData = new ConsumerGroupDescribeResponseData() + .setGroups(describedGroups) + + val response = verifyNoThrottling[ConsumerGroupDescribeResponse](requestChannelRequest) + + assertEquals(expectedConsumerGroupDescribeResponseData, response.data) + } + @Test def testConsumerGroupDescribeReturnsUnsupportedVersion(): Unit = { val groupId = "group0" val consumerGroupDescribeRequestData = new ConsumerGroupDescribeRequestData() consumerGroupDescribeRequestData.groupIds.add(groupId) val requestChannelRequest = buildRequest(new ConsumerGroupDescribeRequest.Builder(consumerGroupDescribeRequestData, true).build()) + val errorCode = Errors.UNSUPPORTED_VERSION.code val expectedDescribedGroup = new DescribedGroup().setGroupId(groupId).setErrorCode(errorCode) val expectedResponse = new ConsumerGroupDescribeResponseData() @@ -6482,6 +6785,53 @@ class KafkaApisTest { assertEquals(expectedResponse, response.data) } + @Test + def testConsumerGroupDescribeAuthorizationFailed(): Unit = { + val consumerGroupDescribeRequestData = new ConsumerGroupDescribeRequestData() + consumerGroupDescribeRequestData.groupIds.add("group-id") + val requestChannelRequest = buildRequest(new ConsumerGroupDescribeRequest.Builder(consumerGroupDescribeRequestData, true).build()) + + val authorizer: Authorizer = mock(classOf[Authorizer]) + when(authorizer.authorize(any[RequestContext], any[util.List[Action]])) + .thenReturn(Seq(AuthorizationResult.DENIED).asJava) + + val future = new CompletableFuture[util.List[ConsumerGroupDescribeResponseData.DescribedGroup]]() + when(groupCoordinator.consumerGroupDescribe( + any[RequestContext], + any[util.List[String]] + )).thenReturn(future) + future.complete(List().asJava) + + createKafkaApis( + authorizer = Some(authorizer), + overrideProperties = Map(KafkaConfig.NewGroupCoordinatorEnableProp -> "true") + ).handle(requestChannelRequest, RequestLocal.NoCaching) + + val response = verifyNoThrottling[ConsumerGroupDescribeResponse](requestChannelRequest) + assertEquals(Errors.GROUP_AUTHORIZATION_FAILED.code, response.data.groups.get(0).errorCode) + } + + @Test + def testConsumerGroupDescribeFutureFailed(): Unit = { + val consumerGroupDescribeRequestData = new ConsumerGroupDescribeRequestData() + consumerGroupDescribeRequestData.groupIds.add("group-id") + val requestChannelRequest = buildRequest(new ConsumerGroupDescribeRequest.Builder(consumerGroupDescribeRequestData, true).build()) + + val future = new CompletableFuture[util.List[ConsumerGroupDescribeResponseData.DescribedGroup]]() + when(groupCoordinator.consumerGroupDescribe( + any[RequestContext], + any[util.List[String]] + )).thenReturn(future) + + createKafkaApis(overrideProperties = Map( + KafkaConfig.NewGroupCoordinatorEnableProp -> "true" + )).handle(requestChannelRequest, RequestLocal.NoCaching) + + future.completeExceptionally(Errors.FENCED_MEMBER_EPOCH.exception) + val response = verifyNoThrottling[ConsumerGroupDescribeResponse](requestChannelRequest) + assertEquals(Errors.FENCED_MEMBER_EPOCH.code, response.data.groups.get(0).errorCode) + } + @Test def testGetTelemetrySubscriptionsNotAllowedForZkClusters(): Unit = { val data = new GetTelemetrySubscriptionsRequestData() @@ -6494,18 +6844,39 @@ class KafkaApisTest { } @Test - def testGetTelemetrySubscriptionsUnsupportedVersionForKRaftClusters(): Unit = { - val data = new GetTelemetrySubscriptionsRequestData() + def testGetTelemetrySubscriptions(): Unit = { + val request = buildRequest(new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData(), true).build()) + + when(clientMetricsManager.isTelemetryReceiverConfigured).thenReturn(true) + when(clientMetricsManager.processGetTelemetrySubscriptionRequest(any[GetTelemetrySubscriptionsRequest](), + any[RequestContext]())).thenReturn(new GetTelemetrySubscriptionsResponse( + new GetTelemetrySubscriptionsResponseData())) + + metadataCache = MetadataCache.kRaftMetadataCache(brokerId) + createKafkaApis(raftSupport = true).handle(request, RequestLocal.NoCaching) + + val response = verifyNoThrottling[GetTelemetrySubscriptionsResponse](request) - val request = buildRequest(new GetTelemetrySubscriptionsRequest.Builder(data, true).build()) - val errorCode = Errors.UNSUPPORTED_VERSION.code val expectedResponse = new GetTelemetrySubscriptionsResponseData() - expectedResponse.setErrorCode(errorCode) + assertEquals(expectedResponse, response.data) + } + + @Test + def testGetTelemetrySubscriptionsWithException(): Unit = { + val request = buildRequest(new GetTelemetrySubscriptionsRequest.Builder( + new GetTelemetrySubscriptionsRequestData(), true).build()) + + when(clientMetricsManager.isTelemetryReceiverConfigured).thenReturn(true) + when(clientMetricsManager.processGetTelemetrySubscriptionRequest(any[GetTelemetrySubscriptionsRequest](), + any[RequestContext]())).thenThrow(new RuntimeException("test")) metadataCache = MetadataCache.kRaftMetadataCache(brokerId) createKafkaApis(raftSupport = true).handle(request, RequestLocal.NoCaching) + val response = verifyNoThrottling[GetTelemetrySubscriptionsResponse](request) + val expectedResponse = new GetTelemetrySubscriptionsResponseData().setErrorCode(Errors.INVALID_REQUEST.code) assertEquals(expectedResponse, response.data) } @@ -6521,18 +6892,34 @@ class KafkaApisTest { } @Test - def testPushTelemetryUnsupportedVersionForKRaftClusters(): Unit = { - val data = new PushTelemetryRequestData() + def testPushTelemetry(): Unit = { + val request = buildRequest(new PushTelemetryRequest.Builder(new PushTelemetryRequestData(), true).build()) - val request = buildRequest(new PushTelemetryRequest.Builder(data, true).build()) - val errorCode = Errors.UNSUPPORTED_VERSION.code - val expectedResponse = new PushTelemetryResponseData() - expectedResponse.setErrorCode(errorCode) + when(clientMetricsManager.isTelemetryReceiverConfigured).thenReturn(true) + when(clientMetricsManager.processPushTelemetryRequest(any[PushTelemetryRequest](), any[RequestContext]())) + .thenReturn(new PushTelemetryResponse(new PushTelemetryResponseData())) + + metadataCache = MetadataCache.kRaftMetadataCache(brokerId) + createKafkaApis(raftSupport = true).handle(request, RequestLocal.NoCaching) + val response = verifyNoThrottling[PushTelemetryResponse](request) + + val expectedResponse = new PushTelemetryResponseData().setErrorCode(Errors.NONE.code) + assertEquals(expectedResponse, response.data) + } + + @Test + def testPushTelemetryWithException(): Unit = { + val request = buildRequest(new PushTelemetryRequest.Builder(new PushTelemetryRequestData(), true).build()) + + when(clientMetricsManager.isTelemetryReceiverConfigured()).thenReturn(true) + when(clientMetricsManager.processPushTelemetryRequest(any[PushTelemetryRequest](), any[RequestContext]())) + .thenThrow(new RuntimeException("test")) metadataCache = MetadataCache.kRaftMetadataCache(brokerId) createKafkaApis(raftSupport = true).handle(request, RequestLocal.NoCaching) val response = verifyNoThrottling[PushTelemetryResponse](request) + val expectedResponse = new PushTelemetryResponseData().setErrorCode(Errors.INVALID_REQUEST.code) assertEquals(expectedResponse, response.data) } } diff --git a/core/src/test/scala/unit/kafka/server/LeaveGroupRequestTest.scala b/core/src/test/scala/unit/kafka/server/LeaveGroupRequestTest.scala new file mode 100644 index 0000000000000..3e04b267584bb --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/LeaveGroupRequestTest.scala @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.test.ClusterInstance +import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, ClusterTestDefaults, Type} +import kafka.test.junit.ClusterTestExtensions +import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.requests.JoinGroupRequest +import org.apache.kafka.coordinator.group.generic.GenericGroupState +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{Tag, Timeout} +import org.junit.jupiter.api.extension.ExtendWith + +@Timeout(120) +@ExtendWith(value = Array(classOf[ClusterTestExtensions])) +@ClusterTestDefaults(clusterType = Type.KRAFT, brokers = 1) +@Tag("integration") +class LeaveGroupRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { + @ClusterTest(serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testLeaveGroupWithOldConsumerGroupProtocolAndNewGroupCoordinator(): Unit = { + testLeaveGroup() + } + + @ClusterTest(clusterType = Type.ALL, serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testLeaveGroupWithOldConsumerGroupProtocolAndOldGroupCoordinator(): Unit = { + testLeaveGroup() + } + + private def testLeaveGroup(): Unit = { + // Creates the __consumer_offsets topics because it won't be created automatically + // in this test because it does not use FindCoordinator API. + createOffsetsTopic() + + // Create the topic. + createTopic( + topic = "foo", + numPartitions = 3 + ) + + for (version <- ApiKeys.LEAVE_GROUP.oldestVersion() to ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) { + // Join the consumer group. Note that we don't heartbeat here so we must use + // a session long enough for the duration of the test. + val (memberId1, _) = joinDynamicConsumerGroupWithOldProtocol("grp-1") + if (version >= 3) { + joinStaticConsumerGroupWithOldProtocol("grp-2", "group-instance-id") + } + + // Request with empty group id. + leaveGroupWithOldProtocol( + groupId = "", + memberIds = List(memberId1), + expectedLeaveGroupError = Errors.INVALID_GROUP_ID, + expectedMemberErrors = List(Errors.NONE), + version = version.toShort + ) + + // Request with invalid group id and unknown member id should still get Errors.INVALID_GROUP_ID. + leaveGroupWithOldProtocol( + groupId = "", + memberIds = List("member-id-unknown"), + expectedLeaveGroupError = Errors.INVALID_GROUP_ID, + expectedMemberErrors = List(Errors.NONE), + version = version.toShort + ) + + // Request with unknown group id gets Errors.UNKNOWN_MEMBER_ID. + leaveGroupWithOldProtocol( + groupId = "grp-unknown", + memberIds = List(memberId1), + expectedLeaveGroupError = if (version >= 3) Errors.NONE else Errors.UNKNOWN_MEMBER_ID, + expectedMemberErrors = if (version >= 3) List(Errors.UNKNOWN_MEMBER_ID) else List.empty, + version = version.toShort + ) + + // Request with unknown member ids. + leaveGroupWithOldProtocol( + groupId = "grp-1", + memberIds = if (version >= 3) List("unknown-member-id", JoinGroupRequest.UNKNOWN_MEMBER_ID) else List("unknown-member-id"), + expectedLeaveGroupError = if (version >= 3) Errors.NONE else Errors.UNKNOWN_MEMBER_ID, + expectedMemberErrors = if (version >= 3) List(Errors.UNKNOWN_MEMBER_ID, Errors.UNKNOWN_MEMBER_ID) else List.empty, + version = version.toShort + ) + + // Success GroupLeave request. + leaveGroupWithOldProtocol( + groupId = "grp-1", + memberIds = List(memberId1), + expectedLeaveGroupError = Errors.NONE, + expectedMemberErrors = if (version >= 3) List(Errors.NONE) else List.empty, + version = version.toShort + ) + + // grp-1 is empty. + assertEquals( + GenericGroupState.EMPTY.toString, + describeGroups(List("grp-1")).head.groupState + ) + + if (version >= 3) { + // Request with fenced group instance id. + leaveGroupWithOldProtocol( + groupId = "grp-2", + memberIds = List("member-id-fenced"), + groupInstanceIds = List("group-instance-id"), + expectedLeaveGroupError = Errors.NONE, + expectedMemberErrors = List(Errors.FENCED_INSTANCE_ID), + version = version.toShort + ) + + // Having unknown member id will not affect the request processing. + leaveGroupWithOldProtocol( + groupId = "grp-2", + memberIds = List(JoinGroupRequest.UNKNOWN_MEMBER_ID), + groupInstanceIds = List("group-instance-id"), + expectedLeaveGroupError = Errors.NONE, + expectedMemberErrors = List(Errors.NONE), + version = version.toShort + ) + + // grp-2 is empty. + assertEquals( + GenericGroupState.EMPTY.toString, + describeGroups(List("grp-2")).head.groupState + ) + } + } + } +} diff --git a/core/src/test/scala/unit/kafka/server/ListGroupsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ListGroupsRequestTest.scala index ba9faf5b0f495..eff202740511c 100644 --- a/core/src/test/scala/unit/kafka/server/ListGroupsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ListGroupsRequestTest.scala @@ -33,7 +33,6 @@ import org.junit.jupiter.api.extension.ExtendWith @Tag("integration") class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -45,7 +44,6 @@ class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBa } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -55,7 +53,6 @@ class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBa } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -81,22 +78,22 @@ class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBa for (version <- ApiKeys.LIST_GROUPS.oldestVersion() to ApiKeys.LIST_GROUPS.latestVersion(isUnstableApiEnabled)) { // Create grp-1 in old protocol and complete a rebalance. Grp-1 is in STABLE state. - val (memberId1InGroup1, _) = joinConsumerGroupWithOldProtocol(groupId = "grp-1") + val (memberId1InGroup1, _) = joinDynamicConsumerGroupWithOldProtocol(groupId = "grp-1") val response1 = new ListGroupsResponseData.ListedGroup() .setGroupId("grp-1") .setGroupState(if (version >= 4) GenericGroupState.STABLE.toString else "") .setProtocolType("consumer") // Create grp-2 in old protocol without completing rebalance. Grp-2 is in COMPLETING_REBALANCE state. - val (memberId1InGroup2, _) = joinConsumerGroupWithOldProtocol(groupId = "grp-2", completeRebalance = false) + val (memberId1InGroup2, _) = joinDynamicConsumerGroupWithOldProtocol(groupId = "grp-2", completeRebalance = false) val response2 = new ListGroupsResponseData.ListedGroup() .setGroupId("grp-2") .setGroupState(if (version >= 4) GenericGroupState.COMPLETING_REBALANCE.toString else "") .setProtocolType("consumer") // Create grp-3 in old protocol and complete a rebalance. Then memeber 1 leaves grp-3. Grp-3 is in EMPTY state. - val (memberId1InGroup3, _) = joinConsumerGroupWithOldProtocol(groupId = "grp-3") - leaveGroup(groupId = "grp-3", memberId = memberId1InGroup3, useNewProtocol = false) + val (memberId1InGroup3, _) = joinDynamicConsumerGroupWithOldProtocol(groupId = "grp-3") + leaveGroup(groupId = "grp-3", memberId = memberId1InGroup3, useNewProtocol = false, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) val response3 = new ListGroupsResponseData.ListedGroup() .setGroupId("grp-3") .setGroupState(if (version >= 4) GenericGroupState.EMPTY.toString else "") @@ -128,7 +125,7 @@ class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBa // Create grp-6 in new protocol. Then member 1 leaves grp-6. Grp-6 is in Empty state. memberId1InGroup6 = joinConsumerGroup("grp-6", true)._1 - leaveGroup(groupId = "grp-6", memberId = memberId1InGroup6, useNewProtocol = true) + leaveGroup(groupId = "grp-6", memberId = memberId1InGroup6, useNewProtocol = true, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) response6 = new ListGroupsResponseData.ListedGroup() .setGroupId("grp-6") .setGroupState(if (version >= 4) ConsumerGroupState.EMPTY.toString else "") @@ -188,12 +185,12 @@ class ListGroupsRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBa ) } - leaveGroup(groupId = "grp-1", memberId = memberId1InGroup1, useNewProtocol = false) - leaveGroup(groupId = "grp-2", memberId = memberId1InGroup2, useNewProtocol = false) + leaveGroup(groupId = "grp-1", memberId = memberId1InGroup1, useNewProtocol = false, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) + leaveGroup(groupId = "grp-2", memberId = memberId1InGroup2, useNewProtocol = false, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) if (useNewProtocol) { - leaveGroup(groupId = "grp-4", memberId = memberId1InGroup4, useNewProtocol = true) - leaveGroup(groupId = "grp-5", memberId = memberId1InGroup5, useNewProtocol = true) - leaveGroup(groupId = "grp-5", memberId = memberId2InGroup5, useNewProtocol = true) + leaveGroup(groupId = "grp-4", memberId = memberId1InGroup4, useNewProtocol = true, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) + leaveGroup(groupId = "grp-5", memberId = memberId1InGroup5, useNewProtocol = true, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) + leaveGroup(groupId = "grp-5", memberId = memberId2InGroup5, useNewProtocol = true, ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled)) } deleteGroups( diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index ebcd063bc127d..91d5d601169e2 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -16,25 +16,23 @@ */ package kafka.server -import org.apache.kafka.common.{Node, TopicPartition, Uuid} +import org.apache.kafka.common.{DirectoryId, Node, TopicPartition, Uuid} import java.util import java.util.Arrays.asList import java.util.Collections - import kafka.api.LeaderAndIsr -import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} +import kafka.server.metadata.{KRaftMetadataCache, MetadataSnapshot, ZkMetadataCache} import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState, UpdateMetadataTopicState} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} import org.apache.kafka.common.record.RecordBatch -import org.apache.kafka.common.requests.UpdateMetadataRequest +import org.apache.kafka.common.requests.{AbstractControlRequest, UpdateMetadataRequest} import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.server.common.MetadataVersion - import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource @@ -782,7 +780,7 @@ class MetadataCacheTest { .setSecurityProtocol(securityProtocol.id) .setListener(listenerName.value)).asJava)) val updateMetadataRequest = new UpdateMetadataRequest.Builder(version, controllerId, controllerEpoch, brokerEpoch, - partitionStates.asJava, brokers.asJava, util.Collections.emptyMap(), false).build() + partitionStates.asJava, brokers.asJava, util.Collections.emptyMap(), false, AbstractControlRequest.Type.UNKNOWN).build() MetadataCacheTest.updateCache(cache, updateMetadataRequest) val partitionState = cache.getPartitionInfo(topic, partitionIndex).get @@ -802,4 +800,251 @@ class MetadataCacheTest { assertEquals(offlineReplicas, partitionState.offlineReplicas()) } } + + def setupInitialAndFullMetadata(): ( + Map[String, Uuid], mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]], + Map[String, Uuid], Seq[UpdateMetadataPartitionState] + ) = { + def addTopic( + name: String, + partitions: Int, + topicStates: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]] + ): Unit = { + val partitionMap = mutable.LongMap.empty[UpdateMetadataPartitionState] + for (i <- 0 until partitions) { + partitionMap.put(i, new UpdateMetadataPartitionState() + .setTopicName(name) + .setPartitionIndex(i) + .setControllerEpoch(2) + .setLeader(0) + .setLeaderEpoch(10) + .setIsr(asList(0, 1)) + .setZkVersion(10) + .setReplicas(asList(0, 1, 2))) + } + topicStates.put(name, partitionMap) + } + + val initialTopicStates = mutable.AnyRefMap.empty[String, mutable.LongMap[UpdateMetadataPartitionState]] + addTopic("test-topic-1", 3, initialTopicStates) + addTopic("test-topic-2", 3, initialTopicStates) + + val initialTopicIds = Map( + "test-topic-1" -> Uuid.fromString("IQ2F1tpCRoSbjfq4zBJwpg"), + "test-topic-2" -> Uuid.fromString("4N8_J-q7SdWHPFkos275pQ") + ) + + val newTopicIds = Map( + "different-topic" -> Uuid.fromString("DraFMNOJQOC5maTb1vtZ8Q") + ) + + val newPartitionStates = Seq(new UpdateMetadataPartitionState() + .setTopicName("different-topic") + .setPartitionIndex(0) + .setControllerEpoch(42) + .setLeader(0) + .setLeaderEpoch(10) + .setIsr(asList[Integer](0, 1, 2)) + .setZkVersion(1) + .setReplicas(asList[Integer](0, 1, 2))) + + (initialTopicIds, initialTopicStates, newTopicIds, newPartitionStates) + } + + /** + * Verify that ZkMetadataCache#maybeInjectDeletedPartitionsFromFullMetadataRequest correctly + * generates deleted topic partition state when deleted topics are detected. This does not check + * any of the logic about when this method should be called, only that it does the correct thing + * when called. + */ + @Test + def testMaybeInjectDeletedPartitionsFromFullMetadataRequest(): Unit = { + val (initialTopicIds, initialTopicStates, newTopicIds, _) = setupInitialAndFullMetadata() + + val initialSnapshot = MetadataSnapshot( + partitionStates = initialTopicStates, + topicIds = initialTopicIds, + controllerId = Some(KRaftCachedControllerId(3000)), + aliveBrokers = mutable.LongMap.empty, + aliveNodes = mutable.LongMap.empty) + + def verifyTopicStates( + updateMetadataRequest: UpdateMetadataRequest + )( + verifier: mutable.AnyRefMap[String, mutable.LongMap[UpdateMetadataPartitionState]] => Unit + ): Unit = { + val finalTopicStates = mutable.AnyRefMap.empty[String, mutable.LongMap[UpdateMetadataPartitionState]] + updateMetadataRequest.topicStates().forEach { topicState => + finalTopicStates.put(topicState.topicName(), mutable.LongMap.empty[UpdateMetadataPartitionState]) + topicState.partitionStates().forEach { partitionState => + finalTopicStates(topicState.topicName()).put(partitionState.partitionIndex(), partitionState) + } + } + verifier.apply(finalTopicStates) + } + + // Empty UMR, deletes everything + var updateMetadataRequest = new UpdateMetadataRequest.Builder(8, 1, 42, brokerEpoch, + Seq.empty.asJava, Seq.empty.asJava, Map.empty[String, Uuid].asJava, true, AbstractControlRequest.Type.FULL).build() + assertEquals( + Seq(Uuid.fromString("IQ2F1tpCRoSbjfq4zBJwpg"), Uuid.fromString("4N8_J-q7SdWHPFkos275pQ")), + ZkMetadataCache.maybeInjectDeletedPartitionsFromFullMetadataRequest( + initialSnapshot, 42, updateMetadataRequest.topicStates()) + ) + verifyTopicStates(updateMetadataRequest) { topicStates => + assertEquals(2, topicStates.size) + assertEquals(3, topicStates("test-topic-1").values.toSeq.count(_.leader() == -2)) + assertEquals(3, topicStates("test-topic-2").values.toSeq.count(_.leader() == -2)) + } + + // One different topic, should remove other two + val oneTopicPartitionState = Seq(new UpdateMetadataPartitionState() + .setTopicName("different-topic") + .setPartitionIndex(0) + .setControllerEpoch(42) + .setLeader(0) + .setLeaderEpoch(10) + .setIsr(asList[Integer](0, 1, 2)) + .setZkVersion(1) + .setReplicas(asList[Integer](0, 1, 2))) + updateMetadataRequest = new UpdateMetadataRequest.Builder(8, 1, 42, brokerEpoch, + oneTopicPartitionState.asJava, Seq.empty.asJava, newTopicIds.asJava, true, AbstractControlRequest.Type.FULL).build() + assertEquals( + Seq(Uuid.fromString("IQ2F1tpCRoSbjfq4zBJwpg"), Uuid.fromString("4N8_J-q7SdWHPFkos275pQ")), + ZkMetadataCache.maybeInjectDeletedPartitionsFromFullMetadataRequest( + initialSnapshot, 42, updateMetadataRequest.topicStates()) + ) + verifyTopicStates(updateMetadataRequest) { topicStates => + assertEquals(3, topicStates.size) + assertEquals(3, topicStates("test-topic-1").values.toSeq.count(_.leader() == -2)) + assertEquals(3, topicStates("test-topic-2").values.toSeq.count(_.leader() == -2)) + } + + // Existing two plus one new topic, nothing gets deleted, all topics should be present + val allTopicStates = initialTopicStates.flatMap(_._2.values).toSeq ++ oneTopicPartitionState + val allTopicIds = initialTopicIds ++ newTopicIds + updateMetadataRequest = new UpdateMetadataRequest.Builder(8, 1, 42, brokerEpoch, + allTopicStates.asJava, Seq.empty.asJava, allTopicIds.asJava, true, AbstractControlRequest.Type.FULL).build() + assertEquals( + Seq.empty, + ZkMetadataCache.maybeInjectDeletedPartitionsFromFullMetadataRequest( + initialSnapshot, 42, updateMetadataRequest.topicStates()) + ) + verifyTopicStates(updateMetadataRequest) { topicStates => + assertEquals(3, topicStates.size) + // Ensure these two weren't deleted (leader = -2) + assertEquals(0, topicStates("test-topic-1").values.toSeq.count(_.leader() == -2)) + assertEquals(0, topicStates("test-topic-2").values.toSeq.count(_.leader() == -2)) + } + } + + /** + * Verify the behavior of ZkMetadataCache when handling "Full" UpdateMetadataRequest + */ + @Test + def testHandleFullUpdateMetadataRequestInZkMigration(): Unit = { + val (initialTopicIds, initialTopicStates, newTopicIds, newPartitionStates) = setupInitialAndFullMetadata() + + val updateMetadataRequestBuilder = () => new UpdateMetadataRequest.Builder(8, 1, 42, brokerEpoch, + newPartitionStates.asJava, Seq.empty.asJava, newTopicIds.asJava, true, AbstractControlRequest.Type.FULL).build() + + def verifyMetadataCache( + updateMetadataRequest: UpdateMetadataRequest, + zkMigrationEnabled: Boolean = true + )( + verifier: ZkMetadataCache => Unit + ): Unit = { + val cache = MetadataCache.zkMetadataCache(1, MetadataVersion.latest(), zkMigrationEnabled = zkMigrationEnabled) + cache.updateMetadata(1, new UpdateMetadataRequest.Builder(8, 1, 42, brokerEpoch, + initialTopicStates.flatMap(_._2.values).toList.asJava, Seq.empty.asJava, initialTopicIds.asJava).build()) + cache.updateMetadata(1, updateMetadataRequest) + verifier.apply(cache) + } + + // KRaft=false Type=FULL, migration disabled + var updateMetadataRequest = updateMetadataRequestBuilder.apply() + updateMetadataRequest.data().setIsKRaftController(true) + updateMetadataRequest.data().setType(AbstractControlRequest.Type.FULL.toByte) + verifyMetadataCache(updateMetadataRequest, zkMigrationEnabled = false) { cache => + assertEquals(3, cache.getAllTopics().size) + assertTrue(cache.contains("test-topic-1")) + assertTrue(cache.contains("test-topic-1")) + } + + // KRaft=true Type=FULL + updateMetadataRequest = updateMetadataRequestBuilder.apply() + verifyMetadataCache(updateMetadataRequest) { cache => + assertEquals(1, cache.getAllTopics().size) + assertFalse(cache.contains("test-topic-1")) + assertFalse(cache.contains("test-topic-1")) + } + + // KRaft=false Type=FULL + updateMetadataRequest = updateMetadataRequestBuilder.apply() + updateMetadataRequest.data().setIsKRaftController(false) + verifyMetadataCache(updateMetadataRequest) { cache => + assertEquals(3, cache.getAllTopics().size) + assertTrue(cache.contains("test-topic-1")) + assertTrue(cache.contains("test-topic-1")) + } + + // KRaft=true Type=INCREMENTAL + updateMetadataRequest = updateMetadataRequestBuilder.apply() + updateMetadataRequest.data().setType(AbstractControlRequest.Type.INCREMENTAL.toByte) + verifyMetadataCache(updateMetadataRequest) { cache => + assertEquals(3, cache.getAllTopics().size) + assertTrue(cache.contains("test-topic-1")) + assertTrue(cache.contains("test-topic-1")) + } + + // KRaft=true Type=UNKNOWN + updateMetadataRequest = updateMetadataRequestBuilder.apply() + updateMetadataRequest.data().setType(AbstractControlRequest.Type.UNKNOWN.toByte) + verifyMetadataCache(updateMetadataRequest) { cache => + assertEquals(3, cache.getAllTopics().size) + assertTrue(cache.contains("test-topic-1")) + assertTrue(cache.contains("test-topic-1")) + } + } + + @Test + def testGetOfflineReplicasConsidersDirAssignment(): Unit = { + case class Broker(id: Int, dirs: util.List[Uuid]) + case class Partition(id: Int, replicas: util.List[Integer], dirs: util.List[Uuid]) + + def offlinePartitions(brokers: Seq[Broker], partitions: Seq[Partition]): Map[Int, util.List[Integer]] = { + val delta = new MetadataDelta.Builder().build() + brokers.foreach(broker => delta.replay( + new RegisterBrokerRecord().setFenced(false). + setBrokerId(broker.id).setLogDirs(broker.dirs). + setEndPoints(new BrokerEndpointCollection(Collections.singleton( + new RegisterBrokerRecord.BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). + setPort(9093.toShort).setName("PLAINTEXT").setHost(s"broker-${broker.id}")).iterator())))) + val topicId = Uuid.fromString("95OVr1IPRYGrcNCLlpImCA") + delta.replay(new TopicRecord().setTopicId(topicId).setName("foo")) + partitions.foreach(partition => delta.replay( + new PartitionRecord().setTopicId(topicId).setPartitionId(partition.id). + setReplicas(partition.replicas).setDirectories(partition.dirs). + setLeader(partition.replicas.get(0)).setIsr(partition.replicas))) + val cache = MetadataCache.kRaftMetadataCache(1) + cache.setImage(delta.apply(MetadataProvenance.EMPTY)) + val topicMetadata = cache.getTopicMetadata(Set("foo"), ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)).head + topicMetadata.partitions().asScala.map(p => (p.partitionIndex(), p.offlineReplicas())).toMap + } + + val brokers = Seq( + Broker(0, asList(Uuid.fromString("broker1logdirjEo71BG0w"))), + Broker(1, asList(Uuid.fromString("broker2logdirRmQQgLxgw"))) + ) + val partitions = Seq( + Partition(0, asList(0, 1), asList(Uuid.fromString("broker1logdirjEo71BG0w"), DirectoryId.LOST)), + Partition(1, asList(0, 1), asList(Uuid.fromString("unknownlogdirjEo71BG0w"), DirectoryId.UNASSIGNED)), + Partition(2, asList(0, 1), asList(DirectoryId.MIGRATING, Uuid.fromString("broker2logdirRmQQgLxgw"))) + ) + assertEquals(Map( + 0 -> asList(1), + 1 -> asList(0), + 2 -> asList(), + ), offlinePartitions(brokers, partitions)) + } } diff --git a/core/src/test/scala/unit/kafka/server/MockFetcherThread.scala b/core/src/test/scala/unit/kafka/server/MockFetcherThread.scala index 94908caef5dd0..49efa0a49ba45 100644 --- a/core/src/test/scala/unit/kafka/server/MockFetcherThread.scala +++ b/core/src/test/scala/unit/kafka/server/MockFetcherThread.scala @@ -109,7 +109,7 @@ class MockFetcherThread(val mockLeader: MockLeaderEndPoint, offsetOfMaxTimestamp, Time.SYSTEM.milliseconds(), state.logStartOffset, - RecordConversionStats.EMPTY, + RecordValidationStats.EMPTY, CompressionType.NONE, FetchResponse.recordsSize(partitionData), batches.headOption.map(_.lastOffset).getOrElse(-1))) diff --git a/core/src/test/scala/unit/kafka/server/MockNodeToControllerChannelManager.scala b/core/src/test/scala/unit/kafka/server/MockNodeToControllerChannelManager.scala index 6c10ba89db7e7..c3265d6be7f3d 100644 --- a/core/src/test/scala/unit/kafka/server/MockNodeToControllerChannelManager.scala +++ b/core/src/test/scala/unit/kafka/server/MockNodeToControllerChannelManager.scala @@ -19,8 +19,11 @@ package kafka.server import org.apache.kafka.clients.{ClientResponse, MockClient, NodeApiVersions} import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.AbstractRequest +import org.apache.kafka.server.{ControllerRequestCompletionHandler, NodeToControllerChannelManager} import org.apache.kafka.server.util.MockTime +import java.util.Optional + class MockNodeToControllerChannelManager( val client: MockClient, time: MockTime, @@ -29,7 +32,7 @@ class MockNodeToControllerChannelManager( val retryTimeoutMs: Int = 60000, val requestTimeoutMs: Int = 30000 ) extends NodeToControllerChannelManager { - val unsentQueue = new java.util.ArrayDeque[NodeToControllerQueueItem]() + val unsentQueue = new java.util.concurrent.ConcurrentLinkedDeque[NodeToControllerQueueItem]() client.setNodeApiVersions(controllerApiVersions) @@ -48,8 +51,8 @@ class MockNodeToControllerChannelManager( )) } - override def controllerApiVersions(): Option[NodeApiVersions] = { - Some(controllerApiVersions) + override def controllerApiVersions(): Optional[NodeApiVersions] = { + Optional.of(controllerApiVersions) } private[server] def handleResponse(request: NodeToControllerQueueItem)(response: ClientResponse): Unit = { diff --git a/core/src/test/scala/unit/kafka/server/OffsetCommitRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetCommitRequestTest.scala index e428ca6617d86..2ff776aa42a4f 100644 --- a/core/src/test/scala/unit/kafka/server/OffsetCommitRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/OffsetCommitRequestTest.scala @@ -31,7 +31,6 @@ import org.junit.jupiter.api.extension.ExtendWith class OffsetCommitRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -43,7 +42,6 @@ class OffsetCommitRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -53,7 +51,6 @@ class OffsetCommitRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") diff --git a/core/src/test/scala/unit/kafka/server/OffsetDeleteRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetDeleteRequestTest.scala index 38a0973fb73b8..719697da1314f 100644 --- a/core/src/test/scala/unit/kafka/server/OffsetDeleteRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/OffsetDeleteRequestTest.scala @@ -30,7 +30,6 @@ import org.junit.jupiter.api.extension.ExtendWith @Tag("integration") class OffsetDeleteRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -42,7 +41,6 @@ class OffsetDeleteRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -52,7 +50,6 @@ class OffsetDeleteRequestTest(cluster: ClusterInstance) extends GroupCoordinator } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") @@ -119,7 +116,8 @@ class OffsetDeleteRequestTest(cluster: ClusterInstance) extends GroupCoordinator leaveGroup( groupId = "grp", memberId = memberId, - useNewProtocol = false + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) ) } diff --git a/core/src/test/scala/unit/kafka/server/OffsetFetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/OffsetFetchRequestTest.scala index c354656856b3d..828f8ab0aaf6f 100644 --- a/core/src/test/scala/unit/kafka/server/OffsetFetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/OffsetFetchRequestTest.scala @@ -39,7 +39,6 @@ import scala.jdk.CollectionConverters._ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -51,7 +50,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -63,7 +61,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -75,7 +72,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -87,7 +83,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -99,7 +94,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -111,7 +105,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -123,7 +116,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "true"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), @@ -135,7 +127,6 @@ class OffsetFetchRequestTest(cluster: ClusterInstance) extends GroupCoordinatorB } @ClusterTest(clusterType = Type.ALL, serverProperties = Array( - new ClusterConfigProperty(key = "unstable.api.versions.enable", value = "false"), new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), new ClusterConfigProperty(key = "group.consumer.max.session.timeout.ms", value = "600000"), new ClusterConfigProperty(key = "group.consumer.session.timeout.ms", value = "600000"), diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala index 7258999cd29fc..053583fd889f3 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala @@ -30,13 +30,13 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetFor import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset import org.apache.kafka.common.protocol.Errors._ import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, RecordConversionStats, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, RecordValidationStats, SimpleRecord} import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, UpdateMetadataRequest} import org.apache.kafka.common.utils.{LogContext, SystemTime} import org.apache.kafka.server.common.{MetadataVersion, OffsetAndEpoch} import org.apache.kafka.server.common.MetadataVersion.IBP_2_6_IV0 -import org.apache.kafka.storage.internals.log.{LogAppendInfo} +import org.apache.kafka.storage.internals.log.LogAppendInfo import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} import org.junit.jupiter.params.ParameterizedTest @@ -772,7 +772,7 @@ class ReplicaFetcherThreadTest { -1L, RecordBatch.NO_TIMESTAMP, -1L, - RecordConversionStats.EMPTY, + RecordValidationStats.EMPTY, CompressionType.NONE, -1, // No records. -1L diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala index 00d7d61f5e443..9813c0b73d9dd 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala @@ -34,17 +34,19 @@ import org.apache.kafka.common.replica.ClientMetadata.DefaultClientMetadata import org.apache.kafka.common.requests.{FetchRequest, ProduceResponse} import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.{IsolationLevel, TopicIdPartition, TopicPartition, Uuid} +import org.apache.kafka.common.{DirectoryId, IsolationLevel, TopicIdPartition, TopicPartition, Uuid} import org.apache.kafka.image.{MetadataDelta, MetadataImage} import org.apache.kafka.metadata.LeaderRecoveryState import org.apache.kafka.metadata.PartitionRegistration +import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesVersion} +import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.util.{MockTime, ShutdownableThread} import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchIsolation, FetchParams, FetchPartitionData, LogConfig, LogDirFailureChannel} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} import org.mockito.Mockito -import scala.collection.mutable +import scala.collection.{immutable, mutable} import scala.jdk.CollectionConverters._ import scala.util.Random @@ -147,6 +149,12 @@ class ReplicaManagerConcurrencyTest { metadataCache: MetadataCache, ): ReplicaManager = { val logDir = TestUtils.tempDir() + val metaProperties = new MetaProperties.Builder(). + setVersion(MetaPropertiesVersion.V1). + setClusterId(Uuid.randomUuid().toString). + setNodeId(1). + build() + TestUtils.formatDirectories(immutable.Seq(logDir.getAbsolutePath), metaProperties, MetadataVersion.latest(), None) val props = new Properties props.put(KafkaConfig.QuorumVotersProp, "100@localhost:12345") @@ -466,6 +474,7 @@ class ReplicaManagerConcurrencyTest { ): PartitionRegistration = { new PartitionRegistration.Builder(). setReplicas(replicaIds.toArray). + setDirectories(DirectoryId.unassignedArray(replicaIds.size)). setIsr(isr.toArray). setLeader(leader). setLeaderRecoveryState(leaderRecoveryState). diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index ec61da1e9c70f..a4ececf219752 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -564,7 +564,7 @@ class ReplicaManagerTest { Collections.singletonMap(topic, Uuid.randomUuid()), Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava, false, - LeaderAndIsrRequest.Type.UNKNOWN + AbstractControlRequest.Type.UNKNOWN ).build() replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ()) replicaManager.getPartitionOrException(new TopicPartition(topic, partition)) @@ -2434,7 +2434,7 @@ class ReplicaManagerTest { verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) // Dynamically enable verification. - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val props = new Properties() props.put(KafkaConfig.TransactionPartitionVerificationEnableProp, "true") config.dynamicConfig.updateBrokerConfig(config.brokerId, props) @@ -2485,7 +2485,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // Disable verification - config.dynamicConfig.initialize(None) + config.dynamicConfig.initialize(None, None) val props = new Properties() props.put(KafkaConfig.TransactionPartitionVerificationEnableProp, "false") config.dynamicConfig.updateBrokerConfig(config.brokerId, props) @@ -2605,7 +2605,7 @@ class ReplicaManagerTest { topicIds.asJava, Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava, true, - LeaderAndIsrRequest.Type.FULL + AbstractControlRequest.Type.FULL ).build() replicaManager.becomeLeaderOrFollower(0, lisr, (_, _) => ()) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index a817881b17a7c..255ada6012bbe 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -722,6 +722,9 @@ class RequestQuotaTest extends BaseRequestTest { case ApiKeys.ASSIGN_REPLICAS_TO_DIRS => new AssignReplicasToDirsRequest.Builder(new AssignReplicasToDirsRequestData()) + case ApiKeys.LIST_CLIENT_METRICS_RESOURCES => + new ListClientMetricsResourcesRequest.Builder(new ListClientMetricsResourcesRequestData()) + case _ => throw new IllegalArgumentException("Unsupported API key " + apiKey) } diff --git a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala index dc3a346a1f9e7..bdf3d0afc9c96 100755 --- a/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala +++ b/core/src/test/scala/unit/kafka/server/ServerStartupTest.scala @@ -24,6 +24,8 @@ import org.apache.kafka.server.log.remote.storage.{NoOpRemoteLogMetadataManager, import org.apache.zookeeper.KeeperException.NodeExistsException import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource class ServerStartupTest extends QuorumTestHarness { @@ -132,4 +134,23 @@ class ServerStartupTest extends QuorumTestHarness { assertEquals(1, brokers.size) assertEquals(brokerId, brokers.head.id) } + + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testDirectoryIdsCreatedOnlyForMigration(migrationEnabled: Boolean): Unit = { + val props = TestUtils.createBrokerConfig(1, zkConnect) + props.setProperty(KafkaConfig.MigrationEnabledProp, migrationEnabled.toString) + if (migrationEnabled) { + // Create Controller properties needed when migration is enabled + props.setProperty(KafkaConfig.QuorumVotersProp, "3000@localhost:9093") + props.setProperty(KafkaConfig.ControllerListenerNamesProp, "CONTROLLER") + props.setProperty(KafkaConfig.ListenerSecurityProtocolMapProp, + "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT") + } + server = new KafkaServer(KafkaConfig.fromProps(props)) + server.startup() + assertEquals(!migrationEnabled, server.logManager.directoryIds.isEmpty) + server.shutdown() + } + } diff --git a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala index d509a51145662..aa9aadc22a7fd 100644 --- a/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/StopReplicaRequestTest.scala @@ -46,7 +46,7 @@ class StopReplicaRequestTest extends BaseRequestTest { val server = servers.head val offlineDir = server.logManager.getLog(tp1).get.dir.getParent - server.replicaManager.handleLogDirFailure(offlineDir, sendZkNotification = false) + server.replicaManager.handleLogDirFailure(offlineDir, notifyController = false) val topicStates = Seq( new StopReplicaTopicState() diff --git a/core/src/test/scala/unit/kafka/server/SyncGroupRequestTest.scala b/core/src/test/scala/unit/kafka/server/SyncGroupRequestTest.scala new file mode 100644 index 0000000000000..5d13ce5a20361 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/SyncGroupRequestTest.scala @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.test.ClusterInstance +import kafka.test.annotation.{ClusterConfigProperty, ClusterTest, ClusterTestDefaults, Type} +import kafka.test.junit.ClusterTestExtensions +import kafka.utils.TestUtils +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol +import org.apache.kafka.common.message.SyncGroupRequestData +import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.coordinator.group.generic.GenericGroupState +import org.junit.jupiter.api.{Tag, Timeout} +import org.junit.jupiter.api.extension.ExtendWith + +import java.util.Collections +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} + +@Timeout(120) +@ExtendWith(value = Array(classOf[ClusterTestExtensions])) +@ClusterTestDefaults(clusterType = Type.KRAFT, brokers = 1) +@Tag("integration") +class SyncGroupRequestTest(cluster: ClusterInstance) extends GroupCoordinatorBaseRequestTest(cluster) { + @ClusterTest(serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "true"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testSyncGroupWithOldConsumerGroupProtocolAndNewGroupCoordinator(): Unit = { + testSyncGroup() + } + + @ClusterTest(clusterType = Type.ALL, serverProperties = Array( + new ClusterConfigProperty(key = "group.coordinator.new.enable", value = "false"), + new ClusterConfigProperty(key = "offsets.topic.num.partitions", value = "1"), + new ClusterConfigProperty(key = "offsets.topic.replication.factor", value = "1") + )) + def testSyncGroupWithOldConsumerGroupProtocolAndOldGroupCoordinator(): Unit = { + testSyncGroup() + } + + private def testSyncGroup(): Unit = { + // Creates the __consumer_offsets topics because it won't be created automatically + // in this test because it does not use FindCoordinator API. + createOffsetsTopic() + + // Create the topic. + createTopic( + topic = "foo", + numPartitions = 3 + ) + + for (version <- ApiKeys.SYNC_GROUP.oldestVersion() to ApiKeys.SYNC_GROUP.latestVersion(isUnstableApiEnabled)) { + // Sync with unknown group id. + syncGroupWithOldProtocol( + groupId = "grp-unknown", + memberId = "member-id", + generationId = -1, + expectedProtocolType = null, + expectedProtocolName = null, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + + val metadata = ConsumerProtocol.serializeSubscription( + new ConsumerPartitionAssignor.Subscription(Collections.singletonList("foo")) + ).array + + // Join a dynamic member without member id. + // Prior to JoinGroup version 4, a new member is immediately added if it sends a join group request with UNKNOWN_MEMBER_ID. + val joinLeaderResponseData = sendJoinRequest( + groupId = "grp", + metadata = metadata + ) + val leaderMemberId = joinLeaderResponseData.memberId + + // Rejoin the group with the member id. + sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata + ) + + if (version >= 5) { + // Sync the leader with unmatched protocolName. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 1, + protocolName = "unmatched", + assignments = List(new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)) + ), + expectedProtocolType = null, + expectedProtocolName = null, + expectedError = Errors.INCONSISTENT_GROUP_PROTOCOL, + version = version.toShort + ) + + // Sync the leader with unmatched protocolType. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 1, + protocolType = "unmatched", + assignments = List(new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)) + ), + expectedProtocolType = null, + expectedProtocolName = null, + expectedError = Errors.INCONSISTENT_GROUP_PROTOCOL, + version = version.toShort + ) + } + + // Sync with unknown member id. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = "member-id-unknown", + generationId = -1, + expectedProtocolType = null, + expectedProtocolName = null, + expectedError = Errors.UNKNOWN_MEMBER_ID, + version = version.toShort + ) + + // Sync with illegal generation id. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 2, + expectedProtocolType = null, + expectedProtocolName = null, + expectedError = Errors.ILLEGAL_GENERATION, + version = version.toShort + ) + + // Sync the leader with empty protocolType and protocolName if version < 5. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 1, + protocolType = if (version < 5) null else "consumer", + protocolName = if (version < 5) null else "consumer-range", + assignments = List(new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)) + ), + expectedProtocolType = if (version < 5) null else "consumer", + expectedProtocolName = if (version < 5) null else "consumer-range", + expectedAssignment = Array[Byte](1), + version = version.toShort + ) + + // Join a second member. + val joinFollowerFuture = Future { + sendJoinRequest( + groupId = "grp", + groupInstanceId = "group-instance-id", + metadata = metadata + ) + } + + TestUtils.waitUntilTrue(() => { + val described = describeGroups(groupIds = List("grp")) + GenericGroupState.PREPARING_REBALANCE.toString == described.head.groupState + }, msg = s"The group is not in PREPARING_REBALANCE state.") + + // The leader rejoins. + val rejoinLeaderResponseData = sendJoinRequest( + groupId = "grp", + memberId = leaderMemberId, + metadata = metadata + ) + + val joinFollowerFutureResponseData = Await.result(joinFollowerFuture, Duration.Inf) + val followerMemberId = joinFollowerFutureResponseData.memberId + + // Sync the leader ahead of the follower. + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = rejoinLeaderResponseData.generationId, + assignments = List( + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)), + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(followerMemberId) + .setAssignment(Array[Byte](2)) + ), + expectedAssignment = Array[Byte](1), + version = version.toShort + ) + + syncGroupWithOldProtocol( + groupId = "grp", + memberId = followerMemberId, + generationId = joinFollowerFutureResponseData.generationId, + expectedAssignment = Array[Byte](2), + version = version.toShort + ) + + // Sync the follower ahead of the leader. + val syncFollowerFuture = Future { + syncGroupWithOldProtocol( + groupId = "grp", + memberId = followerMemberId, + generationId = 2, + expectedAssignment = Array[Byte](2), + version = version.toShort + ) + } + + syncGroupWithOldProtocol( + groupId = "grp", + memberId = leaderMemberId, + generationId = 2, + assignments = List( + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(leaderMemberId) + .setAssignment(Array[Byte](1)), + new SyncGroupRequestData.SyncGroupRequestAssignment() + .setMemberId(followerMemberId) + .setAssignment(Array[Byte](2)) + ), + expectedAssignment = Array[Byte](1), + version = version.toShort + ) + + Await.result(syncFollowerFuture, Duration.Inf) + + leaveGroup( + groupId = "grp", + memberId = leaderMemberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + leaveGroup( + groupId = "grp", + memberId = followerMemberId, + useNewProtocol = false, + version = ApiKeys.LEAVE_GROUP.latestVersion(isUnstableApiEnabled) + ) + + deleteGroups( + groupIds = List("grp"), + expectedErrors = List(Errors.NONE), + version = ApiKeys.DELETE_GROUPS.latestVersion(isUnstableApiEnabled) + ) + } + } +} diff --git a/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileWithFailureHandlerTest.scala b/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileWithFailureHandlerTest.scala index ddbf58d884e30..a7e370d7f4091 100644 --- a/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileWithFailureHandlerTest.scala +++ b/core/src/test/scala/unit/kafka/server/checkpoints/OffsetCheckpointFileWithFailureHandlerTest.scala @@ -97,7 +97,7 @@ class OffsetCheckpointFileWithFailureHandlerTest extends Logging { val logDirFailureChannel = new LogDirFailureChannel(10) val checkpointFile = new CheckpointFileWithFailureHandler(file, OffsetCheckpointFile.CurrentVersion + 1, OffsetCheckpointFile.Formatter, logDirFailureChannel, file.getParent) - checkpointFile.write(Collections.singletonList(new TopicPartition("foo", 5) -> 10L)) + checkpointFile.write(Collections.singletonList(new TopicPartition("foo", 5) -> 10L), true) assertThrows(classOf[KafkaStorageException], () => new OffsetCheckpointFile(checkpointFile.file, logDirFailureChannel).read()) } diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala index 3523142f8871a..a47c902024a9f 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochFileCacheTest.scala @@ -27,7 +27,7 @@ import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import java.io.File -import java.util.{Collections, OptionalInt} +import java.util.{Collections, OptionalInt, Optional} import scala.collection.Seq import scala.jdk.CollectionConverters._ @@ -38,7 +38,7 @@ class LeaderEpochFileCacheTest { val tp = new TopicPartition("TestTopic", 5) private val checkpoint: LeaderEpochCheckpoint = new LeaderEpochCheckpoint { private var epochs: Seq[EpochEntry] = Seq() - override def write(epochs: java.util.Collection[EpochEntry]): Unit = this.epochs = epochs.asScala.toSeq + override def write(epochs: java.util.Collection[EpochEntry], ignored: Boolean): Unit = this.epochs = epochs.asScala.toSeq override def read(): java.util.List[EpochEntry] = this.epochs.asJava } @@ -604,6 +604,23 @@ class LeaderEpochFileCacheTest { assertEquals(OptionalInt.of(2), cache.previousEpoch(cache.latestEpoch.getAsInt)) } + @Test + def testFindPreviousEntry(): Unit = { + assertEquals(Optional.empty(), cache.previousEntry(2)) + + cache.assign(2, 10) + assertEquals(Optional.empty(), cache.previousEntry(2)) + + cache.assign(4, 15) + assertEquals(Optional.of(new EpochEntry(2, 10)), cache.previousEntry(4)) + + cache.assign(10, 20) + assertEquals(Optional.of(new EpochEntry(4, 15)), cache.previousEntry(10)) + + cache.truncateFromEnd(18) + assertEquals(Optional.of(new EpochEntry(2, 10)), cache.previousEntry(cache.latestEpoch.getAsInt)) + } + @Test def testFindNextEpoch(): Unit = { cache.assign(0, 0) diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala index 136fb87e89447..f80e260197952 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala @@ -31,7 +31,7 @@ import org.apache.kafka.clients.admin.{Admin, AlterConfigOp, ConfigEntry, NewTop import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.ConfigResource.Type.BROKER import org.apache.kafka.common.utils.Exit -import org.apache.kafka.common.{TopicPartition, Uuid} +import org.apache.kafka.common.{DirectoryId, TopicPartition, Uuid} import org.apache.kafka.coordinator.group.GroupCoordinator import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataImageTest, MetadataProvenance, TopicImage, TopicsImage} import org.apache.kafka.image.loader.LogDeltaManifest @@ -162,6 +162,7 @@ class BrokerMetadataPublisherTest { val partitionRegistrations = partitions.map { case (partitionId, replicas) => Int.box(partitionId) -> new PartitionRegistration.Builder(). setReplicas(replicas.toArray). + setDirectories(DirectoryId.unassignedArray(replicas.size)). setIsr(replicas.toArray). setLeader(replicas.head). setLeaderRecoveryState(LeaderRecoveryState.RECOVERED). diff --git a/core/src/test/scala/unit/kafka/server/metadata/ZkConfigRepositoryTest.scala b/core/src/test/scala/unit/kafka/server/metadata/ZkConfigRepositoryTest.scala index f8737751fa590..fbb2c9e8cf92e 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/ZkConfigRepositoryTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/ZkConfigRepositoryTest.scala @@ -22,6 +22,7 @@ import kafka.server.metadata.ZkConfigRepository import kafka.zk.KafkaZkClient import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.ConfigResource.Type +import org.apache.kafka.common.errors.InvalidRequestException import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows} import org.junit.jupiter.api.Test import org.mockito.Mockito.{mock, when} @@ -48,7 +49,9 @@ class ZkConfigRepositoryTest { def testUnsupportedTypes(): Unit = { val zkClient: KafkaZkClient = mock(classOf[KafkaZkClient]) val zkConfigRepository = ZkConfigRepository(zkClient) - Type.values().foreach(value => if (value != Type.BROKER && value != Type.TOPIC) + Type.values().foreach(value => if (value != Type.BROKER && value != Type.TOPIC && value != Type.CLIENT_METRICS) assertThrows(classOf[IllegalArgumentException], () => zkConfigRepository.config(new ConfigResource(value, value.toString)))) + // Validate exception for CLIENT_METRICS. + assertThrows(classOf[InvalidRequestException], () => zkConfigRepository.config(new ConfigResource(Type.CLIENT_METRICS, Type.CLIENT_METRICS.toString))) } } diff --git a/core/src/test/scala/unit/kafka/tools/StorageToolTest.scala b/core/src/test/scala/unit/kafka/tools/StorageToolTest.scala index 5a4cca14c58e4..db4482e8ddb5f 100644 --- a/core/src/test/scala/unit/kafka/tools/StorageToolTest.scala +++ b/core/src/test/scala/unit/kafka/tools/StorageToolTest.scala @@ -33,6 +33,8 @@ import org.apache.kafka.common.metadata.UserScramCredentialRecord import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesEnsemble, MetaPropertiesVersion, PropertiesUtils} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue} import org.junit.jupiter.api.{Test, Timeout} +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -203,8 +205,8 @@ Found problem: def testDefaultMetadataVersion(): Unit = { val namespace = StorageTool.parseArguments(Array("format", "-c", "config.props", "-t", "XcZZOzUqS4yHOjhMQB6JLQ")) val mv = StorageTool.getMetadataVersion(namespace, defaultVersionString = None) - assertEquals(MetadataVersion.latest().featureLevel(), mv.featureLevel(), - "Expected the default metadata.version to be the latest version") + assertEquals(MetadataVersion.LATEST_PRODUCTION.featureLevel(), mv.featureLevel(), + "Expected the default metadata.version to be the latest production version") } @Test @@ -390,4 +392,45 @@ Found problem: assertFalse(DirectoryId.reserved(metaProps.directoryId().get())) } finally Utils.delete(tempDir) } + + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testFormattingUnstableMetadataVersionBlocked(enableUnstable: Boolean): Unit = { + var exitString: String = "" + var exitStatus: Int = 1 + def exitProcedure(status: Int, message: Option[String]) : Nothing = { + exitStatus = status + exitString = message.getOrElse("") + throw new StorageToolTestException(exitString) + } + Exit.setExitProcedure(exitProcedure) + val properties = newSelfManagedProperties() + val propsFile = TestUtils.tempFile() + val propsStream = Files.newOutputStream(propsFile.toPath) + try { + properties.setProperty(KafkaConfig.LogDirsProp, TestUtils.tempDir().toString) + properties.setProperty(KafkaConfig.UnstableMetadataVersionsEnableProp, enableUnstable.toString) + properties.store(propsStream, "config.props") + } finally { + propsStream.close() + } + val args = Array("format", "-c", s"${propsFile.toPath}", + "-t", "XcZZOzUqS4yHOjhMQB6JLQ", + "--release-version", MetadataVersion.latest().toString) + try { + StorageTool.main(args) + } catch { + case _: StorageToolTestException => + } finally { + Exit.resetExitProcedure() + } + if (enableUnstable) { + assertEquals("", exitString) + assertEquals(0, exitStatus) + } else { + assertEquals(s"Metadata version ${MetadataVersion.latest().toString} is not ready for " + + "production use yet.", exitString) + assertEquals(1, exitStatus) + } + } } diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index be859f8a17a84..913298b719ab0 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -25,7 +25,7 @@ import java.nio.file.{Files, StandardOpenOption} import java.security.cert.X509Certificate import java.time.Duration import java.util -import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReference} import java.util.concurrent.{Callable, CompletableFuture, ExecutionException, Executors, TimeUnit} import java.util.{Arrays, Collections, Optional, Properties} import com.yammer.metrics.core.{Gauge, Histogram, Meter} @@ -39,6 +39,7 @@ import kafka.network.RequestChannel import kafka.server._ import kafka.server.checkpoints.OffsetCheckpointFile import kafka.server.metadata.{ConfigRepository, MockConfigRepository} +import kafka.tools.StorageTool import kafka.utils.Implicits._ import kafka.zk._ import org.apache.kafka.admin.BrokerMetadata @@ -69,8 +70,10 @@ import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySe import org.apache.kafka.common.utils.Utils._ import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.controller.QuorumController +import org.apache.kafka.metadata.properties.MetaProperties +import org.apache.kafka.server.ControllerRequestCompletionHandler import org.apache.kafka.server.authorizer.{AuthorizableRequestContext, Authorizer => JAuthorizer} -import org.apache.kafka.server.common.MetadataVersion +import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.util.MockTime import org.apache.kafka.storage.internals.log.{CleanerConfig, LogConfig, LogDirFailureChannel, ProducerStateManagerConfig} @@ -84,7 +87,7 @@ import org.mockito.Mockito import scala.annotation.nowarn import scala.collection.mutable.{ArrayBuffer, ListBuffer} -import scala.collection.{Map, Seq, mutable} +import scala.collection.{Map, Seq, immutable, mutable} import scala.concurrent.duration.FiniteDuration import scala.concurrent.{Await, ExecutionContext, Future} import scala.jdk.CollectionConverters._ @@ -328,6 +331,7 @@ object TestUtils extends Logging { }.mkString(",") val props = new Properties + props.put(KafkaConfig.UnstableMetadataVersionsEnableProp, "true") if (zkConnect == null) { props.setProperty(KafkaConfig.ServerMaxStartupTimeMsProp, TimeUnit.MINUTES.toMillis(10).toString) props.put(KafkaConfig.NodeIdProp, nodeId.toString) @@ -593,10 +597,10 @@ object TestUtils extends Logging { * Wait until the leader is elected and the metadata is propagated to all brokers. * Return the leader for each partition. */ - def createTopic(zkClient: KafkaZkClient, + def createTopic[B <: KafkaBroker](zkClient: KafkaZkClient, topic: String, partitionReplicaAssignment: collection.Map[Int, Seq[Int]], - servers: Seq[KafkaBroker]): scala.collection.immutable.Map[Int, Int] = { + servers: Seq[B]): scala.collection.immutable.Map[Int, Int] = { createTopic(zkClient, topic, partitionReplicaAssignment, servers, new Properties()) } @@ -1410,6 +1414,27 @@ object TestUtils extends Logging { }.mkString("\n") } + def formatDirectories( + directories: immutable.Seq[String], + metaProperties: MetaProperties, + metadataVersion: MetadataVersion, + optionalMetadataRecords: Option[ArrayBuffer[ApiMessageAndVersion]] + ): Unit = { + val stream = new ByteArrayOutputStream() + var out: PrintStream = null + try { + out = new PrintStream(stream) + val bootstrapMetadata = StorageTool.buildBootstrapMetadata(metadataVersion, optionalMetadataRecords, "format command") + if (StorageTool.formatCommand(out, directories, metaProperties, bootstrapMetadata, metadataVersion, ignoreFormatted = false) != 0) { + throw new RuntimeException(stream.toString()) + } + debug(s"Formatted storage directory(ies) ${directories}") + } finally { + if (out != null) out.close() + stream.close() + } + } + /** * Create new LogManager instance with default configuration for testing */ @@ -1504,6 +1529,7 @@ object TestUtils extends Logging { val expands: AtomicInteger = new AtomicInteger(0) val shrinks: AtomicInteger = new AtomicInteger(0) val failures: AtomicInteger = new AtomicInteger(0) + val directory: AtomicReference[String] = new AtomicReference[String]() override def markIsrExpand(): Unit = expands.incrementAndGet() @@ -1511,10 +1537,13 @@ object TestUtils extends Logging { override def markFailed(): Unit = failures.incrementAndGet() + override def assignDir(dir: String): Unit = directory.set(dir) + def reset(): Unit = { expands.set(0) shrinks.set(0) failures.set(0) + directory.set(null) } } diff --git a/core/src/test/scala/unit/kafka/zk/migration/ZkMigrationClientTest.scala b/core/src/test/scala/unit/kafka/zk/migration/ZkMigrationClientTest.scala index 773c42a66e4a6..93b29c701c4b3 100644 --- a/core/src/test/scala/unit/kafka/zk/migration/ZkMigrationClientTest.scala +++ b/core/src/test/scala/unit/kafka/zk/migration/ZkMigrationClientTest.scala @@ -23,7 +23,7 @@ import kafka.server.{ConfigType, KafkaConfig} import org.apache.kafka.common.config.{ConfigResource, TopicConfig} import org.apache.kafka.common.errors.ControllerMovedException import org.apache.kafka.common.metadata.{ConfigRecord, MetadataRecordType, PartitionRecord, ProducerIdsRecord, TopicRecord} -import org.apache.kafka.common.{TopicPartition, Uuid} +import org.apache.kafka.common.{DirectoryId, TopicPartition, Uuid} import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.metadata.migration.{KRaftMigrationZkWriter, ZkMigrationLeadershipState} import org.apache.kafka.metadata.{LeaderRecoveryState, PartitionRegistration} @@ -79,8 +79,24 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { assertEquals(0, migrationState.migrationZkVersion()) val partitions = Map( - 0 -> new PartitionRegistration.Builder().setReplicas(Array(0, 1, 2)).setIsr(Array(1, 2)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(6).setPartitionEpoch(-1).build(), - 1 -> new PartitionRegistration.Builder().setReplicas(Array(1, 2, 3)).setIsr(Array(3)).setLeader(3).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(7).setPartitionEpoch(-1).build() + 0 -> new PartitionRegistration.Builder() + .setReplicas(Array(0, 1, 2)) + .setDirectories(DirectoryId.migratingArray(3)) + .setIsr(Array(1, 2)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(6) + .setPartitionEpoch(-1) + .build(), + 1 -> new PartitionRegistration.Builder() + .setReplicas(Array(1, 2, 3)) + .setDirectories(DirectoryId.migratingArray(3)) + .setIsr(Array(3)) + .setLeader(3) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(7) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => Integer.valueOf(k) -> v }.asJava migrationState = migrationClient.topicClient().updateTopicPartitions(Map("test" -> partitions).asJava, migrationState) assertEquals(1, migrationState.migrationZkVersion()) @@ -106,8 +122,24 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { assertEquals(0, migrationState.migrationZkVersion()) val partitions = Map( - 0 -> new PartitionRegistration.Builder().setReplicas(Array(0, 1, 2)).setIsr(Array(0, 1, 2)).setLeader(0).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build(), - 1 -> new PartitionRegistration.Builder().setReplicas(Array(1, 2, 3)).setIsr(Array(1, 2, 3)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build() + 0 -> new PartitionRegistration.Builder() + .setReplicas(Array(0, 1, 2)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(0, 1, 2)) + .setLeader(0) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build(), + 1 -> new PartitionRegistration.Builder() + .setReplicas(Array(1, 2, 3)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(1, 2, 3)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => Integer.valueOf(k) -> v }.asJava migrationState = migrationClient.topicClient().createTopic("test", Uuid.randomUuid(), partitions, migrationState) assertEquals(1, migrationState.migrationZkVersion()) @@ -129,8 +161,24 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { assertEquals(0, migrationState.migrationZkVersion()) val partitions = Map( - 0 -> new PartitionRegistration.Builder().setReplicas(Array(0, 1, 2)).setIsr(Array(0, 1, 2)).setLeader(0).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build(), - 1 -> new PartitionRegistration.Builder().setReplicas(Array(1, 2, 3)).setIsr(Array(1, 2, 3)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build() + 0 -> new PartitionRegistration.Builder() + .setReplicas(Array(0, 1, 2)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(0, 1, 2)) + .setLeader(0) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build(), + 1 -> new PartitionRegistration.Builder() + .setReplicas(Array(1, 2, 3)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(1, 2, 3)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => Integer.valueOf(k) -> v }.asJava val topicId = Uuid.randomUuid() migrationState = migrationClient.topicClient().createTopic("test", topicId, partitions, migrationState) @@ -375,8 +423,24 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { val topicId = Uuid.randomUuid() val partitions = Map( - 0 -> new PartitionRegistration.Builder().setReplicas(Array(0, 1, 2)).setIsr(Array(0, 1, 2)).setLeader(0).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build(), - 1 -> new PartitionRegistration.Builder().setReplicas(Array(1, 2, 3)).setIsr(Array(1, 2, 3)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build() + 0 -> new PartitionRegistration.Builder() + .setReplicas(Array(0, 1, 2)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(0, 1, 2)) + .setLeader(0) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build(), + 1 -> new PartitionRegistration.Builder() + .setReplicas(Array(1, 2, 3)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(1, 2, 3)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => Integer.valueOf(k) -> v }.asJava migrationState = migrationClient.topicClient().createTopic("test", topicId, partitions, migrationState) assertEquals(1, migrationState.migrationZkVersion()) @@ -384,8 +448,24 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { // Change assignment in partitions and update the topic assignment. See the change is // reflected. val changedPartitions = Map( - 0 -> new PartitionRegistration.Builder().setReplicas(Array(1, 2, 3)).setIsr(Array(1, 2, 3)).setLeader(0).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build(), - 1 -> new PartitionRegistration.Builder().setReplicas(Array(0, 1, 2)).setIsr(Array(0, 1, 2)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build() + 0 -> new PartitionRegistration.Builder() + .setReplicas(Array(1, 2, 3)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(1, 2, 3)) + .setLeader(0) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build(), + 1 -> new PartitionRegistration.Builder() + .setReplicas(Array(0, 1, 2)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(0, 1, 2)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => Integer.valueOf(k) -> v }.asJava migrationState = migrationClient.topicClient().updateTopic("test", topicId, changedPartitions, migrationState) assertEquals(2, migrationState.migrationZkVersion()) @@ -406,7 +486,15 @@ class ZkMigrationClientTest extends ZkMigrationTestHarness { // Add a new Partition. val newPartition = Map( - 2 -> new PartitionRegistration.Builder().setReplicas(Array(2, 3, 4)).setIsr(Array(2, 3, 4)).setLeader(1).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(0).setPartitionEpoch(-1).build() + 2 -> new PartitionRegistration.Builder() + .setReplicas(Array(2, 3, 4)) + .setDirectories(DirectoryId.unassignedArray(3)) + .setIsr(Array(2, 3, 4)) + .setLeader(1) + .setLeaderRecoveryState(LeaderRecoveryState.RECOVERED) + .setLeaderEpoch(0) + .setPartitionEpoch(-1) + .build() ).map { case (k, v) => int2Integer(k) -> v }.asJava migrationState = migrationClient.topicClient().createTopicPartitions(Map("test" -> newPartition).asJava, migrationState) assertEquals(3, migrationState.migrationZkVersion()) diff --git a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala index 30bc557829649..ebb6ccf44533a 100644 --- a/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala +++ b/core/src/test/scala/unit/kafka/zookeeper/ZooKeeperClientTest.scala @@ -20,7 +20,6 @@ import java.nio.charset.StandardCharsets import java.util.UUID import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} import java.util.concurrent.{ArrayBlockingQueue, ConcurrentLinkedQueue, CountDownLatch, Executors, Semaphore, TimeUnit} - import scala.collection.Seq import com.yammer.metrics.core.{Gauge, Meter, MetricName} import kafka.server.KafkaConfig @@ -35,7 +34,7 @@ import org.apache.zookeeper.ZooKeeper.States import org.apache.zookeeper.client.ZKClientConfig import org.apache.zookeeper.{CreateMode, WatchedEvent, ZooDefs} import org.junit.jupiter.api.Assertions.{assertArrayEquals, assertEquals, assertFalse, assertThrows, assertTrue, fail} -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, TestInfo, Timeout} import scala.jdk.CollectionConverters._ @@ -333,6 +332,7 @@ class ZooKeeperClientTest extends QuorumTestHarness { } @Test + @Timeout(60) def testBlockOnRequestCompletionFromStateChangeHandler(): Unit = { // This tests the scenario exposed by KAFKA-6879 in which the expiration callback awaits // completion of a request which is handled by another thread diff --git a/docker/README.md b/docker/README.md index 005ce4bec1741..05801053ce62a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,32 +1,61 @@ Docker Images ============= -This directory contains docker image for Kafka. -The scripts take a url containing kafka as input and generate the respective docker image. -There are interactive python scripts to release the docker image and promote a release candidate. +This directory contains scripts to build, test, push and promote docker image for kafka. Local Setup ----------- -Make sure you have python (>= 3.7.x) and java (>= 17) installed before running the tests and scripts. +Make sure you have python (>= 3.7.x) and java (>= 17) (java needed only for running tests) installed before running the tests and scripts. Run `pip install -r requirements.txt` to get all the requirements for running the scripts. +Make sure you have docker installed with support for buildx enabled. (For pushing multi-architecture image to docker registry) + Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. - kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. - Sanity tests for the docker image are present in test/docker_sanity_test.py. -- By default image will be built and tested, but if you only want to build the image, pass `-b` flag and if you only want to test the given image pass `-t` flag. +- By default image will be built and tested, but if you only want to build the image, pass `--build` (or `-b`) flag and if you only want to test the given image pass `--test` (or `-t`) flag. - An html test report will be generated after the tests are executed containing the results. +Example command:- +To build and test an image named test under kafka namespace with 3.6.0 tag and jvm image type ensuring kafka to be containerised should be https://downloads.apache.org/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following command can be used +``` +python docker_build_test.py kafka/test --image-tag=3.6.0 --image-type=jvm --kafka-url=https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` + Bulding image and running tests using github actions ---------------------------------------------------- This is the recommended way to build, test and get a CVE report for the docker image. -Just choose the image type and provide kafka url to `Docker build test` workflow. It will generate a test report and CVE report that can be shared to the community. +Just choose the image type and provide kafka url to `Docker Build Test` workflow. It will generate a test report and CVE report that can be shared with the community. + +kafka-url - This is the url to download kafka tarball from. For example kafka tarball url from (https://archive.apache.org/dist/kafka). For building RC image this will be an RC tarball url. + +image-type - This is the type of image that we intend to build. This will be dropdown menu type selection in the workflow. `jvm` image type is for official docker image (to be hosted on apache/kafka) as described in [KIP-975](https://cwiki.apache.org/confluence/display/KAFKA/KIP-975%3A+Docker+Image+for+Apache+Kafka) + +Example command:- +To build and test a jvm image type ensuring kafka to be containerised should be https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following inputs in github actions workflow are recommended. +``` +image_type: jvm +kafka_url: https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` Creating a release ------------------ -`docker_release.py` provides an interactive way to build multi arch image and publish it to a docker registry. +- `docker_release.py` script builds a multi-architecture image and pushes it to provided docker registry. +- Ensure you are logged in to the docker registry before triggering the script. +- kafka binary tarball url along with image name (in the format `//:`) and type is needed to build the image. For detailed usage description check `python docker_release.py --help`. + +Example command:- +To push an image named test under kafka dockerhub namespace with 3.6.0 tag and jvm image type ensuring kafka to be containerised should be https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following command can be used. (Make sure you have push access to the docker repo) +``` +# kafka/test is an example repo. Please replace with the docker hub repo you have push access to. + +python docker_release.py kafka/test:3.6.0 --kafka-url https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` + +Please note that we use docker buildx for preparing the multi-architecture image and pushing it to docker registry. It's possible to encounter build failures because of buildx. Please retry the command in case some buildx related error occurs. Promoting a release ------------------- @@ -36,35 +65,38 @@ Using the image in a docker container ------------------------------------- - The image uses the kafka downloaded from provided kafka url - The image can be run in a container in default mode by running -`docker run -p 9092:9092` + `docker run -p 9092:9092 ` - Default configs run kafka in kraft mode with plaintext listners on 9092 port. -- Default configs can be overriden by user using 2 ways:- +- Once user provided config properties are provided default configs will get replaced. +- User can provide kafka configs following two ways:- - By mounting folder containing property files - Mount the folder containing kafka property files to `/mnt/shared/config` - - These files will override the default config files + - These files will replace the default config files - Using environment variables - Kafka properties defined via env variables will override properties defined in file input - - Replace . with _ - - Replace _ with __(double underscore) - - Replace - with ___(triple underscore) - - Prefix the result with KAFKA_ - - Examples: - - For abc.def, use KAFKA_ABC_DEF - - For abc-def, use KAFKA_ABC___DEF - - For abc_def, use KAFKA_ABC__DEF -- Hence order of precedence of properties is the follwing:- + - If properties are provided via environment variables only, default configs will be replaced by user provided properties + - Input format for env variables:- + - Replace . with _ + - Replace _ with __(double underscore) + - Replace - with ___(triple underscore) + - Prefix the result with KAFKA_ + - Examples: + - For abc.def, use KAFKA_ABC_DEF + - For abc-def, use KAFKA_ABC___DEF + - For abc_def, use KAFKA_ABC__DEF +- Hence order of precedence of properties is the following:- - Env variable (highest) - File input - - Default (lowest) + - Default configs (only when there is no user provided config) - Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts Steps to release docker image ----------------------------- -- Make sure you have executed release.py script to prepare RC tarball in apache sftp server. -- Use the RC tarball url as input kafka url to build docker image and run sanity tests. +- Make sure you have executed `release.py` script to prepare RC tarball in apache sftp server. +- Use the RC tarball url (make sure you choose scala 2.13 version) as input kafka url to build docker image and run sanity tests. - Trigger github actions workflow using the RC branch, provide RC tarball url as kafka url. - This will generate test report and CVE report for docker images. - If the reports look fine, RC docker image can be built and published. -- Execute `docker_release.py` script to build and publish RC docker image in your own dockerhub account. +- Execute `docker_release.py` script to build and publish RC docker image in your dockerhub account. - Share the RC docker image, test report and CVE report with the community in RC vote email. - Once approved and ready, take help from someone in PMC to trigger `docker_promote.py` script and promote the RC docker image to apache/kafka dockerhub repo diff --git a/docker/common.py b/docker/common.py index 66c4888c17853..1c94e173eec19 100644 --- a/docker/common.py +++ b/docker/common.py @@ -16,6 +16,10 @@ # limitations under the License. import subprocess +import tempfile +import os +from distutils.dir_util import copy_tree +import shutil def execute(command): if subprocess.run(command).returncode != 0: @@ -25,4 +29,18 @@ def get_input(message): value = input(message) if value == "": raise ValueError("This field cannot be empty") - return value \ No newline at end of file + return value + +def jvm_image(command): + temp_dir_path = tempfile.mkdtemp() + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/jvm", f"{temp_dir_path}/jvm") + copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/jvm/resources") + command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/jvm/Dockerfile") + command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/jvm") + try: + execute(command.split()) + except: + raise SystemError("Docker Image Build failed") + finally: + shutil.rmtree(temp_dir_path) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 32fc48ebc2e29..d376796d41f55 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,50 +15,63 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Python script to build and test a docker image +This script is used to generate a test report + +Usage: + docker_build_test.py --help + Get detailed description of each option + + Example command:- + docker_build_test.py --image-tag --image-type --kafka-url + + This command will build an image with as image name, as image_tag (it will be latest by default), + as image type (jvm by default), for the kafka inside the image and run tests on the image. + -b can be passed as additional argument if you just want to build the image. + -t can be passed if you just want to run tests on the image. +""" + from datetime import date import argparse from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests -from common import execute +from common import execute, jvm_image +import tempfile +import os def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' - copy_tree("resources", "jvm/resources") - execute(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", - "--build-arg", f'build_date={date.today()}', "jvm"]) - - shutil.rmtree("jvm/resources") - -def build_native(image, tag, kafka_url): - image = f'{image}:{tag}' - copy_tree("resources", "native-image/resources") - result = subprocess.run( - ["docker", "build", "-f", "native-image/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", - "--build-arg", f'build_date={date.today()}', "native-image"]) - if result.stderr: - print(result.stdout) - return - shutil.rmtree("native-image/resources") + jvm_image(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR") def run_jvm_tests(image, tag, kafka_url): - execute(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - execute(["mkdir", "./test/fixtures/kafka"]) - execute(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) - failure_count = run_tests(f"{image}:{tag}", "jvm") - execute(["rm", "kafka.tgz"]) - shutil.rmtree("./test/fixtures/kafka") + temp_dir_path = tempfile.mkdtemp() + try: + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures") + execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url]) + execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"]) + execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) + failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path) + except: + raise SystemError("Failed to run the tests") + finally: + shutil.rmtree(temp_dir_path) + test_report_location_text = f"To view test report please check {current_dir}/test/report_jvm.html" if failure_count != 0: - raise SystemError("Test Failure. Error count is non 0") + raise SystemError(f"{failure_count} tests have failed. {test_report_location_text}") + else: + print(f"All tests passed successfully. {test_report_location_text}") if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") - parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", choices=["jvm", "native"], default="jvm", dest="image_type", help="Image type you want to build") - parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") - parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") - parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") + parser.add_argument("--image-tag", "-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") + parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + parser.add_argument("--build", "-b", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") + parser.add_argument("--test", "-t", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() if args.image_type == "jvm" and (args.build_only or not (args.build_only or args.test_only)): @@ -67,12 +80,5 @@ def run_jvm_tests(image, tag, kafka_url): else: raise ValueError("--kafka-url is a required argument for jvm image") - if args.image_type == "native" and (args.build_only or not (args.build_only or args.test_only)): - if args.kafka_url: - build_native(args.image, args.tag, args.kafka_url) - else: - raise ValueError("--kafka-url is a required argument for building docker image") - if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): run_jvm_tests(args.image, args.tag, args.kafka_url) - diff --git a/docker/docker_promote.py b/docker/docker_promote.py index 75cd2fc06da1f..1074947174df5 100644 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -82,4 +82,4 @@ def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") else: - print("Image promotion aborted") \ No newline at end of file + print("Image promotion aborted") diff --git a/docker/docker_release.py b/docker/docker_release.py index 3a774ef11f9e4..50eee56a490a1 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -16,30 +16,39 @@ # limitations under the License. """ -Python script to build and push docker image +Python script to build and push a multiarch docker image This script is used to prepare and publish docker release candidate -Usage: docker_release.py +Pre requisites: + Ensure that you are logged in the docker registry and you have access to push to that registry. + Ensure that docker buildx is enabled for you. -Interactive utility to push the docker image to dockerhub +Usage: + docker_release.py --help + Get detailed description of argument + + Example command:- + docker_release --kafka-url --image-type + + This command will build the multiarch image of type (jvm by default), + named using to download kafka and push it to the docker image name provided. + Make sure image is in the format of //:. """ -from distutils.dir_util import copy_tree from datetime import date -import shutil - -from common import execute, get_input +import argparse -def push_jvm(image, kafka_url): - copy_tree("resources", "jvm/resources") - execute(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", - "--push", - "--platform", "linux/amd64,linux/arm64", - "--tag", image, "jvm"]) - shutil.rmtree("jvm/resources") +from common import execute, jvm_image -def login(): - execute(["docker", "login"]) +def build_push_jvm(image, kafka_url): + try: + create_builder() + jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ + --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR") + except: + raise SystemError("Docker image push failed") + finally: + remove_builder() def create_builder(): execute(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) @@ -47,32 +56,20 @@ def create_builder(): def remove_builder(): execute(["docker", "buildx", "rm", "kafka-builder"]) -def get_input(message): - value = input(message) - if value == "": - raise ValueError("This field cannot be empty") - return value - if __name__ == "__main__": print("\ - This script will build and push docker images of apache kafka.\n\ - Please ensure that image has been sanity tested before pushing the image") - login() - docker_registry = input("Enter the docker registry you want to push the image to [docker.io]: ") - if docker_registry == "": - docker_registry = "docker.io" - docker_namespace = input("Enter the docker namespace you want to push the image to: ") - image_name = get_input("Enter the image name: ") - image_tag = get_input("Enter the image tag for the image: ") - kafka_url = get_input("Enter the url for kafka binary tarball: ") - image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" - print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") - proceed = input("Should we proceed? [y/N]: ") - if proceed == "y": - print("Building and pushing the image") - create_builder() - push_jvm(image, kafka_url) - remove_builder() - print(f"Image has been pushed to {image}") - else: - print("Image push aborted") + This script will build and push docker images of apache kafka.\n \ + Please ensure that image has been sanity tested before pushing the image. \n \ + Please ensure you are logged in the docker registry that you are trying to push to.") + parser = argparse.ArgumentParser() + parser.add_argument("image", help="Dockerhub image that you want to push to (in the format //:)") + parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + args = parser.parse_args() + + print(f"Docker image of type {args.image_type} containing kafka downloaded from {args.kafka_url} will be pushed to {args.image}") + + print("Building and pushing the image") + if args.image_type == "jvm": + build_push_jvm(args.image, args.kafka_url) + print(f"Image has been pushed to {args.image}") diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e5c9d7a47cb9a..9a74e96125359 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -16,11 +16,13 @@ # limitations under the License. ############################################################################### -FROM golang:latest AS build-ub +FROM golang:latest AS build-utility WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser resources/ub/ ./ -RUN go build -ldflags="-w -s" ./ub.go +COPY --chown=appuser:appuser resources/utility/ ./ + +# Generate utility executable for dealing with env variables +RUN go build -ldflags="-w -s" ./utility.go USER appuser RUN go test ./... @@ -37,11 +39,16 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat procps netcat-openbsd; \ + apk add --no-cache wget gcompat gpg gpg-agent procps netcat-openbsd uuidgen; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ - tar xfz kafka.tgz -C /opt/kafka --strip-components 1; + wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ + wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ + gpg --import KEYS; \ + gpg --batch --verify kafka.tgz.asc kafka.tgz +# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command RUN /etc/kafka/docker/jsa_launch @@ -61,13 +68,12 @@ LABEL org.label-schema.name="kafka" \ org.label-schema.description="Apache Kafka" \ org.label-schema.build-date="${build_date}" \ org.label-schema.vcs-url="https://github.com/apache/kafka" \ - org.label-schema.schema-version="1.0" \ - maintainer="apache" + maintainer="Apache Kafka" RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ + apk add --no-cache wget gpg gpg-agent gcompat; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ @@ -81,13 +87,15 @@ RUN set -eux ; \ chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /etc/kafka; \ chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \ + cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \ + cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \ rm kafka.tgz kafka.tgz.asc KEYS; \ - apk del curl wget gpg dirmngr gpg-agent; \ + apk del wget gpg gpg-agent; \ apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa -COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin +COPY --chown=appuser:appuser --from=build-utility /build/utility /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 50d620d6724ac..d7efe5845c7dc 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -15,6 +15,7 @@ # limitations under the License. KAFKA_CLUSTER_ID="5L6g3nShT-eMCtK--X86sw" +TOPIC="$(uuidgen)" KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=storage.jsa" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties @@ -29,13 +30,13 @@ check_timeout() { sleep 1 } -opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 +opt/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 +echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic $TOPIC --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 +opt/kafka/bin/kafka-console-consumer.sh --topic $TOPIC --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 [ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh diff --git a/docker/jvm/launch b/docker/jvm/launch index ccab3e02cda7a..de5e6e008065f 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -24,7 +24,7 @@ fi # The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. # For host n/w, this is the IP that the hostname on the host resolves to. -# If you have more that one n/w configured, hostname -i gives you all the IPs, +# If you have more than one n/w configured, hostname -i gives you all the IPs, # the default is to pick the first IP (or network). export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} @@ -34,22 +34,26 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi +# Make a temp env variable to store user provided performance otps if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="" else export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS" fi +# We will first use CDS for storage to format storage export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/storage.jsa" echo "===> Using provided cluster id $CLUSTER_ID ..." -# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this +# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } +# Using temp env variable to get rid of storage CDS command export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS" +# Now we will use CDS for kafka to start kafka server export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" # Start kafka broker diff --git a/docker/native-image/Dockerfile2 b/docker/native-image/Dockerfile2 new file mode 100644 index 0000000000000..53d7fba76e8af --- /dev/null +++ b/docker/native-image/Dockerfile2 @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.21-bullseye AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser resources/ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image + +WORKDIR /app + +COPY kafka_2.13-3.7.0-SNAPSHOT.tgz kafka_2.13-3.7.0-SNAPSHOT.tgz +COPY native-image-configs native-image-configs + +RUN tar -xzf kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + rm kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + cd kafka_2.13-3.7.0-SNAPSHOT ; \ + native-image --no-fallback \ + --allow-incomplete-classpath \ + --report-unsupported-elements-at-runtime \ + --install-exit-handlers \ + -H:+ReportExceptionStackTraces \ + -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \ + -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \ + -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \ + -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \ + -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \ + -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \ + --verbose \ + -cp "libs/*" kafka.KafkaNativeWrapper + + +FROM alpine:latest +RUN apk update && \ + apk add --no-cache gcompat && \ + apk add --no-cache bash && \ + apk -v cache clean && \ + mkdir -p /etc/kafka/config +WORKDIR /app + +COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . +COPY resources/common-scripts /etc/kafka/docker +COPY launch /etc/kafka/docker/ + +CMD ["/etc/kafka/docker/launch"] diff --git a/docker/report_jvm.html b/docker/report_jvm.html new file mode 100644 index 0000000000000..64d5790c2577f --- /dev/null +++ b/docker/report_jvm.html @@ -0,0 +1,276 @@ + + + + + Test Report + + + + + + + + + +
    +

    Test Report

    +

    Start Time: 2023-11-23 23:36:44

    +

    Duration: 0:00:41.693944

    +

    Status: Pass 1

    + +

    This demonstrates the report output.

    +
    + + + +

    Show +Summary +Failed +All +

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Test Group/Test caseCountPassFailErrorView
    test.docker_sanity_test.DockerSanityTestJVM1100Detail
    test_bed
    		 + + + + pass + + + + +
    Total1100 
    + +
     
    + + + diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 4e812ad7cd731..d68b988f212b8 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -14,15 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# --- for broker - -# If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode - # unset KAFKA_ADVERTISED_LISTENERS from ENV in KRaft mode when running as controller only if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] then echo "Running in KRaft mode..." - ub ensure CLUSTER_ID + utility ensure CLUSTER_ID if [[ $KAFKA_PROCESS_ROLES == "controller" ]] then if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] @@ -30,11 +26,10 @@ then echo "KAFKA_ADVERTISED_LISTENERS is not supported on a KRaft controller." exit 1 else + # Unset in case env variable is set with empty value unset KAFKA_ADVERTISED_LISTENERS fi - else - ub ensure KAFKA_ADVERTISED_LISTENERS - fi + fi fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing @@ -46,67 +41,41 @@ then KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') fi -ub path /opt/kafka/config/ writable - -# advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. -if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] -then - echo "advertised.port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_ADVERTISED_HOST-}" ]] -then - echo "advertised.host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_HOST-}" ]] -then - echo "host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_PORT-}" ]] -then - echo "port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi +utility path /opt/kafka/config/ writable # Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] then echo "SSL is enabled." - ub ensure KAFKA_SSL_KEYSTORE_FILENAME + utility ensure KAFKA_SSL_KEYSTORE_FILENAME export KAFKA_SSL_KEYSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_FILENAME" - ub path "$KAFKA_SSL_KEYSTORE_LOCATION" existence + utility path "$KAFKA_SSL_KEYSTORE_LOCATION" existence - ub ensure KAFKA_SSL_KEY_CREDENTIALS + utility ensure KAFKA_SSL_KEY_CREDENTIALS KAFKA_SSL_KEY_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEY_CREDENTIALS" - ub path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence export KAFKA_SSL_KEY_PASSWORD KAFKA_SSL_KEY_PASSWORD=$(cat "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION") - ub ensure KAFKA_SSL_KEYSTORE_CREDENTIALS + utility ensure KAFKA_SSL_KEYSTORE_CREDENTIALS KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_CREDENTIALS" - ub path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence export KAFKA_SSL_KEYSTORE_PASSWORD KAFKA_SSL_KEYSTORE_PASSWORD=$(cat "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION") if [[ -n "${KAFKA_SSL_CLIENT_AUTH-}" ]] && ( [[ $KAFKA_SSL_CLIENT_AUTH == *"required"* ]] || [[ $KAFKA_SSL_CLIENT_AUTH == *"requested"* ]] ) then - ub ensure KAFKA_SSL_TRUSTSTORE_FILENAME + utility ensure KAFKA_SSL_TRUSTSTORE_FILENAME export KAFKA_SSL_TRUSTSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_FILENAME" - ub path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence + utility path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence - ub ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS + utility ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_CREDENTIALS" - ub path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence export KAFKA_SSL_TRUSTSTORE_PASSWORD KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") fi - fi # Set if KAFKA_ADVERTISED_LISTENERS has SASL_PLAINTEXT:// or SASL_SSL:// endpoints. @@ -114,7 +83,7 @@ if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS = then echo "SASL" is enabled. - ub ensure KAFKA_OPTS + utility ensure KAFKA_OPTS if [[ ! $KAFKA_OPTS == *"java.security.auth.login.config"* ]] then @@ -130,12 +99,32 @@ then fi fi -mv /etc/kafka/docker/server.properties /opt/kafka/config/server.properties -mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties -mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties +# Copy the bundled log4j.properties and tools-log4j.properties. This is done to handle property modification during container restart +cp /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties +cp /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties +# Copy all the user provided property files through file input cp -R /mnt/shared/config/. /opt/kafka/config/ -echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties -echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties -echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties +# Check the presence of user provided kafka configs via file input +if [ -e "/mnt/shared/config/server.properties" ] +then + echo "User provided kafka configs found via file input. Any properties provided via env variables will be appended to this." + # Append configs provided via env variables. + echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties +else + # Create the kafka config property file using user provided environment variables. + echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" > /opt/kafka/config/server.properties + if grep -q '[^[:space:]]' "/opt/kafka/config/server.properties"; then + echo "User provided kafka configs found via environment variables." + fi +fi + +# If no user provided kafka configs found, use default configs +if ! grep -q '[^[:space:]]' "/opt/kafka/config/server.properties"; then + echo "User provided kafka configs not found (neither via file input nor via environment variables). Falling back to default configs." + cp /opt/kafka/config/kraft/server.properties /opt/kafka/config/server.properties +fi + +echo -e "\n$(utility render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties +echo -e "\n$(utility render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/utility/go.mod b/docker/resources/utility/go.mod new file mode 100644 index 0000000000000..3ec7b827348d0 --- /dev/null +++ b/docker/resources/utility/go.mod @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module ub + +go 1.19 + +require ( + github.com/spf13/cobra v1.7.0 + golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c + golang.org/x/sys v0.7.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/docker/resources/utility/go.sum b/docker/resources/utility/go.sum new file mode 100644 index 0000000000000..5f20e19b6de1e --- /dev/null +++ b/docker/resources/utility/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c h1:HDdYQYKOkvJT/Plb5HwJJywTVyUnIctjQm6XSnZ/0CY= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docker/resources/utility/testResources/sampleFile b/docker/resources/utility/testResources/sampleFile new file mode 100755 index 0000000000000..91eacc92e8be9 --- /dev/null +++ b/docker/resources/utility/testResources/sampleFile @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/utility/testResources/sampleFile2 b/docker/resources/utility/testResources/sampleFile2 new file mode 100755 index 0000000000000..91eacc92e8be9 --- /dev/null +++ b/docker/resources/utility/testResources/sampleFile2 @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/utility/testResources/sampleLog4j.template b/docker/resources/utility/testResources/sampleLog4j.template new file mode 100644 index 0000000000000..8bc1f5e3dbd4d --- /dev/null +++ b/docker/resources/utility/testResources/sampleLog4j.template @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout + +{{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} +{{ range $k, $v := splitToMapDefaults "," "" $loggers}} +log4j.logger.{{ $k }}={{ $v -}} +{{ end }} \ No newline at end of file diff --git a/docker/resources/utility/utility.go b/docker/resources/utility/utility.go new file mode 100644 index 0000000000000..521b837b41662 --- /dev/null +++ b/docker/resources/utility/utility.go @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + pt "path" + "regexp" + "sort" + "strings" + "text/template" + + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "golang.org/x/sys/unix" +) + +type ConfigSpec struct { + Prefixes map[string]bool `json:"prefixes"` + Excludes []string `json:"excludes"` + Renamed map[string]string `json:"renamed"` + Defaults map[string]string `json:"defaults"` + ExcludeWithPrefix string `json:"excludeWithPrefix"` +} + +var ( + re = regexp.MustCompile("[^_]_[^_]") + + ensureCmd = &cobra.Command{ + Use: "ensure ", + Short: "checks if environment variable is set or not", + Args: cobra.ExactArgs(1), + RunE: runEnsureCmd, + } + + pathCmd = &cobra.Command{ + Use: "path ", + Short: "checks if an operation is permitted on a file", + Args: cobra.ExactArgs(2), + RunE: runPathCmd, + } + + renderTemplateCmd = &cobra.Command{ + Use: "render-template ", + Short: "renders template to stdout", + Args: cobra.ExactArgs(1), + RunE: runRenderTemplateCmd, + } + + renderPropertiesCmd = &cobra.Command{ + Use: "render-properties ", + Short: "creates and renders properties to stdout using the json config spec.", + Args: cobra.ExactArgs(1), + RunE: runRenderPropertiesCmd, + } +) + +func ensure(envVar string) bool { + _, found := os.LookupEnv(envVar) + return found +} + +func path(filePath string, operation string) (bool, error) { + switch operation { + + case "readable": + err := unix.Access(filePath, unix.R_OK) + if err != nil { + return false, err + } + return true, nil + case "executable": + info, err := os.Stat(filePath) + if err != nil { + err = fmt.Errorf("error checking executable status of file %q: %w", filePath, err) + return false, err + } + return info.Mode()&0111 != 0, nil //check whether file is executable by anyone, use 0100 to check for execution rights for owner + case "existence": + if _, err := os.Stat(filePath); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil + case "writable": + err := unix.Access(filePath, unix.W_OK) + if err != nil { + return false, err + } + return true, nil + default: + err := fmt.Errorf("unknown operation %q", operation) + return false, err + } +} + +func renderTemplate(templateFilePath string) error { + funcs := template.FuncMap{ + "getEnv": getEnvOrDefault, + "splitToMapDefaults": splitToMapDefaults, + } + t, err := template.New(pt.Base(templateFilePath)).Funcs(funcs).ParseFiles(templateFilePath) + if err != nil { + err = fmt.Errorf("error %q: %w", templateFilePath, err) + return err + } + return buildTemplate(os.Stdout, *t) +} + +func buildTemplate(writer io.Writer, template template.Template) error { + err := template.Execute(writer, GetEnvironment()) + if err != nil { + err = fmt.Errorf("error building template file : %w", err) + return err + } + return nil +} + +func renderConfig(writer io.Writer, configSpec ConfigSpec) error { + return writeConfig(writer, buildProperties(configSpec, GetEnvironment())) +} + +// ConvertKey Converts an environment variable name to a property-name according to the following rules: +// - a single underscore (_) is replaced with a . +// - a double underscore (__) is replaced with a single underscore +// - a triple underscore (___) is replaced with a dash +// Moreover, the whole string is converted to lower-case. +// The behavior of sequences of four or more underscores is undefined. +func ConvertKey(key string) string { + singleReplaced := re.ReplaceAllStringFunc(key, replaceUnderscores) + singleTripleReplaced := strings.ReplaceAll(singleReplaced, "___", "-") + return strings.ToLower(strings.ReplaceAll(singleTripleReplaced, "__", "_")) +} + +// replaceUnderscores replaces every underscore '_' by a dot '.' +func replaceUnderscores(s string) string { + return strings.ReplaceAll(s, "_", ".") +} + +// ListToMap splits each and entry of the kvList argument at '=' into a key/value pair and returns a map of all the k/v pair thus obtained. +// this method will only consider values in the list formatted as key=value +func ListToMap(kvList []string) map[string]string { + m := make(map[string]string, len(kvList)) + for _, l := range kvList { + parts := strings.Split(l, "=") + if len(parts) == 2 { + m[parts[0]] = parts[1] + } + } + return m +} + +func splitToMapDefaults(separator string, defaultValues string, value string) map[string]string { + values := KvStringToMap(defaultValues, separator) + for k, v := range KvStringToMap(value, separator) { + values[k] = v + } + return values +} + +func KvStringToMap(kvString string, sep string) map[string]string { + return ListToMap(strings.Split(kvString, sep)) +} + +// GetEnvironment returns the current environment as a map. +func GetEnvironment() map[string]string { + return ListToMap(os.Environ()) +} + +// buildProperties creates a map suitable to be output as Java properties from a ConfigSpec and a map representing an environment. +func buildProperties(spec ConfigSpec, environment map[string]string) map[string]string { + config := make(map[string]string) + for key, value := range spec.Defaults { + config[key] = value + } + + for envKey, envValue := range environment { + if newKey, found := spec.Renamed[envKey]; found { + config[newKey] = envValue + } else { + if !slices.Contains(spec.Excludes, envKey) && !(len(spec.ExcludeWithPrefix) > 0 && strings.HasPrefix(envKey, spec.ExcludeWithPrefix)) { + for prefix, keep := range spec.Prefixes { + if strings.HasPrefix(envKey, prefix) { + var effectiveKey string + if keep { + effectiveKey = envKey + } else { + effectiveKey = envKey[len(prefix)+1:] + } + config[ConvertKey(effectiveKey)] = envValue + } + } + } + } + } + return config +} + +func writeConfig(writer io.Writer, config map[string]string) error { + // Go randomizes iterations over map by design. We sort properties by name to ease debugging: + sortedNames := make([]string, 0, len(config)) + for name := range config { + sortedNames = append(sortedNames, name) + } + sort.Strings(sortedNames) + for _, n := range sortedNames { + _, err := fmt.Fprintf(writer, "%s=%s\n", n, config[n]) + if err != nil { + err = fmt.Errorf("error printing configs: %w", err) + return err + } + } + return nil +} + +func loadConfigSpec(path string) (ConfigSpec, error) { + var spec ConfigSpec + bytes, err := os.ReadFile(path) + if err != nil { + err = fmt.Errorf("error reading from json file %q : %w", path, err) + return spec, err + } + + errParse := json.Unmarshal(bytes, &spec) + if errParse != nil { + err = fmt.Errorf("error parsing json file %q : %w", path, errParse) + return spec, err + } + return spec, nil +} + +func getEnvOrDefault(envVar string, defaultValue string) string { + val := os.Getenv(envVar) + if len(val) == 0 { + return defaultValue + } + return val +} + +func runEnsureCmd(_ *cobra.Command, args []string) error { + success := ensure(args[0]) + if !success { + err := fmt.Errorf("environment variable %q is not set", args[0]) + return err + } + return nil +} + +func runPathCmd(_ *cobra.Command, args []string) error { + success, err := path(args[0], args[1]) + if err != nil { + err = fmt.Errorf("error in checking operation %q on file %q: %w", args[1], args[0], err) + return err + } + if !success { + err = fmt.Errorf("operation %q on file %q is unsuccessful", args[1], args[0]) + return err + } + return nil +} + +func runRenderTemplateCmd(_ *cobra.Command, args []string) error { + err := renderTemplate(args[0]) + if err != nil { + err = fmt.Errorf("error in rendering template %q: %w", args[0], err) + return err + } + return nil +} + +func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { + configSpec, err := loadConfigSpec(args[0]) + if err != nil { + err = fmt.Errorf("error in loading config from file %q: %w", args[0], err) + return err + } + err = renderConfig(os.Stdout, configSpec) + if err != nil { + err = fmt.Errorf("error in building properties from file %q: %w", args[0], err) + return err + } + return nil +} + +func main() { + rootCmd := &cobra.Command{ + Use: "utility", + Short: "utility commands for kafka docker images", + Run: func(cmd *cobra.Command, args []string) {}, + } + + rootCmd.AddCommand(pathCmd) + rootCmd.AddCommand(ensureCmd) + rootCmd.AddCommand(renderTemplateCmd) + rootCmd.AddCommand(renderPropertiesCmd) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if err := rootCmd.ExecuteContext(ctx); err != nil { + fmt.Fprintf(os.Stderr, "error in executing the command: %s", err) + os.Exit(1) + } +} diff --git a/docker/resources/utility/utility_test.go b/docker/resources/utility/utility_test.go new file mode 100644 index 0000000000000..9d09c5c413e27 --- /dev/null +++ b/docker/resources/utility/utility_test.go @@ -0,0 +1,355 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "reflect" + "testing" +) + +func assertEqual(a string, b string, t *testing.T) { + if a != b { + t.Error(a + " != " + b) + } +} + +func Test_ensure(t *testing.T) { + type args struct { + envVar string + } + err := os.Setenv("ENV_VAR", "value") + if err != nil { + t.Fatal("Unable to set ENV_VAR for the test") + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "should exist", + args: args{ + envVar: "ENV_VAR", + }, + want: true, + }, + { + name: "should not exist", + args: args{ + envVar: "RANDOM_ENV_VAR", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ensure(tt.args.envVar); got != tt.want { + t.Errorf("ensure() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_path(t *testing.T) { + type args struct { + filePath string + operation string + } + const ( + sampleFile = "testResources/sampleFile" + sampleFile2 = "testResources/sampleFile2" + fileDoesNotExist = "testResources/sampleFile3" + ) + err := os.Chmod(sampleFile, 0777) + if err != nil { + t.Error("Unable to set permissions for the file") + } + err = os.Chmod(sampleFile2, 0000) + if err != nil { + t.Error("Unable to set permissions for the file") + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "file readable", + args: args{filePath: sampleFile, + operation: "readable"}, + want: true, + wantErr: false, + }, + { + name: "file writable", + args: args{filePath: sampleFile, + operation: "writable"}, + want: true, + wantErr: false, + }, + { + name: "file executable", + args: args{filePath: sampleFile, + operation: "executable"}, + want: true, + wantErr: false, + }, + { + name: "file existence", + args: args{filePath: sampleFile, + operation: "existence"}, + want: true, + wantErr: false, + }, + { + name: "file not readable", + args: args{filePath: sampleFile2, + operation: "readable"}, + want: false, + wantErr: true, + }, + { + name: "file not writable", + args: args{filePath: sampleFile2, + operation: "writable"}, + want: false, + wantErr: true, + }, + { + name: "file not executable", + args: args{filePath: sampleFile2, + operation: "executable"}, + want: false, + wantErr: false, + }, + { + name: "file does not exist", + args: args{filePath: fileDoesNotExist, + operation: "existence"}, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := path(tt.args.filePath, tt.args.operation) + if (err != nil) != tt.wantErr { + t.Errorf("path() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("path() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_renderTemplate(t *testing.T) { + type args struct { + templateFilePath string + } + const ( + fileExistsAndRenderable = "testResources/sampleLog4j.template" + fileDoesNotExist = "testResources/RandomFileName" + ) + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "render template success", + args: args{templateFilePath: fileExistsAndRenderable}, + wantErr: false, + }, + { + name: "render template failure ", + args: args{templateFilePath: fileDoesNotExist}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := renderTemplate(tt.args.templateFilePath); (err != nil) != tt.wantErr { + t.Errorf("renderTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} +func Test_convertKey(t *testing.T) { + type args struct { + key string + } + tests := []struct { + name string + args args + wantString string + }{ + { + name: "Capitals", + args: args{key: "KEY"}, + wantString: "key", + }, + { + name: "Capitals with underscore", + args: args{key: "KEY_FOO"}, + wantString: "key.foo", + }, + { + name: "Capitals with double underscore", + args: args{key: "KEY__UNDERSCORE"}, + wantString: "key_underscore", + }, + { + name: "Capitals with double and single underscore", + args: args{key: "KEY_WITH__UNDERSCORE_AND__MORE"}, + wantString: "key.with_underscore.and_more", + }, + { + name: "Capitals with triple underscore", + args: args{key: "KEY___DASH"}, + wantString: "key-dash", + }, + { + name: "capitals with double,triple and single underscore", + args: args{key: "KEY_WITH___DASH_AND___MORE__UNDERSCORE"}, + wantString: "key.with-dash.and-more_underscore", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := ConvertKey(tt.args.key); result != tt.wantString { + t.Errorf("ConvertKey() result = %v, wantStr %v", result, tt.wantString) + } + }) + } +} + +func Test_buildProperties(t *testing.T) { + type args struct { + spec ConfigSpec + environment map[string]string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "only defaults", + args: args{ + spec: ConfigSpec{ + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value"}, + }, + { + name: "server properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "localhost:9092", "default.property.key": "default.property.value"}, + }, + { + name: "kafka properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "KAFKA_FOO": "foo", + "KAFKA_FOO_BAR": "bar", + "KAFKA_IGNORED": "ignored", + "KAFKA_WITH__UNDERSCORE": "with underscore", + "KAFKA_WITH__UNDERSCORE_AND_MORE": "with underscore and more", + "KAFKA_WITH___DASH": "with dash", + "KAFKA_WITH___DASH_AND_MORE": "with dash and more", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value", "foo": "foo", "foo.bar": "bar", "with-dash": "with dash", "with-dash.and.more": "with dash and more", "with_underscore": "with underscore", "with_underscore.and.more": "with underscore and more"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildProperties(tt.args.spec, tt.args.environment); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildProperties() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_splitToMapDefaults(t *testing.T) { + type args struct { + separator string + defaultValues string + value string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "split to default", + args: args{ + separator: ",", + defaultValues: "kafka=INFO,kafka.producer.async.DefaultEventHandler=DEBUG,state.change.logger=TRACE", + value: "kafka.producer.async.DefaultEventHandler=ERROR,kafka.request.logger=WARN", + }, + want: map[string]string{"kafka": "INFO", "kafka.producer.async.DefaultEventHandler": "ERROR", "kafka.request.logger": "WARN", "state.change.logger": "TRACE"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := splitToMapDefaults(tt.args.separator, tt.args.defaultValues, tt.args.value); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitToMapDefaults() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/docker/test/__init__.py b/docker/test/__init__.py index 977976beec24a..8f97ef9f46252 100644 --- a/docker/test/__init__.py +++ b/docker/test/__init__.py @@ -13,4 +13,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/docker/test/constants.py b/docker/test/constants.py index 4f1fc3ce59ecf..cc8e82490662f 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,24 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" -KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" -KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" -KAFKA_RUN_CLASS="./test/fixtures/kafka/bin/kafka-run-class.sh" +KAFKA_TOPICS="fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_RUN_CLASS="fixtures/kafka/bin/kafka-run-class.sh" -JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" +JVM_COMBINED_MODE_COMPOSE="fixtures/jvm/combined/docker-compose.yml" +JVM_ISOLATED_COMPOSE="fixtures/jvm/isolated/docker-compose.yml" -CLIENT_TIMEOUT=40 +CLIENT_TIMEOUT=40000 SSL_FLOW_TESTS="SSL Flow Tests" -SSL_CLIENT_CONFIG="./test/fixtures/secrets/client-ssl.properties" +SSL_CLIENT_CONFIG="fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" FILE_INPUT_FLOW_TESTS="File Input Flow Tests" FILE_INPUT_TOPIC="test-topic-file-input" BROKER_RESTART_TESTS="Broker Restart Tests" -BROKER_CONTAINER="broker" +BROKER_CONTAINER="broker1" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" BROKER_METRICS_TESTS="Broker Metrics Tests" @@ -41,4 +42,4 @@ SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" -BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" \ No newline at end of file +BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 522927545de78..b42c6fd00e237 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -19,65 +19,63 @@ import subprocess from HTMLTestRunner import HTMLTestRunner import test.constants as constants +import os class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" - + FIXTURES_DIR="." + def resume_container(self): subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) def stop_container(self) -> None: subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) - def start_compose(self, filename) -> None: - old_string="image: {$IMAGE}" - new_string=f"image: {self.IMAGE}" + def update_file(self, filename, old_string, new_string): with open(filename) as f: s = f.read() with open(filename, 'w') as f: s = s.replace(old_string, new_string) f.write(s) + def start_compose(self, filename) -> None: + self.update_file(filename, "image: {$IMAGE}", f"image: {self.IMAGE}") + self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", "{$DIR}", self.FIXTURES_DIR) subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - + def destroy_compose(self, filename) -> None: - old_string=f"image: {self.IMAGE}" - new_string="image: {$IMAGE}" subprocess.run(["docker-compose", "-f", filename, "down"]) - with open(filename) as f: - s = f.read() - with open(filename, 'w') as f: - s = s.replace(old_string, new_string) - f.write(s) + self.update_file(filename, f"image: {self.IMAGE}", "image: {$IMAGE}") + self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", self.FIXTURES_DIR, "{$DIR}") def create_topic(self, topic, topic_config): - command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_TOPICS}", "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [constants.KAFKA_TOPICS, "--list"] + check_command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_TOPICS}", "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command) if topic in output.decode("utf-8"): return True return False - + def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_PRODUCER}", "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'", "--timeout", f"{constants.CLIENT_TIMEOUT}"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) - + subprocess.run(["bash", "-c", " ".join(command)]) + def consume_message(self, topic, consumer_config): - command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_CONSUMER}", "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1", "--timeout-ms", f"{constants.CLIENT_TIMEOUT}"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip() - + def get_metrics(self, jmx_tool_config): - command = [constants.KAFKA_RUN_CLASS, constants.JMX_TOOL] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_RUN_CLASS}", constants.JMX_TOOL] command.extend(jmx_tool_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip().split() - + def broker_metrics_flow(self): print(f"Running {constants.BROKER_METRICS_TESTS}") errors = [] @@ -104,7 +102,7 @@ def broker_metrics_flow(self): except AssertionError as e: errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) return errors - + metrics_after_message = self.get_metrics(jmx_tool_config) try: self.assertEqual(len(metrics_before_message), 2) @@ -125,38 +123,38 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}"])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", constants.SSL_CLIENT_CONFIG] + "--producer.config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}"] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", constants.SSL_CLIENT_CONFIG, + "--consumer.config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", ] message = self.consume_message(topic, consumer_config) try: self.assertEqual(message, "key:message") except AssertionError as e: errors.append(test_error_prefix + str(e)) - + return errors - + def broker_restart_flow(self): print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] - + try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors - + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") @@ -171,7 +169,7 @@ def broker_restart_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - + return errors def execute(self): @@ -196,35 +194,45 @@ def execute(self): except Exception as e: print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) - + self.assertEqual(total_errors, []) -class DockerSanityTestJVM(DockerSanityTest): +class DockerSanityTestJVMCombinedMode(DockerSanityTest): def setUp(self) -> None: - self.start_compose(constants.JVM_COMPOSE) + self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMBINED_MODE_COMPOSE}") def tearDown(self) -> None: - self.destroy_compose(constants.JVM_COMPOSE) + self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMBINED_MODE_COMPOSE}") def test_bed(self): self.execute() -def run_tests(image, mode): +class DockerSanityTestJVMIsolatedMode(DockerSanityTest): + def setUp(self) -> None: + self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_ISOLATED_COMPOSE}") + def tearDown(self) -> None: + self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_ISOLATED_COMPOSE}") + def test_bed(self): + self.execute() + +def run_tests(image, mode, fixtures_dir): DockerSanityTest.IMAGE = image + DockerSanityTest.FIXTURES_DIR = fixtures_dir test_classes_to_run = [] if mode == "jvm": - test_classes_to_run = [DockerSanityTestJVM] - + test_classes_to_run = [DockerSanityTestJVMCombinedMode, DockerSanityTestJVMIsolatedMode] + loader = unittest.TestLoader() suites_list = [] for test_class in test_classes_to_run: suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) - big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report_{mode}.html", "w") + combined_suite = unittest.TestSuite(suites_list) + cur_directory = os.path.dirname(os.path.realpath(__file__)) + outfile = open(f"{cur_directory}/report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( - stream=outfile, - title='Test Report', - description='This demonstrates the report output.' - ) - result = runner.run(big_suite) + stream=outfile, + title='Test Report', + description='This demonstrates the report output.' + ) + result = runner.run(combined_suite) return result.failure_count diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index cb5d420c76e6a..781f058650ec4 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -13,21 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 +advertised.listeners=PLAINTEXT://localhost:19093,SSL://localhost:9094 controller.listener.names=CONTROLLER -controller.quorum.voters=3@broker-ssl-file-input:29093 group.initial.rebalance.delay.ms=0 -inter.broker.listener.name=BROKER -listener.name.internal.ssl.endpoint.identification.algorithm= -listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT -listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 +inter.broker.listener.name=PLAINTEXT +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT log.dirs=/tmp/kraft-combined-logs offsets.topic.replication.factor=1 -process.roles=invalid,value +process.roles=to be overridden ssl.client.auth=required -ssl.endpoint.identification.algorithm= ssl.key.password=abcdefgh -ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks +ssl.keystore.location=/etc/kafka/secrets/kafka02.keystore.jks ssl.keystore.password=abcdefgh ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh diff --git a/docker/test/fixtures/jvm/combined/docker-compose.yml b/docker/test/fixtures/jvm/combined/docker-compose.yml new file mode 100644 index 0000000000000..b62f7a84529f3 --- /dev/null +++ b/docker/test/fixtures/jvm/combined/docker-compose.yml @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + broker1: + image: {$IMAGE} + hostname: broker1 + container_name: broker1 + ports: + - "9092:9092" + - "9101:9101" + - "19091:19091" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SSL://localhost:19091' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker1:29093' + KAFKA_LISTENERS: 'CONTROLLER://broker1:29093,PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:19091' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + + broker2: + image: {$IMAGE} + hostname: broker2 + container_name: broker2 + ports: + - "9093:9093" + - "19092:19092" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:9093" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker2:29093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker2:29093" + KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + + broker3: + image: {$IMAGE} + hostname: broker3 + container_name: broker3 + ports: + - "19093:19093" + - "9094:9094" + volumes: + - ../../secrets:/etc/kafka/secrets + - ../../file-input:/mnt/shared/config + environment: + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set properties absent from the file + KAFKA_NODE_ID: 3 + KAFKA_CONTROLLER_QUORUM_VOTERS: '3@broker3:29093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094,CONTROLLER://broker3:29093' + # Override an existing property + KAFKA_PROCESS_ROLES: 'broker,controller' diff --git a/docker/test/fixtures/jvm/isolated/docker-compose.yml b/docker/test/fixtures/jvm/isolated/docker-compose.yml new file mode 100644 index 0000000000000..a3bf6de105790 --- /dev/null +++ b/docker/test/fixtures/jvm/isolated/docker-compose.yml @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + controller1: + image: {$IMAGE} + hostname: controller1 + container_name: controller1 + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller1:29093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller2: + image: {$IMAGE} + hostname: controller2 + container_name: controller2 + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller2:39093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller3: + image: {$IMAGE} + hostname: controller3 + container_name: controller3 + environment: + KAFKA_NODE_ID: 3 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller3:49093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + broker1: + image: {$IMAGE} + hostname: broker1 + container_name: broker1 + ports: + - "9092:9092" + - "19091:19091" + - "9101:9101" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 4 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SSL:SSL,PLAINTEXT:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SSL://localhost:19091' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:19091' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + depends_on: + - controller1 + - controller2 + - controller3 + + broker2: + image: {$IMAGE} + hostname: broker2 + container_name: broker2 + ports: + - "9093:9093" + - "19092:19092" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 5 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:9093" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093" + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + depends_on: + - controller1 + - controller2 + - controller3 + + broker3: + image: {$IMAGE} + hostname: broker3 + container_name: broker3 + ports: + - "19093:19093" + - "9094:9094" + volumes: + - ../../secrets:/etc/kafka/secrets + - ../../file-input:/mnt/shared/config + environment: + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set a property absent from the file + KAFKA_NODE_ID: 6 + # Override existing properties + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094" + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + depends_on: + - controller1 + - controller2 + - controller3 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index df1b20f259e3a..ad4f886555106 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -14,11 +14,10 @@ # limitations under the License. security.protocol=SSL -ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks +ssl.truststore.location={$DIR}/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh -ssl.keystore.location=./test/fixtures/secrets/client.keystore.jks +ssl.keystore.location={$DIR}/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh -ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1 ssl.client.auth=required -ssl.endpoint.identification.algorithm= \ No newline at end of file +ssl.endpoint.identification.algorithm= diff --git a/docker/test/fixtures/secrets/kafka_keystore_creds b/docker/test/fixtures/secrets/kafka_keystore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_keystore_creds +++ b/docker/test/fixtures/secrets/kafka_keystore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/test/fixtures/secrets/kafka_ssl_key_creds b/docker/test/fixtures/secrets/kafka_ssl_key_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_ssl_key_creds +++ b/docker/test/fixtures/secrets/kafka_ssl_key_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/test/fixtures/secrets/kafka_truststore_creds b/docker/test/fixtures/secrets/kafka_truststore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_truststore_creds +++ b/docker/test/fixtures/secrets/kafka_truststore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/test/report.html b/docker/test/report.html new file mode 100644 index 0000000000000..0321ea7990488 --- /dev/null +++ b/docker/test/report.html @@ -0,0 +1,278 @@ + + + + + Test Report + + + + + + + + + +
    +

    Test Report

    +

    Start Time: 2023-10-31 13:29:53

    +

    Duration: 0:01:21.112730

    +

    Status: Pass 1

    + +

    This demonstrates the report output.

    +
    + + + +

    Show +Summary +Failed +All +

    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Test Group/Test caseCountPassFailErrorView
    DockerSanityTestKraftMode1100Detail
    test_bed
    		 + + + + pass + + + + +
    Total1100 
    + +
     
    + + + diff --git a/docs/configuration.html b/docs/configuration.html index 03038223b2189..7bcb097b94462 100644 --- a/docs/configuration.html +++ b/docs/configuration.html @@ -16,7 +16,7 @@ --> - -
    -

    Test Report

    -

    Start Time: 2023-11-23 23:36:44

    -

    Duration: 0:00:41.693944

    -

    Status: Pass 1

    - -

    This demonstrates the report output.

    -
    - - - -

    Show -Summary -Failed -All -

    - -------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Test Group/Test caseCountPassFailErrorView
    test.docker_sanity_test.DockerSanityTestJVM1100Detail
    test_bed
    		 - - - - pass - - - - -
    Total1100 
    - -
     
    - - - diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties deleted file mode 100644 index 7621ac44f42b8..0000000000000 --- a/docker/resources/common-scripts/log4j.properties +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - - -log4j.logger.kafka=INFO -log4j.logger.kafka.authorizer.logger=WARN -log4j.logger.kafka.controller=TRACE -log4j.logger.kafka.log.LogCleaner=INFO -log4j.logger.kafka.network.RequestChannel$=WARN -log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG -log4j.logger.kafka.request.logger=WARN -log4j.logger.state.change.logger=TRACE diff --git a/docker/resources/common-scripts/server.properties b/docker/resources/common-scripts/server.properties deleted file mode 100644 index caa597ad96df5..0000000000000 --- a/docker/resources/common-scripts/server.properties +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 -controller.listener.names=CONTROLLER -controller.quorum.voters=1@localhost:29093 -group.initial.rebalance.delay.ms=0 -inter.broker.listener.name=PLAINTEXT -listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT -listeners=PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092 -log.dirs=/tmp/kraft-combined-logs -node.id=1 -offsets.topic.replication.factor=1 -process.roles=broker,controller -transaction.state.log.min.isr=1 -transaction.state.log.replication.factor=1 diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties deleted file mode 100644 index 84f0e09405ddd..0000000000000 --- a/docker/resources/common-scripts/tools-log4j.properties +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=WARN, stderr - -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n -log4j.appender.stderr.Target=System.err diff --git a/docker/resources/ub/go.mod b/docker/resources/ub/go.mod deleted file mode 100644 index 3ec7b827348d0..0000000000000 --- a/docker/resources/ub/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module ub - -go 1.19 - -require ( - github.com/spf13/cobra v1.7.0 - golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c - golang.org/x/sys v0.7.0 -) - -require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect -) diff --git a/docker/resources/ub/go.sum b/docker/resources/ub/go.sum deleted file mode 100644 index 5f20e19b6de1e..0000000000000 --- a/docker/resources/ub/go.sum +++ /dev/null @@ -1,14 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c h1:HDdYQYKOkvJT/Plb5HwJJywTVyUnIctjQm6XSnZ/0CY= -golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docker/resources/ub/testResources/sampleFile b/docker/resources/ub/testResources/sampleFile deleted file mode 100755 index 91eacc92e8be9..0000000000000 --- a/docker/resources/ub/testResources/sampleFile +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleFile2 b/docker/resources/ub/testResources/sampleFile2 deleted file mode 100755 index 91eacc92e8be9..0000000000000 --- a/docker/resources/ub/testResources/sampleFile2 +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template deleted file mode 100644 index 8bc1f5e3dbd4d..0000000000000 --- a/docker/resources/ub/testResources/sampleLog4j.template +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout - -{{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} -{{ range $k, $v := splitToMapDefaults "," "" $loggers}} -log4j.logger.{{ $k }}={{ $v -}} -{{ end }} \ No newline at end of file diff --git a/docker/resources/ub/ub.go b/docker/resources/ub/ub.go deleted file mode 100644 index 6c84fd2c35c90..0000000000000 --- a/docker/resources/ub/ub.go +++ /dev/null @@ -1,323 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "os/signal" - pt "path" - "regexp" - "sort" - "strings" - "text/template" - - "github.com/spf13/cobra" - "golang.org/x/exp/slices" - "golang.org/x/sys/unix" -) - -type ConfigSpec struct { - Prefixes map[string]bool `json:"prefixes"` - Excludes []string `json:"excludes"` - Renamed map[string]string `json:"renamed"` - Defaults map[string]string `json:"defaults"` - ExcludeWithPrefix string `json:"excludeWithPrefix"` -} - -var ( - re = regexp.MustCompile("[^_]_[^_]") - - ensureCmd = &cobra.Command{ - Use: "ensure ", - Short: "checks if environment variable is set or not", - Args: cobra.ExactArgs(1), - RunE: runEnsureCmd, - } - - pathCmd = &cobra.Command{ - Use: "path ", - Short: "checks if an operation is permitted on a file", - Args: cobra.ExactArgs(2), - RunE: runPathCmd, - } - - renderTemplateCmd = &cobra.Command{ - Use: "render-template ", - Short: "renders template to stdout", - Args: cobra.ExactArgs(1), - RunE: runRenderTemplateCmd, - } - - renderPropertiesCmd = &cobra.Command{ - Use: "render-properties ", - Short: "creates and renders properties to stdout using the json config spec.", - Args: cobra.ExactArgs(1), - RunE: runRenderPropertiesCmd, - } -) - -func ensure(envVar string) bool { - _, found := os.LookupEnv(envVar) - return found -} - -func path(filePath string, operation string) (bool, error) { - switch operation { - - case "readable": - err := unix.Access(filePath, unix.R_OK) - if err != nil { - return false, err - } - return true, nil - case "executable": - info, err := os.Stat(filePath) - if err != nil { - err = fmt.Errorf("error checking executable status of file %q: %w", filePath, err) - return false, err - } - return info.Mode()&0111 != 0, nil //check whether file is executable by anyone, use 0100 to check for execution rights for owner - case "existence": - if _, err := os.Stat(filePath); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil - case "writable": - err := unix.Access(filePath, unix.W_OK) - if err != nil { - return false, err - } - return true, nil - default: - err := fmt.Errorf("unknown operation %q", operation) - return false, err - } -} - -func renderTemplate(templateFilePath string) error { - funcs := template.FuncMap{ - "getEnv": getEnvOrDefault, - "splitToMapDefaults": splitToMapDefaults, - } - t, err := template.New(pt.Base(templateFilePath)).Funcs(funcs).ParseFiles(templateFilePath) - if err != nil { - err = fmt.Errorf("error %q: %w", templateFilePath, err) - return err - } - return buildTemplate(os.Stdout, *t) -} - -func buildTemplate(writer io.Writer, template template.Template) error { - err := template.Execute(writer, GetEnvironment()) - if err != nil { - err = fmt.Errorf("error building template file : %w", err) - return err - } - return nil -} - -func renderConfig(writer io.Writer, configSpec ConfigSpec) error { - return writeConfig(writer, buildProperties(configSpec, GetEnvironment())) -} - -// ConvertKey Converts an environment variable name to a property-name according to the following rules: -// - a single underscore (_) is replaced with a . -// - a double underscore (__) is replaced with a single underscore -// - a triple underscore (___) is replaced with a dash -// Moreover, the whole string is converted to lower-case. -// The behavior of sequences of four or more underscores is undefined. -func ConvertKey(key string) string { - singleReplaced := re.ReplaceAllStringFunc(key, replaceUnderscores) - singleTripleReplaced := strings.ReplaceAll(singleReplaced, "___", "-") - return strings.ToLower(strings.ReplaceAll(singleTripleReplaced, "__", "_")) -} - -// replaceUnderscores replaces every underscore '_' by a dot '.' -func replaceUnderscores(s string) string { - return strings.ReplaceAll(s, "_", ".") -} - -// ListToMap splits each and entry of the kvList argument at '=' into a key/value pair and returns a map of all the k/v pair thus obtained. -// this method will only consider values in the list formatted as key=value -func ListToMap(kvList []string) map[string]string { - m := make(map[string]string, len(kvList)) - for _, l := range kvList { - parts := strings.Split(l, "=") - if len(parts) == 2 { - m[parts[0]] = parts[1] - } - } - return m -} - -func splitToMapDefaults(separator string, defaultValues string, value string) map[string]string { - values := KvStringToMap(defaultValues, separator) - for k, v := range KvStringToMap(value, separator) { - values[k] = v - } - return values -} - -func KvStringToMap(kvString string, sep string) map[string]string { - return ListToMap(strings.Split(kvString, sep)) -} - -// GetEnvironment returns the current environment as a map. -func GetEnvironment() map[string]string { - return ListToMap(os.Environ()) -} - -// buildProperties creates a map suitable to be output as Java properties from a ConfigSpec and a map representing an environment. -func buildProperties(spec ConfigSpec, environment map[string]string) map[string]string { - config := make(map[string]string) - for key, value := range spec.Defaults { - config[key] = value - } - - for envKey, envValue := range environment { - if newKey, found := spec.Renamed[envKey]; found { - config[newKey] = envValue - } else { - if !slices.Contains(spec.Excludes, envKey) && !(len(spec.ExcludeWithPrefix) > 0 && strings.HasPrefix(envKey, spec.ExcludeWithPrefix)) { - for prefix, keep := range spec.Prefixes { - if strings.HasPrefix(envKey, prefix) { - var effectiveKey string - if keep { - effectiveKey = envKey - } else { - effectiveKey = envKey[len(prefix)+1:] - } - config[ConvertKey(effectiveKey)] = envValue - } - } - } - } - } - return config -} - -func writeConfig(writer io.Writer, config map[string]string) error { - // Go randomizes iterations over map by design. We sort properties by name to ease debugging: - sortedNames := make([]string, 0, len(config)) - for name := range config { - sortedNames = append(sortedNames, name) - } - sort.Strings(sortedNames) - for _, n := range sortedNames { - _, err := fmt.Fprintf(writer, "%s=%s\n", n, config[n]) - if err != nil { - err = fmt.Errorf("error printing configs: %w", err) - return err - } - } - return nil -} - -func loadConfigSpec(path string) (ConfigSpec, error) { - var spec ConfigSpec - bytes, err := os.ReadFile(path) - if err != nil { - err = fmt.Errorf("error reading from json file %q : %w", path, err) - return spec, err - } - - errParse := json.Unmarshal(bytes, &spec) - if errParse != nil { - err = fmt.Errorf("error parsing json file %q : %w", path, errParse) - return spec, err - } - return spec, nil -} - -func getEnvOrDefault(envVar string, defaultValue string) string { - val := os.Getenv(envVar) - if len(val) == 0 { - return defaultValue - } - return val -} - -func runEnsureCmd(_ *cobra.Command, args []string) error { - success := ensure(args[0]) - if !success { - err := fmt.Errorf("environment variable %q is not set", args[0]) - return err - } - return nil -} - -func runPathCmd(_ *cobra.Command, args []string) error { - success, err := path(args[0], args[1]) - if err != nil { - err = fmt.Errorf("error in checking operation %q on file %q: %w", args[1], args[0], err) - return err - } - if !success { - err = fmt.Errorf("operation %q on file %q is unsuccessful", args[1], args[0]) - return err - } - return nil -} - -func runRenderTemplateCmd(_ *cobra.Command, args []string) error { - err := renderTemplate(args[0]) - if err != nil { - err = fmt.Errorf("error in rendering template %q: %w", args[0], err) - return err - } - return nil -} - -func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { - configSpec, err := loadConfigSpec(args[0]) - if err != nil { - err = fmt.Errorf("error in loading config from file %q: %w", args[0], err) - return err - } - err = renderConfig(os.Stdout, configSpec) - if err != nil { - err = fmt.Errorf("error in building properties from file %q: %w", args[0], err) - return err - } - return nil -} - -func main() { - rootCmd := &cobra.Command{ - Use: "ub", - Short: "utility commands for cp docker images", - Run: func(cmd *cobra.Command, args []string) {}, - } - - rootCmd.AddCommand(pathCmd) - rootCmd.AddCommand(ensureCmd) - rootCmd.AddCommand(renderTemplateCmd) - rootCmd.AddCommand(renderPropertiesCmd) - - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - defer cancel() - if err := rootCmd.ExecuteContext(ctx); err != nil { - fmt.Fprintf(os.Stderr, "error in executing the command: %s", err) - os.Exit(1) - } -} diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go deleted file mode 100644 index 9d09c5c413e27..0000000000000 --- a/docker/resources/ub/ub_test.go +++ /dev/null @@ -1,355 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "reflect" - "testing" -) - -func assertEqual(a string, b string, t *testing.T) { - if a != b { - t.Error(a + " != " + b) - } -} - -func Test_ensure(t *testing.T) { - type args struct { - envVar string - } - err := os.Setenv("ENV_VAR", "value") - if err != nil { - t.Fatal("Unable to set ENV_VAR for the test") - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "should exist", - args: args{ - envVar: "ENV_VAR", - }, - want: true, - }, - { - name: "should not exist", - args: args{ - envVar: "RANDOM_ENV_VAR", - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := ensure(tt.args.envVar); got != tt.want { - t.Errorf("ensure() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_path(t *testing.T) { - type args struct { - filePath string - operation string - } - const ( - sampleFile = "testResources/sampleFile" - sampleFile2 = "testResources/sampleFile2" - fileDoesNotExist = "testResources/sampleFile3" - ) - err := os.Chmod(sampleFile, 0777) - if err != nil { - t.Error("Unable to set permissions for the file") - } - err = os.Chmod(sampleFile2, 0000) - if err != nil { - t.Error("Unable to set permissions for the file") - } - tests := []struct { - name string - args args - want bool - wantErr bool - }{ - { - name: "file readable", - args: args{filePath: sampleFile, - operation: "readable"}, - want: true, - wantErr: false, - }, - { - name: "file writable", - args: args{filePath: sampleFile, - operation: "writable"}, - want: true, - wantErr: false, - }, - { - name: "file executable", - args: args{filePath: sampleFile, - operation: "executable"}, - want: true, - wantErr: false, - }, - { - name: "file existence", - args: args{filePath: sampleFile, - operation: "existence"}, - want: true, - wantErr: false, - }, - { - name: "file not readable", - args: args{filePath: sampleFile2, - operation: "readable"}, - want: false, - wantErr: true, - }, - { - name: "file not writable", - args: args{filePath: sampleFile2, - operation: "writable"}, - want: false, - wantErr: true, - }, - { - name: "file not executable", - args: args{filePath: sampleFile2, - operation: "executable"}, - want: false, - wantErr: false, - }, - { - name: "file does not exist", - args: args{filePath: fileDoesNotExist, - operation: "existence"}, - want: false, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := path(tt.args.filePath, tt.args.operation) - if (err != nil) != tt.wantErr { - t.Errorf("path() error = %v, wantErr %v", err, tt.wantErr) - } - if got != tt.want { - t.Errorf("path() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_renderTemplate(t *testing.T) { - type args struct { - templateFilePath string - } - const ( - fileExistsAndRenderable = "testResources/sampleLog4j.template" - fileDoesNotExist = "testResources/RandomFileName" - ) - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "render template success", - args: args{templateFilePath: fileExistsAndRenderable}, - wantErr: false, - }, - { - name: "render template failure ", - args: args{templateFilePath: fileDoesNotExist}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := renderTemplate(tt.args.templateFilePath); (err != nil) != tt.wantErr { - t.Errorf("renderTemplate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} -func Test_convertKey(t *testing.T) { - type args struct { - key string - } - tests := []struct { - name string - args args - wantString string - }{ - { - name: "Capitals", - args: args{key: "KEY"}, - wantString: "key", - }, - { - name: "Capitals with underscore", - args: args{key: "KEY_FOO"}, - wantString: "key.foo", - }, - { - name: "Capitals with double underscore", - args: args{key: "KEY__UNDERSCORE"}, - wantString: "key_underscore", - }, - { - name: "Capitals with double and single underscore", - args: args{key: "KEY_WITH__UNDERSCORE_AND__MORE"}, - wantString: "key.with_underscore.and_more", - }, - { - name: "Capitals with triple underscore", - args: args{key: "KEY___DASH"}, - wantString: "key-dash", - }, - { - name: "capitals with double,triple and single underscore", - args: args{key: "KEY_WITH___DASH_AND___MORE__UNDERSCORE"}, - wantString: "key.with-dash.and-more_underscore", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if result := ConvertKey(tt.args.key); result != tt.wantString { - t.Errorf("ConvertKey() result = %v, wantStr %v", result, tt.wantString) - } - }) - } -} - -func Test_buildProperties(t *testing.T) { - type args struct { - spec ConfigSpec - environment map[string]string - } - tests := []struct { - name string - args args - want map[string]string - }{ - { - name: "only defaults", - args: args{ - spec: ConfigSpec{ - Defaults: map[string]string{ - "default.property.key": "default.property.value", - "bootstrap.servers": "unknown", - }, - }, - environment: map[string]string{ - "PATH": "thePath", - "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "KAFKA_IGNORED": "ignored", - "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", - }, - }, - want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value"}, - }, - { - name: "server properties", - args: args{ - spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false}, - Excludes: []string{"KAFKA_IGNORED"}, - Renamed: map[string]string{}, - Defaults: map[string]string{ - "default.property.key": "default.property.value", - "bootstrap.servers": "unknown", - }, - ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", - }, - environment: map[string]string{ - "PATH": "thePath", - "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "KAFKA_IGNORED": "ignored", - "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", - }, - }, - want: map[string]string{"bootstrap.servers": "localhost:9092", "default.property.key": "default.property.value"}, - }, - { - name: "kafka properties", - args: args{ - spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false}, - Excludes: []string{"KAFKA_IGNORED"}, - Renamed: map[string]string{}, - Defaults: map[string]string{ - "default.property.key": "default.property.value", - "bootstrap.servers": "unknown", - }, - ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", - }, - environment: map[string]string{ - "KAFKA_FOO": "foo", - "KAFKA_FOO_BAR": "bar", - "KAFKA_IGNORED": "ignored", - "KAFKA_WITH__UNDERSCORE": "with underscore", - "KAFKA_WITH__UNDERSCORE_AND_MORE": "with underscore and more", - "KAFKA_WITH___DASH": "with dash", - "KAFKA_WITH___DASH_AND_MORE": "with dash and more", - }, - }, - want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value", "foo": "foo", "foo.bar": "bar", "with-dash": "with dash", "with-dash.and.more": "with dash and more", "with_underscore": "with underscore", "with_underscore.and.more": "with underscore and more"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := buildProperties(tt.args.spec, tt.args.environment); !reflect.DeepEqual(got, tt.want) { - t.Errorf("buildProperties() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_splitToMapDefaults(t *testing.T) { - type args struct { - separator string - defaultValues string - value string - } - tests := []struct { - name string - args args - want map[string]string - }{ - { - name: "split to default", - args: args{ - separator: ",", - defaultValues: "kafka=INFO,kafka.producer.async.DefaultEventHandler=DEBUG,state.change.logger=TRACE", - value: "kafka.producer.async.DefaultEventHandler=ERROR,kafka.request.logger=WARN", - }, - want: map[string]string{"kafka": "INFO", "kafka.producer.async.DefaultEventHandler": "ERROR", "kafka.request.logger": "WARN", "state.change.logger": "TRACE"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := splitToMapDefaults(tt.args.separator, tt.args.defaultValues, tt.args.value); !reflect.DeepEqual(got, tt.want) { - t.Errorf("splitToMapDefaults() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml deleted file mode 100644 index 64a89967fdf28..0000000000000 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ /dev/null @@ -1,89 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -version: '2' -services: - broker: - image: {$IMAGE} - hostname: broker - container_name: broker - ports: - - "9092:9092" - - "9101:9101" - environment: - KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - - broker-ssl: - image: {$IMAGE} - hostname: broker-ssl - container_name: broker-ssl - ports: - - "9093:9093" - volumes: - - ../secrets:/etc/kafka/secrets - environment: - KAFKA_NODE_ID: 2 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT" - KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker-ssl:29093' - KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl:29093" - KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER" - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" - KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" - KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" - KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - broker-ssl-file-input: - image: {$IMAGE} - hostname: broker-ssl-file-input - container_name: broker-ssl-file-input - ports: - - "9094:9093" - volumes: - - ../secrets:/etc/kafka/secrets - - ../file-input:/mnt/shared/config - environment: - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - # Set a property absent from the file - KAFKA_NODE_ID: 3 - # Override an existing property - KAFKA_PROCESS_ROLES: 'broker,controller' \ No newline at end of file diff --git a/docker/test/report.html b/docker/test/report.html deleted file mode 100644 index 0321ea7990488..0000000000000 --- a/docker/test/report.html +++ /dev/null @@ -1,278 +0,0 @@ - - - - - Test Report - - - - - - - - - -
    -

    Test Report

    -

    Start Time: 2023-10-31 13:29:53

    -

    Duration: 0:01:21.112730

    -

    Status: Pass 1

    - -

    This demonstrates the report output.

    -
    - - - -

    Show -Summary -Failed -All -

    - -------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Test Group/Test caseCountPassFailErrorView
    DockerSanityTestKraftMode1100Detail
    test_bed
    		 - - - - pass - - - - -
    Total1100 
    - -
     
    - - -