From afb07feac939fc8d4f8af53228c8f56a965260be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 08:41:14 +0000 Subject: [PATCH 01/11] Bump mockito-core from 4.11.0 to 5.0.0 Bumps [mockito-core](https://github.com/mockito/mockito) from 4.11.0 to 5.0.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v4.11.0...v5.0.0) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d0750f..aed9485 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ org.mockito mockito-core - 4.11.0 + 5.0.0 test From 18862d09366c3bbfea742d008683a2326f6ec2ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 09:00:37 +0000 Subject: [PATCH 02/11] Bump mockito-core from 5.0.0 to 5.1.1 Bumps [mockito-core](https://github.com/mockito/mockito) from 5.0.0 to 5.1.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v5.0.0...v5.1.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ee4fa2..343df62 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ org.mockito mockito-core - 5.0.0 + 5.1.1 test From 112c6e523b93e60dabdf860257e5e48ce38432fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 09:56:35 +0000 Subject: [PATCH 03/11] Bump maven-assembly-plugin from 3.4.2 to 3.5.0 Bumps [maven-assembly-plugin](https://github.com/apache/maven-assembly-plugin) from 3.4.2 to 3.5.0. - [Release notes](https://github.com/apache/maven-assembly-plugin/releases) - [Commits](https://github.com/apache/maven-assembly-plugin/compare/maven-assembly-plugin-3.4.2...maven-assembly-plugin-3.5.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-assembly-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 343df62..bd059f7 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.4.2 + 3.5.0 package From 028c728a80ea87ba127f192baf42a51b3af99c38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 09:56:40 +0000 Subject: [PATCH 04/11] Bump maven-compiler-plugin from 3.10.1 to 3.11.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.10.1 to 3.11.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.10.1...maven-compiler-plugin-3.11.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 343df62..01079cd 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.10.1 + 3.11.0 11 11 From 0a41c9f9f56f9d34bebead2beb113960099facc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Apr 2023 09:56:45 +0000 Subject: [PATCH 05/11] Bump mockito-core from 5.1.1 to 5.2.0 Bumps [mockito-core](https://github.com/mockito/mockito) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ada5958..9d75a18 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ org.mockito mockito-core - 5.1.1 + 5.2.0 test From 0ad2cde23f61f6d908f2506da2f9feaf285dfcf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Apr 2023 09:56:48 +0000 Subject: [PATCH 06/11] Bump maven-surefire-plugin from 2.22.2 to 3.0.0 Bumps [maven-surefire-plugin](https://github.com/apache/maven-surefire) from 2.22.2 to 3.0.0. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-2.22.2...surefire-3.0.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-surefire-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ada5958..69890ed 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.0.0 From 01ab567c7271c26a341b478a1e79d3f394bc3298 Mon Sep 17 00:00:00 2001 From: Jaakko Malkki <32361480+mjaakko@users.noreply.github.com> Date: Thu, 20 Apr 2023 14:12:16 +0300 Subject: [PATCH 07/11] Update README --- README.md | 60 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index b359440..a4524a9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -[![Build Status](https://travis-ci.org/HSLdevcom/mqtt-pulsar-gateway.svg?branch=master)](https://travis-ci.org/HSLdevcom/mqtt-pulsar-gateway) +## mqtt-pulsar-gateway [![Test and create Docker image](https://github.com/HSLdevcom/mqtt-pulsar-gateway/actions/workflows/test-and-build.yml/badge.svg)](https://github.com/HSLdevcom/mqtt-pulsar-gateway/actions/workflows/test-and-build.yml) -## Description +Application for reading data from MQTT topic and feeding it into Pulsar topic. This application doesn't care about the message payload, it just transfers the bytes. -Application for reading data from MQTT topic and feeding it into Pulsar topic. -This application doesn't care about the payload, it just transfers the bytes. +This project is part of [transitdata pipeline](https://github.com/HSLdevcom/transitdata), but it can also be used in other applications. ## Building @@ -11,36 +10,43 @@ This application doesn't care about the payload, it just transfers the bytes. This project depends on [transitdata-common](https://github.com/HSLdevcom/transitdata-common) project. -Either use released versions from public maven repository or build your own and install to local maven repository: - - ```cd transitdata-common && mvn install``` - ### Locally -- ```mvn compile``` -- ```mvn package``` +- `mvn compile` +- `mvn package` ### Docker image - Run [this script](build-image.sh) to build the Docker image - ## Running -Requirements: -- Pulsar Cluster - - By default uses localhost, override host in PULSAR_HOST if needed. - - Tip: f.ex if running inside Docker in OSX set `PULSAR_HOST=host.docker.internal` to connect to the parent machine - - You can use [this script](https://github.com/HSLdevcom/transitdata/blob/master/bin/pulsar/pulsar-up.sh) to launch it as Docker container -- Connection to an external MQTT server. - - Configure username and password via files - - Set filepath for username via env variable FILEPATH_USERNAME_SECRET, default is `/run/secrets/mqtt_broker_username` - - Set filepath for password via env variable FILEPATH_PASSWORD_SECRET, default is `/run/secrets/mqtt_broker_password` - - Mandatory: Set mqtt-topic via env variable MQTT_TOPIC - - Remember to use a unique MQTT client-id's if you have multiple instances connected to a single broker. - -All other configuration options are configured in the [config file](src/main/resources/environment.conf) -which can also be configured externally via env variable CONFIG_PATH - -Launch Docker container with +### Dependencies -```docker-compose -f compose-config-file.yml up ``` +* Pulsar + * You can use [this script](https://github.com/HSLdevcom/transitdata/blob/master/bin/pulsar/pulsar-up.sh) to launch it as Docker container +* Connection to an MQTT broker + +### Environment variables + +#### MQTT + +* `MQTT_CREDENTIALS_REQUIRED`: whether the broker needs credentials +* `FILEPATH_USERNAME_SECRET`: path to the file containing the username, default is `/run/secrets/mqtt_broker_username` +* `FILEPATH_PASSWORD_SECRET`: path to the file containing the password, default is `/run/secrets/mqtt_broker_password` +* `MQTT_BROKER_HOST`: MQTT broker URL +* `MQTT_TOPIC`: MQTT topic to subscribe +* `MQTT_QOS`: MQTT QoS +* `MQTT_MAX_INFLIGHT`: maximum MQTT messages inflight +* `MQTT_CLEAN_SESSION`: whether to open a clean MQTT session, i.e. if true, unacked messages are not redelivered +* `MQTT_CLIENT_ID`: client ID to use when subscribing to the MQTT topic. Remember to use unique client ID if there is more than one subscription +* `MQTT_ADD_RANDOM_TO_CLIENT_ID`: whether to append random string to the client ID +* `MQTT_KEEP_ALIVE_INTERVAL`: interval in seconds for MQTT keep-alive +* `MQTT_MANUAL_ACK`: whether to ack messages to the MQTT broker only after they have been successfully sent to Pulsar. Setting this to true might cause performance problems with high message rate + +#### Other + +* `IN_FLIGHT_ALERT_THRESHOLD`: send log message if the inflight message count is higher than this +* `MAX_MESSAGES_PER_SECOND`: maximum amount messages per second to be sent to Pulsar. If negative, no limit +* `MSG_MONITORING_INTERVAL`: send log message after this amount of messages has been processed +* `UNHEALTHY_MSG_SEND_INTERVAL_SECS`: consider the service unhealthy if no message has been sent to Pulsar in this period. If negative, no healthcheck for last sent message From dd62d3952845e9a8e129143a57540068b6c1c89f Mon Sep 17 00:00:00 2001 From: Jaakko Malkki <32361480+mjaakko@users.noreply.github.com> Date: Thu, 20 Apr 2023 14:15:41 +0300 Subject: [PATCH 08/11] Update version to 1.2.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d75a18..bb28949 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 fi.hsl.transitlog mqtt-pulsar-gateway - 1.2.0 + 1.2.1 jar From 15bb408f71464f3f4c0798dff88db97e9524b93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Fri, 5 Apr 2024 14:37:43 +0300 Subject: [PATCH 09/11] Update common version to 1.6.1-RC --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7e6e5e..23fd97a 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ UTF-8 11 11 - 1.4.4 + 1.6.1-RC 1.2.5 From 0f1cef72ada09b6273525b64f8aca432d86c31f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Fri, 5 Apr 2024 14:38:07 +0300 Subject: [PATCH 10/11] Add logging --- src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java b/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java index 7cc3940..f6cf679 100644 --- a/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java +++ b/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java @@ -88,6 +88,7 @@ public void processMessages() throws InterruptedException { public void sendMessageFromQueue() throws InterruptedException { QueuedMessage message = messageQueue.take(); + log.info("messageQueue size: " + messageQueue.size()); message.message.sendAsync() //Use timeout to more quickly detect if Pulsar connection is down @@ -114,6 +115,8 @@ public void sendMessageFromQueue() throws InterruptedException { } else { log.info("Currently messages in flight: {}", inFlight); } + } else { + log.info("++msgCounter ({}) % MSG_MONITORING_INTERVAL ({}) != 0", msgCounter, MSG_MONITORING_INTERVAL); } } From 116aefc4a94994403bed98c51e077e1c7c5ae717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Fri, 5 Apr 2024 14:56:26 +0300 Subject: [PATCH 11/11] Revert "Add logging" This reverts commit 0f1cef72ada09b6273525b64f8aca432d86c31f9. --- src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java b/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java index f6cf679..7cc3940 100644 --- a/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java +++ b/src/main/java/fi/hsl/pulsar/mqtt/MessageProcessor.java @@ -88,7 +88,6 @@ public void processMessages() throws InterruptedException { public void sendMessageFromQueue() throws InterruptedException { QueuedMessage message = messageQueue.take(); - log.info("messageQueue size: " + messageQueue.size()); message.message.sendAsync() //Use timeout to more quickly detect if Pulsar connection is down @@ -115,8 +114,6 @@ public void sendMessageFromQueue() throws InterruptedException { } else { log.info("Currently messages in flight: {}", inFlight); } - } else { - log.info("++msgCounter ({}) % MSG_MONITORING_INTERVAL ({}) != 0", msgCounter, MSG_MONITORING_INTERVAL); } }