From b1bc464e8faba3fd700b41be98bc74ffd9ada132 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 21 Oct 2025 17:30:00 +0200 Subject: [PATCH 1/3] add apache-kafka and update queue/messaging libs --- airflow/boil-config.toml | 2 +- .../constraints/3.0.6/constraints-python3.12.txt | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/airflow/boil-config.toml b/airflow/boil-config.toml index a787c09ac..54d5f62c2 100644 --- a/airflow/boil-config.toml +++ b/airflow/boil-config.toml @@ -58,6 +58,6 @@ s3fs-version = "2024.9.0" cyclonedx-bom-version = "6.0.0" tini-version = "0.19.0" uv-version = "0.7.8" -airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino" +airflow-extras = "amazon,apache-kafka,async,celery,cncf-kubernetes,common-messaging,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino" opa-auth-manager = "airflow-3" nodejs-version = "20" diff --git a/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt b/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt index 904a717b6..4e025f60c 100644 --- a/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt +++ b/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt @@ -109,7 +109,10 @@ apache-airflow-providers-apache-hdfs==4.10.2 apache-airflow-providers-apache-hive==9.1.2 apache-airflow-providers-apache-iceberg==1.3.2 apache-airflow-providers-apache-impala==1.7.2 -apache-airflow-providers-apache-kafka==1.10.2 +# Stackable patch: +# The original version 1.10.2 has sync/async issues when using event scheduling +# with the triggerer. The fix could not be traced to a specific change/commit. +apache-airflow-providers-apache-kafka==1.10.4 apache-airflow-providers-apache-kylin==3.9.2 apache-airflow-providers-apache-livy==4.4.2 apache-airflow-providers-apache-pig==4.7.2 @@ -125,7 +128,12 @@ apache-airflow-providers-cncf-kubernetes==10.7.0 apache-airflow-providers-cohere==1.5.3 apache-airflow-providers-common-compat==1.7.3 apache-airflow-providers-common-io==1.6.2 -apache-airflow-providers-common-messaging==1.0.5 +# Stackable patch: +# Update from previous version 1.0.5 due to refactoring and compatability with +# apache-airflow-providers-apache-kafka==1.10.4 (see above). +# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67 +# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67 +apache-airflow-providers-common-messaging==2.0.0 apache-airflow-providers-common-sql==1.27.5 apache-airflow-providers-databricks==7.7.1 apache-airflow-providers-datadog==3.9.2 From e90a59722b30feedf6debfdf27b751f2e64b008e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 22 Oct 2025 18:29:39 +0200 Subject: [PATCH 2/3] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f900dd94..b2edcc7b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. - hbase: Add `2.6.3` ([#1296]). - airflow,superset: Build from source ([#1304]). - kafka: Add updated JMX config `server.yaml` for KRaft ([#1308]). +- airflow: Add apache-kafka provider plus dependencies to image ([#1313]). ### Changed @@ -106,6 +107,7 @@ All notable changes to this project will be documented in this file. [#1308]: https://github.com/stackabletech/docker-images/pull/1308 [#1309]: https://github.com/stackabletech/docker-images/pull/1309 [#1311]: https://github.com/stackabletech/docker-images/pull/1311 +[#1313]: https://github.com/stackabletech/docker-images/pull/1313 ## [25.7.0] - 2025-07-23 From fa6923a8e002130808e0a41565ec99c8440df902 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:43:56 +0200 Subject: [PATCH 3/3] Update CHANGELOG.md Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2edcc7b8..eb2e72185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ All notable changes to this project will be documented in this file. - hbase: Add `2.6.3` ([#1296]). - airflow,superset: Build from source ([#1304]). - kafka: Add updated JMX config `server.yaml` for KRaft ([#1308]). -- airflow: Add apache-kafka provider plus dependencies to image ([#1313]). +- airflow: Add apache-kafka provider plus dependencies to `3.0.6` image ([#1313]). ### Changed