From 1cbb74d5f230990010a73408ea3811ac93760609 Mon Sep 17 00:00:00 2001
From: Michael Robinson <68482867+merobi-hub@users.noreply.github.com>
Date: Tue, 25 Mar 2025 12:20:04 -0400
Subject: [PATCH 01/24] Arcade demo on homepage (#3009)
* Arcade demo in about page of docs.
Signed-off-by: merobi-hub
* Move demo to homepage component.
Signed-off-by: merobi-hub
---------
Signed-off-by: merobi-hub
Signed-off-by: swar00pduthks
---
docs/src/components/HomepageFeatures/index.tsx | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/docs/src/components/HomepageFeatures/index.tsx b/docs/src/components/HomepageFeatures/index.tsx
index a1a1323292..54c7bc52fc 100644
--- a/docs/src/components/HomepageFeatures/index.tsx
+++ b/docs/src/components/HomepageFeatures/index.tsx
@@ -52,7 +52,7 @@ const featureList: FeatureItem[] = [
quality systems.
>
- }
+ },
]
type FeatureItem = {
@@ -90,6 +90,22 @@ export default function HomepageFeatures(): JSX.Element {
))}
+
+
+
Interactive Demo
+
+
+
+
+
);
}
From bac556f81260d09c50ae5add10567906042e7b30 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 27 Mar 2025 00:05:35 -0700
Subject: [PATCH 02/24] Update dependency
org.opensearch.client:opensearch-rest-client to v2.19.1 (#3005)
Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Signed-off-by: swar00pduthks
---
api/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/api/build.gradle b/api/build.gradle
index 3965a751e5..6eb2e60b05 100644
--- a/api/build.gradle
+++ b/api/build.gradle
@@ -51,7 +51,7 @@ dependencies {
implementation 'com.graphql-java:graphql-java:20.9'
implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
- implementation 'org.opensearch.client:opensearch-rest-client:2.17.1'
+ implementation 'org.opensearch.client:opensearch-rest-client:2.19.1'
implementation 'org.opensearch.client:opensearch-java:2.16.0'
testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"
From 14aa83084359bae5b0709968c29088cd95950429 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 27 Mar 2025 00:24:26 -0700
Subject: [PATCH 03/24] Update dependency org.opensearch.client:opensearch-java
to v2.22.0 (#3004)
Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Signed-off-by: swar00pduthks
---
api/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/api/build.gradle b/api/build.gradle
index 6eb2e60b05..b4effde600 100644
--- a/api/build.gradle
+++ b/api/build.gradle
@@ -52,7 +52,7 @@ dependencies {
implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
implementation 'org.opensearch.client:opensearch-rest-client:2.19.1'
- implementation 'org.opensearch.client:opensearch-java:2.16.0'
+ implementation 'org.opensearch.client:opensearch-java:2.22.0'
testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"
testImplementation "org.jdbi:jdbi3-testing:${jdbi3Version}"
From ae39ff37c7d30698dd0a084b4fd100794d608f87 Mon Sep 17 00:00:00 2001
From: Willy Lulciuc
Date: Thu, 27 Mar 2025 00:32:33 -0700
Subject: [PATCH 04/24] Prepare for release 0.51.1
Signed-off-by: Willy Lulciuc
Signed-off-by: swar00pduthks
---
.circleci/db-migration.sh | 2 +-
.env.example | 2 +-
chart/Chart.yaml | 2 +-
chart/values.yaml | 4 ++--
clients/java/README.md | 4 ++--
docker/up.sh | 4 ++--
gradle.properties | 2 +-
spec/openapi.yml | 2 +-
8 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh
index a3c21f35d3..5e7ed5275e 100755
--- a/.circleci/db-migration.sh
+++ b/.circleci/db-migration.sh
@@ -13,7 +13,7 @@
# Version of PostgreSQL
readonly POSTGRES_VERSION="14"
# Version of Marquez
-readonly MARQUEZ_VERSION=0.51.0
+readonly MARQUEZ_VERSION=0.51.1
# Build version of Marquez
readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
readonly POSTGRES_PORT=5432
diff --git a/.env.example b/.env.example
index b4fd292d19..933a3a26c7 100644
--- a/.env.example
+++ b/.env.example
@@ -3,4 +3,4 @@ API_ADMIN_PORT=5001
WEB_PORT=3000
POSTGRES_PORT=5432
SEARCH_PORT=9200
-TAG=0.51.0
+TAG=0.51.1
diff --git a/chart/Chart.yaml b/chart/Chart.yaml
index 5bd4dcd0a9..fe8e0d9ec6 100644
--- a/chart/Chart.yaml
+++ b/chart/Chart.yaml
@@ -29,4 +29,4 @@ name: marquez
sources:
- https://github.com/MarquezProject/marquez
- https://marquezproject.github.io/marquez/
-version: 0.51.0
+version: 0.51.1
diff --git a/chart/values.yaml b/chart/values.yaml
index 7afc81a56a..09e2ed124d 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -20,7 +20,7 @@ marquez:
image:
registry: docker.io
repository: marquezproject/marquez
- tag: 0.51.0
+ tag: 0.51.1
pullPolicy: IfNotPresent
## Name of the existing secret containing credentials for the Marquez installation.
## When this is specified, it will take precedence over the values configured in the 'db' section.
@@ -89,7 +89,7 @@ web:
image:
registry: docker.io
repository: marquezproject/marquez-web
- tag: 0.51.0
+ tag: 0.51.1
pullPolicy: IfNotPresent
## Marquez website will run on this port
##
diff --git a/clients/java/README.md b/clients/java/README.md
index 77d2180ba1..f4454159b2 100644
--- a/clients/java/README.md
+++ b/clients/java/README.md
@@ -10,14 +10,14 @@ Maven:
io.github.marquezproject
marquez-java
- 0.51.0
+ 0.51.1
```
or Gradle:
```groovy
-implementation 'io.github.marquezproject:marquez-java:0.51.0
+implementation 'io.github.marquezproject:marquez-java:0.51.1
```
## Usage
diff --git a/docker/up.sh b/docker/up.sh
index 9ccbe11a42..4127545659 100755
--- a/docker/up.sh
+++ b/docker/up.sh
@@ -8,9 +8,9 @@
set -e
# Version of Marquez
-readonly VERSION=0.51.0
+readonly VERSION=0.51.1
# Build version of Marquez
-readonly BUILD_VERSION=0.51.0
+readonly BUILD_VERSION=0.51.1
title() {
echo -e "\033[1m${1}\033[0m"
diff --git a/gradle.properties b/gradle.properties
index 157521a53d..18cbac0b99 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -7,4 +7,4 @@ org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAME
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-version=0.52.0-SNAPSHOT
+version=0.51.1
diff --git a/spec/openapi.yml b/spec/openapi.yml
index f15419a4c0..1810a1a45a 100644
--- a/spec/openapi.yml
+++ b/spec/openapi.yml
@@ -3,7 +3,7 @@
openapi: 3.0.2
info:
title: Marquez
- version: 0.52.0-SNAPSHOT
+ version: 0.51.1
description: Marquez is an open source **metadata service** for the **collection**, **aggregation**,
and **visualization** of a data ecosystem's metadata.
license:
From e16b8038a8261450bb94ef84e88a6b71624856b6 Mon Sep 17 00:00:00 2001
From: Willy Lulciuc
Date: Thu, 27 Mar 2025 00:32:36 -0700
Subject: [PATCH 05/24] Prepare next development version 0.51.2-SNAPSHOT
Signed-off-by: Willy Lulciuc
Signed-off-by: swar00pduthks
---
.circleci/api-load-test.sh | 2 +-
clients/python/marquez_client/__init__.py | 2 +-
clients/python/setup.cfg | 2 +-
clients/python/setup.py | 2 +-
gradle.properties | 2 +-
spec/openapi.yml | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/.circleci/api-load-test.sh b/.circleci/api-load-test.sh
index 3cf8d69cc8..6d8e2e76ff 100755
--- a/.circleci/api-load-test.sh
+++ b/.circleci/api-load-test.sh
@@ -14,7 +14,7 @@
set -e
# Build version of Marquez
-readonly MARQUEZ_VERSION=0.52.0-SNAPSHOT
+readonly MARQUEZ_VERSION=0.51.2-SNAPSHOT
# Fully qualified path to marquez.jar
readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
diff --git a/clients/python/marquez_client/__init__.py b/clients/python/marquez_client/__init__.py
index 90677076ea..b0dad01f5d 100644
--- a/clients/python/marquez_client/__init__.py
+++ b/clients/python/marquez_client/__init__.py
@@ -4,7 +4,7 @@
# -*- coding: utf-8 -*-
__author__ = """Marquez Project"""
-__version__ = "0.52.0"
+__version__ = "0.51.2"
from marquez_client.client import MarquezClient # noqa: F401
from marquez_client.clients import Clients # noqa: F401
diff --git a/clients/python/setup.cfg b/clients/python/setup.cfg
index 3cba87f95f..38642fd3e0 100644
--- a/clients/python/setup.cfg
+++ b/clients/python/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.52.0
+current_version = 0.51.2
commit = False
tag = False
parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P.*)
diff --git a/clients/python/setup.py b/clients/python/setup.py
index 064f09e8a3..ad5a9a8d66 100644
--- a/clients/python/setup.py
+++ b/clients/python/setup.py
@@ -25,7 +25,7 @@
setup(
name="marquez-python",
- version="0.52.0",
+ version="0.51.2",
description="Marquez Python Client",
long_description=readme,
long_description_content_type="text/markdown",
diff --git a/gradle.properties b/gradle.properties
index 18cbac0b99..511d2c07c0 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -7,4 +7,4 @@ org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAME
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-version=0.51.1
+version=0.51.2-SNAPSHOT
diff --git a/spec/openapi.yml b/spec/openapi.yml
index 1810a1a45a..f9fad148bd 100644
--- a/spec/openapi.yml
+++ b/spec/openapi.yml
@@ -3,7 +3,7 @@
openapi: 3.0.2
info:
title: Marquez
- version: 0.51.1
+ version: 0.51.2-SNAPSHOT
description: Marquez is an open source **metadata service** for the **collection**, **aggregation**,
and **visualization** of a data ecosystem's metadata.
license:
From f0fd8150efb21769a75c97012fd5554350c9e186 Mon Sep 17 00:00:00 2001
From: swar00pduthks
Date: Mon, 7 Apr 2025 08:24:55 +0100
Subject: [PATCH 06/24] upgrade dropwizard & jakartaee
Signed-off-by: swar00pduthks
---
api/build.gradle | 53 +++++-
api/src/main/java/marquez/MarquezApp.java | 36 ++--
api/src/main/java/marquez/MarquezConfig.java | 2 +-
api/src/main/java/marquez/MarquezContext.java | 3 +-
.../main/java/marquez/api/BaseResource.java | 4 +-
.../marquez/api/ColumnLineageResource.java | 19 +-
.../java/marquez/api/DatasetResource.java | 54 +++---
.../main/java/marquez/api/JobResource.java | 63 ++++---
.../java/marquez/api/NamespaceResource.java | 37 ++--
.../java/marquez/api/OpenLineageResource.java | 46 ++---
.../main/java/marquez/api/RunResource.java | 14 +-
.../main/java/marquez/api/SearchResource.java | 26 +--
.../main/java/marquez/api/SourceResource.java | 33 ++--
.../main/java/marquez/api/StatsResource.java | 21 +--
.../main/java/marquez/api/TagResource.java | 29 ++-
.../exceptions/DatasetNotFoundException.java | 2 +-
.../DatasetVersionNotFoundException.java | 2 +-
.../marquez/api/exceptions/FacetNotValid.java | 2 +-
.../exceptions/FieldNotFoundException.java | 2 +-
.../JdbiExceptionExceptionMapper.java | 14 +-
.../api/exceptions/JobNotFoundException.java | 2 +-
.../JobVersionNotFoundException.java | 2 +-
.../JsonProcessingExceptionMapper.java | 14 +-
.../NamespaceNotFoundException.java | 2 +-
.../exceptions/RunAlreadyExistsException.java | 2 +-
.../api/exceptions/RunNotFoundException.java | 2 +-
.../exceptions/SourceNotFoundException.java | 2 +-
.../api/exceptions/TagNotFoundException.java | 2 +-
.../marquez/api/filter/JobRedirectFilter.java | 12 +-
.../java/marquez/api/models/JobVersion.java | 2 +-
.../java/marquez/api/models/Metadata.java | 2 +-
.../java/marquez/api/models/VersionId.java | 2 +-
.../marquez/api/v2beta/SearchResource.java | 19 +-
.../java/marquez/cli/DbMigrateCommand.java | 4 +-
.../java/marquez/cli/DbRetentionCommand.java | 4 +-
.../java/marquez/cli/MetadataCommand.java | 4 +-
.../main/java/marquez/cli/SeedCommand.java | 4 +-
api/src/main/java/marquez/common/Utils.java | 4 +-
.../common/base/MorePreconditions.java | 2 +-
.../java/marquez/common/models/Field.java | 2 +-
.../java/marquez/db/DatasetFacetsDao.java | 2 +-
.../main/java/marquez/db/FlywayFactory.java | 2 +-
.../main/java/marquez/db/JobFacetsDao.java | 2 +-
api/src/main/java/marquez/db/LineageDao.java | 2 +-
api/src/main/java/marquez/db/SearchDao.java | 2 +-
.../marquez/db/exceptions/DbException.java | 2 +-
.../db/exceptions/DbRetentionException.java | 2 +-
.../db/models/ColumnLineageNodeData.java | 14 +-
.../marquez/db/models/DatasetFieldRow.java | 2 +-
.../java/marquez/db/models/DatasetRow.java | 2 +-
.../marquez/db/models/DatasetSymlinkRow.java | 2 +-
.../marquez/db/models/DatasetVersionRow.java | 2 +-
.../db/models/ExtendedDatasetVersionRow.java | 2 +-
.../marquez/db/models/ExtendedRunRow.java | 2 +-
.../marquez/db/models/InputFieldNodeData.java | 13 +-
.../main/java/marquez/db/models/JobRow.java | 2 +-
.../java/marquez/db/models/JobVersionRow.java | 2 +-
.../java/marquez/db/models/NamespaceRow.java | 2 +-
.../main/java/marquez/db/models/RunRow.java | 2 +-
.../java/marquez/db/models/SourceRow.java | 2 +-
.../main/java/marquez/db/models/TagRow.java | 2 +-
.../graphql/MarquezGraphqlServletBuilder.java | 3 +-
.../java/marquez/jobs/DbRetentionConfig.java | 2 +-
.../marquez/logging/LoggingMdcFilter.java | 14 +-
.../marquez/service/ColumnLineageService.java | 1 +
.../java/marquez/service/LineageService.java | 5 +-
.../service/NodeIdNotFoundException.java | 39 ----
.../main/java/marquez/service/RunService.java | 2 +-
.../service/RunTransitionListener.java | 2 +-
.../java/marquez/service/SearchService.java | 4 +-
.../exceptions/NodeIdNotFoundException.java | 18 ++
.../marquez/service/models/ColumnLineage.java | 4 +-
.../models/ColumnLineageInputField.java | 2 +-
.../java/marquez/service/models/Dataset.java | 2 +-
.../marquez/service/models/DatasetData.java | 2 +-
.../marquez/service/models/DatasetEvent.java | 4 +-
.../marquez/service/models/DatasetMeta.java | 2 +-
.../service/models/DatasetVersion.java | 2 +-
.../java/marquez/service/models/DbTable.java | 2 +-
.../marquez/service/models/DbTableMeta.java | 2 +-
.../service/models/DbTableVersion.java | 2 +-
.../main/java/marquez/service/models/Job.java | 2 +-
.../java/marquez/service/models/JobData.java | 2 +-
.../java/marquez/service/models/JobEvent.java | 4 +-
.../marquez/service/models/JobFacets.java | 2 +-
.../java/marquez/service/models/JobMeta.java | 2 +-
.../marquez/service/models/LineageEvent.java | 6 +-
.../marquez/service/models/Namespace.java | 2 +-
.../marquez/service/models/NamespaceMeta.java | 2 +-
.../java/marquez/service/models/Node.java | 2 +-
.../java/marquez/service/models/NodeId.java | 2 +-
.../main/java/marquez/service/models/Run.java | 2 +-
.../marquez/service/models/RunFacets.java | 2 +-
.../java/marquez/service/models/RunMeta.java | 2 +-
.../java/marquez/service/models/Source.java | 2 +-
.../marquez/service/models/SourceMeta.java | 2 +-
.../java/marquez/service/models/Stream.java | 2 +-
.../marquez/service/models/StreamMeta.java | 2 +-
.../marquez/service/models/StreamVersion.java | 2 +-
.../main/java/marquez/service/models/Tag.java | 2 +-
.../TracingContainerResponseFilter.java | 12 +-
.../marquez/tracing/TracingServletFilter.java | 12 +-
api/src/test/TestMetrics.java | 11 ++
.../marquez/OpenLineageIntegrationTest.java | 2 +-
.../marquez/api/NamespaceResourceTest.java | 2 +-
.../marquez/api/OpenLineageResourceTest.java | 2 +-
.../java/marquez/api/StatResourceTest.java | 2 +-
.../java/marquez/api/models/ActiveRun.java | 2 +-
.../java/marquez/db/LineageTestUtils.java | 2 +-
api/src/test/java/marquez/db/TestingDb.java | 2 +-
.../MarquezJdbiExternalPostgresExtension.java | 21 ++-
.../marquez/jobs/DbRetentionConfigTest.java | 6 +-
.../marquez/logging/LoggingMdcFilterTest.java | 2 +-
.../service/ColumnLineageServiceTest.java | 7 +-
build.gradle | 19 +-
clients/java/build.gradle | 17 ++
.../java/marquez/client/MarquezClient.java | 2 +-
.../client/MarquezClientException.java | 2 +-
.../main/java/marquez/client/MarquezHttp.java | 2 +-
.../marquez/client/MarquezHttpException.java | 2 +-
.../java/marquez/client/MarquezPathV1.java | 2 +-
.../main/java/marquez/client/MarquezUrl.java | 2 +-
.../java/marquez/client/models/Dataset.java | 2 +-
.../marquez/client/models/DatasetMeta.java | 2 +-
.../client/models/DatasetNodeData.java | 2 +-
.../marquez/client/models/DatasetVersion.java | 2 +-
.../java/marquez/client/models/DbTable.java | 2 +-
.../marquez/client/models/DbTableMeta.java | 2 +-
.../marquez/client/models/DbTableVersion.java | 2 +-
.../java/marquez/client/models/Field.java | 2 +-
.../main/java/marquez/client/models/Job.java | 2 +-
.../java/marquez/client/models/JobMeta.java | 2 +-
.../marquez/client/models/JobNodeData.java | 2 +-
.../marquez/client/models/JobVersion.java | 2 +-
.../marquez/client/models/MarquezClient.java | 108 +++++++++++
.../java/marquez/client/models/Namespace.java | 2 +-
.../marquez/client/models/NamespaceMeta.java | 2 +-
.../main/java/marquez/client/models/Node.java | 2 +-
.../java/marquez/client/models/NodeId.java | 2 +-
.../main/java/marquez/client/models/Run.java | 2 +-
.../java/marquez/client/models/RunMeta.java | 2 +-
.../java/marquez/client/models/Source.java | 2 +-
.../marquez/client/models/SourceMeta.java | 2 +-
.../java/marquez/client/models/Stream.java | 2 +-
.../marquez/client/models/StreamMeta.java | 2 +-
.../marquez/client/models/StreamVersion.java | 2 +-
.../main/java/marquez/client/models/Tag.java | 2 +-
.../marquez/client/MarquezClientTest.java | 174 ++++++++----------
.../client/models/DbTableMetaTest.java | 6 +-
.../marquez/client/models/JobMetaTest.java | 6 +-
deps.txt | Bin 0 -> 443484 bytes
dropwizard-core.pom | 0
152 files changed, 700 insertions(+), 549 deletions(-)
delete mode 100644 api/src/main/java/marquez/service/NodeIdNotFoundException.java
create mode 100644 api/src/main/java/marquez/service/exceptions/NodeIdNotFoundException.java
create mode 100644 api/src/test/TestMetrics.java
create mode 100644 clients/java/src/main/java/marquez/client/models/MarquezClient.java
create mode 100644 deps.txt
create mode 100644 dropwizard-core.pom
diff --git a/api/build.gradle b/api/build.gradle
index b4effde600..c81f113808 100644
--- a/api/build.gradle
+++ b/api/build.gradle
@@ -18,6 +18,8 @@ import org.apache.tools.ant.filters.ReplaceTokens
plugins {
id 'maven-publish'
id 'signing'
+ id 'application'
+ id 'com.github.johnrengelman.shadow'
}
ext {
@@ -34,11 +36,28 @@ dependencies {
implementation "io.dropwizard:dropwizard-json-logging:${dropwizardVersion}"
implementation "io.dropwizard:dropwizard-http2:${dropwizardVersion}"
implementation "io.dropwizard:dropwizard-assets:${dropwizardVersion}"
- implementation "io.prometheus:simpleclient:${prometheusVersion}"
- implementation "io.openlineage:openlineage-java:${openlineageVersion}"
- implementation "io.prometheus:simpleclient_dropwizard:${prometheusVersion}"
- implementation "io.prometheus:simpleclient_hotspot:${prometheusVersion}"
- implementation "io.prometheus:simpleclient_servlet:${prometheusVersion}"
+
+ // Jakarta EE dependencies
+ implementation platform("jakarta.platform:jakarta.jakartaee-bom:${jakartaVersion}")
+ implementation "jakarta.platform:jakarta.jakartaee-api:${jakartaVersion}"
+ implementation "jakarta.validation:jakarta.validation-api:${jakartaValidationVersion}"
+ implementation 'jakarta.annotation:jakarta.annotation-api:2.1.1'
+ implementation 'jakarta.transaction:jakarta.transaction-api:2.0.1'
+ implementation 'jakarta.servlet:jakarta.servlet-api:5.0.0'
+ implementation 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
+
+ // GraphQL dependencies with Jakarta EE 9 support
+ implementation "com.graphql-java:graphql-java:${graphqlJavaVersion}"
+ implementation "com.graphql-java-kickstart:graphql-java-servlet:${graphqlServletVersion}"
+ implementation "com.graphql-java-kickstart:graphql-java-kickstart:${graphqlServletVersion}"
+
+ implementation "io.prometheus:simpleclient_servlet_jakarta:0.16.0"
+
+ implementation "io.prometheus:simpleclient_common:0.16.0"
+ implementation "io.prometheus:simpleclient_dropwizard:0.16.0"
+ implementation "io.prometheus:simpleclient_hotspot:0.16.0"
+
+
implementation "org.jdbi:jdbi3-core:${jdbi3Version}"
implementation "org.jdbi:jdbi3-jackson2:${jdbi3Version}"
implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
@@ -48,8 +67,7 @@ dependencies {
implementation "io.sentry:sentry:${sentryVersion}"
implementation 'org.flywaydb:flyway-core:8.5.13'
implementation "org.postgresql:postgresql:${postgresqlVersion}"
- implementation 'com.graphql-java:graphql-java:20.9'
- implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
+ implementation "io.openlineage:openlineage-java:0.30.1"
implementation 'org.opensearch.client:opensearch-rest-client:2.19.1'
implementation 'org.opensearch.client:opensearch-java:2.22.0'
@@ -61,6 +79,19 @@ dependencies {
testImplementation "org.testcontainers:postgresql:${testcontainersVersion}"
testImplementation "org.testcontainers:junit-jupiter:${testcontainersVersion}"
testImplementation 'org.apache.httpcomponents:httpclient:4.5.14'
+ testImplementation "org.junit.jupiter:junit-jupiter-api:${junit5Version}"
+ testImplementation "org.junit.jupiter:junit-jupiter-engine:${junit5Version}"
+ testImplementation "org.junit.jupiter:junit-jupiter-params:${junit5Version}"
+ testImplementation "org.testcontainers:testcontainers:${testcontainersVersion}"
+
+ // Add Jakarta EE dependencies for tests
+ testImplementation platform("jakarta.platform:jakarta.jakartaee-bom:${jakartaVersion}")
+ testImplementation "jakarta.platform:jakarta.jakartaee-api:${jakartaVersion}"
+ testImplementation "jakarta.validation:jakarta.validation-api:${jakartaValidationVersion}"
+ testImplementation 'jakarta.annotation:jakarta.annotation-api:2.1.1'
+ testImplementation 'jakarta.transaction:jakarta.transaction-api:2.0.1'
+ testImplementation 'jakarta.servlet:jakarta.servlet-api:5.0.0'
+ testImplementation 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
}
task testUnit(type: Test) {
@@ -158,6 +189,14 @@ shadowJar {
from(projectDir) {
include 'LICENSE'
}
+ mergeServiceFiles()
+ dependencies {
+ include(dependency('jakarta.platform:jakarta.jakartaee-api'))
+ include(dependency('jakarta.platform:jakarta.jakartaee-bom'))
+ include(dependency('jakarta.annotation:jakarta.annotation-api'))
+ include(dependency('jakarta.validation:jakarta.validation-api'))
+ include(dependency('io.dropwizard:dropwizard-jdbi3'))
+ }
manifest {
attributes(
'Created-By': "Gradle ${gradle.gradleVersion}",
diff --git a/api/src/main/java/marquez/MarquezApp.java b/api/src/main/java/marquez/MarquezApp.java
index 15406d95c2..1eab119fef 100644
--- a/api/src/main/java/marquez/MarquezApp.java
+++ b/api/src/main/java/marquez/MarquezApp.java
@@ -1,28 +1,23 @@
-/*
- * Copyright 2018-2023 contributors to the Marquez project
- * SPDX-License-Identifier: Apache-2.0
- */
-
package marquez;
import com.codahale.metrics.jdbi3.InstrumentedSqlLogger;
import com.fasterxml.jackson.databind.SerializationFeature;
-import io.dropwizard.Application;
import io.dropwizard.assets.AssetsBundle;
import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
import io.dropwizard.configuration.SubstitutingSourceProvider;
+import io.dropwizard.core.Application;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
import io.dropwizard.db.DataSourceFactory;
import io.dropwizard.db.ManagedDataSource;
import io.dropwizard.jdbi3.JdbiFactory;
-import io.dropwizard.setup.Bootstrap;
-import io.dropwizard.setup.Environment;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.dropwizard.DropwizardExports;
-import io.prometheus.client.exporter.MetricsServlet;
import io.prometheus.client.hotspot.DefaultExports;
+import io.prometheus.client.servlet.jakarta.exporter.MetricsServlet;
import io.sentry.Sentry;
+import jakarta.servlet.DispatcherType;
import java.util.EnumSet;
-import javax.servlet.DispatcherType;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import marquez.api.filter.JobRedirectFilter;
@@ -76,20 +71,18 @@ public String getName() {
@Override
public void initialize(@NonNull Bootstrap bootstrap) {
- // Enable metric collection for prometheus.
+ // Enable Prometheus metrics
CollectorRegistry.defaultRegistry.register(
new DropwizardExports(bootstrap.getMetricRegistry()));
DatabaseMetrics.registry.register(new DropwizardExports(bootstrap.getMetricRegistry()));
- DefaultExports.initialize(); // Add metrics for CPU, JVM memory, etc.
+ DefaultExports.initialize();
DefaultExports.register(DatabaseMetrics.registry);
- // Enable variable substitution with environment variables.
bootstrap.setConfigurationSourceProvider(
new SubstitutingSourceProvider(
bootstrap.getConfigurationSourceProvider(),
new EnvironmentVariableSubstitutor(ERROR_ON_UNDEFINED)));
- // Add CLI commands
bootstrap.addCommand(new DbMigrateCommand());
bootstrap.addCommand(new DbRetentionCommand());
bootstrap.addCommand(new MetadataCommand());
@@ -98,7 +91,6 @@ public void initialize(@NonNull Bootstrap bootstrap) {
bootstrap.getObjectMapper().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
Utils.addZonedDateTimeMixin(bootstrap.getObjectMapper());
- // Add graphql playground
bootstrap.addBundle(
new AssetsBundle(
"/assets",
@@ -118,8 +110,7 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
DbMigration.migrateDbOrError(config.getFlywayFactory(), source, config.isMigrateOnStartup());
} catch (FlywayException errorOnDbMigrate) {
log.info("Stopping app...");
- // Propagate throwable up the stack.
- onFatalError(errorOnDbMigrate); // Signal app termination.
+ onFatalError(errorOnDbMigrate);
}
if (isSentryEnabled(config)) {
@@ -149,16 +140,12 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
registerServlets(env);
registerFilters(env, marquezContext);
- // Add scheduled jobs to lifecycle.
if (config.hasDbRetentionPolicy()) {
- // Add job to apply retention policy to database.
env.lifecycle().manage(new DbRetentionJob(jdbi, config.getDbRetention()));
}
- // Add job to refresh materialized views.
env.lifecycle().manage(new MaterializeViewRefresherJob(jdbi));
- // set namespaceFilter
ExclusionsConfig exclusions = config.getExclude();
Exclusions.use(exclusions);
}
@@ -168,7 +155,6 @@ private boolean isSentryEnabled(MarquezConfig config) {
&& !config.getSentry().getDsn().equals(SentryConfig.DEFAULT_DSN);
}
- /** Returns a new {@link Jdbi} object. */
private Jdbi newJdbi(
@NonNull MarquezConfig config, @NonNull Environment env, @NonNull ManagedDataSource source) {
final JdbiFactory factory = new JdbiFactory();
@@ -197,6 +183,9 @@ public void registerResources(
.addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
}
+ // Prometheus metrics endpoint
+ env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
+
log.debug("Registering resources...");
for (final Object resource : context.getResources()) {
env.jersey().register(resource);
@@ -205,9 +194,6 @@ public void registerResources(
private void registerServlets(@NonNull Environment env) {
log.debug("Registering servlets...");
-
- // Expose metrics for monitoring.
- env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
env.servlets()
.addServlet(PROMETHEUS_V2, new MetricsServlet(DatabaseMetrics.registry))
.addMapping(PROMETHEUS_ENDPOINT_V2);
diff --git a/api/src/main/java/marquez/MarquezConfig.java b/api/src/main/java/marquez/MarquezConfig.java
index d678fc7de5..04f65223f5 100644
--- a/api/src/main/java/marquez/MarquezConfig.java
+++ b/api/src/main/java/marquez/MarquezConfig.java
@@ -7,7 +7,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableSet;
-import io.dropwizard.Configuration;
+import io.dropwizard.core.Configuration;
import io.dropwizard.db.DataSourceFactory;
import lombok.Getter;
import lombok.NoArgsConstructor;
diff --git a/api/src/main/java/marquez/MarquezContext.java b/api/src/main/java/marquez/MarquezContext.java
index dd789e82ed..bf47a3551a 100644
--- a/api/src/main/java/marquez/MarquezContext.java
+++ b/api/src/main/java/marquez/MarquezContext.java
@@ -8,7 +8,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
-import graphql.kickstart.servlet.GraphQLHttpServlet;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
@@ -110,7 +109,7 @@ public final class MarquezContext {
@Getter private final ImmutableList