diff --git a/.gitignore b/.gitignore index add98add9..9c3364cfc 100644 --- a/.gitignore +++ b/.gitignore @@ -75,8 +75,8 @@ crash.log crash.*.log # Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars *.tfvars.json @@ -105,3 +105,9 @@ terraform.rc # Ignore the spark shell bash file because it contains secrets start-spark-shell.sh + +# Ignore nextjs env files +.env* + +tables.jar +auth-token-cli.tar diff --git a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/util/SparkJobUtil.java b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/util/SparkJobUtil.java index 3cff853cd..9c1aad2bd 100644 --- a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/util/SparkJobUtil.java +++ b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/util/SparkJobUtil.java @@ -63,6 +63,9 @@ public static String createDeleteStatement( String granularity, int count, ZonedDateTime now) { + // Convert to UTC to match Spark SQL's timestamp interpretation + String nowUtc = + now.withZoneSameInstant(java.time.ZoneId.of("UTC")).toLocalDateTime().toString(); if (!StringUtils.isBlank(columnPattern)) { String query = String.format( @@ -71,7 +74,7 @@ public static String createDeleteStatement( String.format( RETENTION_CONDITION_WITH_PATTERN_TEMPLATE, columnName, - now.toLocalDateTime(), + nowUtc, count, granularity, columnPattern)); @@ -92,7 +95,7 @@ public static String createDeleteStatement( RETENTION_CONDITION_TEMPLATE, columnName, granularity, - now.toLocalDateTime(), + nowUtc, count, granularity)); log.info("Table: {}. No column pattern provided: deleteQuery: {}", fqtn, query); diff --git a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/util/SparkJobUtilTest.java b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/util/SparkJobUtilTest.java index 5c2cdb191..cec9bdd6e 100644 --- a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/util/SparkJobUtilTest.java +++ b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/util/SparkJobUtilTest.java @@ -11,10 +11,12 @@ public class SparkJobUtilTest { @Test void testCreateDeleteStatement() { ZonedDateTime now = ZonedDateTime.now(); + String nowUtc = + now.withZoneSameInstant(java.time.ZoneId.of("UTC")).toLocalDateTime().toString(); String expected = String.format( "DELETE FROM `db`.`table-name` WHERE timestamp < date_trunc('day', timestamp '%s' - INTERVAL 2 days)", - now.toLocalDateTime()); + nowUtc); Assertions.assertEquals( expected, SparkJobUtil.createDeleteStatement("db.table-name", "timestamp", "", "day", 2, now)); @@ -30,10 +32,12 @@ void testGetQuotedFqtn() { @Test void testCreateDeleteStatementWithStringColumnPartition() { ZonedDateTime now = ZonedDateTime.now(); + String nowUtc = + now.withZoneSameInstant(java.time.ZoneId.of("UTC")).toLocalDateTime().toString(); String expected = String.format( "DELETE FROM `db`.`table-name` WHERE string_partition < cast(date_format(timestamp '%s' - INTERVAL 2 DAYs, 'yyyy-MM-dd-HH') as string)", - now.toLocalDateTime()); + nowUtc); Assertions.assertEquals( expected, SparkJobUtil.createDeleteStatement( diff --git a/infra/recipes/docker-compose/common/mysql-services.yml b/infra/recipes/docker-compose/common/mysql-services.yml index 5dd72e825..128c615a3 100644 --- a/infra/recipes/docker-compose/common/mysql-services.yml +++ b/infra/recipes/docker-compose/common/mysql-services.yml @@ -9,3 +9,9 @@ services: - MYSQL_PASSWORD=oh_password - MYSQL_USER=oh_user - MYSQL_DATABASE=oh_db + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-poh_root_password"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 30s diff --git a/infra/recipes/docker-compose/common/oh-services.yml b/infra/recipes/docker-compose/common/oh-services.yml index 73ef5a656..343db1b4b 100644 --- a/infra/recipes/docker-compose/common/oh-services.yml +++ b/infra/recipes/docker-compose/common/oh-services.yml @@ -1,5 +1,11 @@ version: "3.3" services: + openhouse-spectacle: + build: + context: ../../../.. + dockerfile: spectacle-service.Dockerfile + ports: + - 8003:3000 openhouse-tables: build: context: ../../../.. diff --git a/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.spectacle.yml b/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.spectacle.yml new file mode 100644 index 000000000..550a1e23e --- /dev/null +++ b/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.spectacle.yml @@ -0,0 +1,137 @@ +version: "3.3" +services: + openhouse-spectacle: + container_name: local.openhouse-spectacle + extends: + file: ../common/oh-services.yml + service: openhouse-spectacle + depends_on: + - openhouse-tables + - openhouse-jobs + environment: + - NEXT_PUBLIC_TABLES_SERVICE_URL=http://localhost:8000 + - NEXT_PUBLIC_JOBS_SERVICE_URL=http://localhost:8002 + + openhouse-tables: + container_name: local.openhouse-tables + extends: + file: ../common/oh-services.yml + service: openhouse-tables + volumes: + - ./:/var/config/ + depends_on: + - openhouse-housetables + - namenode + - datanode + - prometheus + - opa + + openhouse-jobs: + container_name: local.openhouse-jobs + extends: + file: ../common/oh-services.yml + service: openhouse-jobs + volumes: + - ./:/var/config/ + depends_on: + - openhouse-housetables + - prometheus + + openhouse-jobs-scheduler: + container_name: local.openhouse-jobs-scheduler + extends: + file: ../common/oh-services.yml + service: openhouse-jobs-scheduler + volumes: + - ./:/var/config/ + depends_on: + - openhouse-tables + - openhouse-jobs + - prometheus + - namenode + + profiles: + - with_jobs_scheduler + + openhouse-housetables: + container_name: local.openhouse-housetables + extends: + file: ../common/oh-services.yml + service: openhouse-housetables + volumes: + - ./:/var/config/ + depends_on: + prometheus: + condition: service_started + mysql: + condition: service_healthy + restart: on-failure + environment: + - HTS_DB_USER=oh_user + - HTS_DB_PASSWORD=oh_password + + prometheus: + extends: + file: ../common/oh-services.yml + service: prometheus + + namenode: + container_name: local.namenode + extends: + file: ../common/hdfs-services.yml + service: namenode + volumes: + - hadoop_namenode:/hadoop/dfs/name + + datanode: + container_name: local.datanode + extends: + file: ../common/hdfs-services.yml + service: datanode + volumes: + - hadoop_datanode:/hadoop/dfs/data + + spark-master: + container_name: local.spark-master + extends: + file: ../common/spark-services.yml + service: spark-master + volumes: + - ./spark/apps:/opt/spark-apps + + spark-worker-a: + container_name: local.spark-worker-a + extends: + file: ../common/spark-services.yml + service: spark-worker-a + depends_on: + - spark-master + + spark-livy: + container_name: local.spark-livy + extends: + file: ../common/spark-services.yml + service: spark-livy + depends_on: + - spark-master + + mysql: + container_name: local.mysql + extends: + file: ../common/mysql-services.yml + service: mysql + + opa: + container_name: local.opa + extends: + file: ../common/opa-services.yml + service: opa + +volumes: + hadoop_namenode: + hadoop_datanode: + + +networks: + openhouse-network: + driver: bridge diff --git a/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.yml b/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.yml index 8cd9e89c3..24ec5be90 100644 --- a/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.yml +++ b/infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.yml @@ -81,6 +81,8 @@ services: extends: file: ../common/spark-services.yml service: spark-master + volumes: + - ./spark/apps:/opt/spark-apps spark-worker-a: container_name: local.spark-worker-a diff --git a/infra/recipes/docker-compose/oh-hadoop-spark/spark/apps/populate-catalog.scala b/infra/recipes/docker-compose/oh-hadoop-spark/spark/apps/populate-catalog.scala new file mode 100644 index 000000000..4a2e1f69b --- /dev/null +++ b/infra/recipes/docker-compose/oh-hadoop-spark/spark/apps/populate-catalog.scala @@ -0,0 +1,645 @@ +// ============================================================================ +// OpenHouse Catalog Population Script +// ============================================================================ +// +// PURPOSE: +// This script populates the OpenHouse catalog with 5 test tables across 3 databases: +// - 2 unpartitioned tables with 3 columns each +// - 3 partitioned tables with 3 data columns + 1 timestamp column (hourly partitioning) +// +// DATABASES POPULATED: +// - openhouse.testdb (primary test database) +// - openhouse.devdb (development database) +// - openhouse.stagingdb (staging database) +// +// Each table goes through multiple operations to create a rich snapshot history: +// - DROP TABLE IF EXISTS (ensures clean state) +// - 4 insert commits (separate batches) +// - 1 delete operation +// - 1 additional insert commit after the delete +// This creates at least 6 snapshots per table for testing time travel and metadata queries +// +// HOW TO RUN: +// 1. Start the docker environment: +// cd infra/recipes/docker-compose/oh-hadoop-spark +// docker-compose -f docker-compose.spectacle.yml up -d +// +// 2. Run this script from spark-master container: +// docker exec -it local.spark-master spark-shell \ +// --conf spark.sql.catalog.openhouse=org.apache.iceberg.spark.SparkCatalog \ +// --conf spark.sql.catalog.openhouse.type=rest \ +// --conf spark.sql.catalog.openhouse.uri=http://local.openhouse-tables:8000 \ +// --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ +// -i /opt/spark-apps/populate-catalog.scala +// +// TABLES CREATED (in each database): +// ┌─────────────────────┬──────────────────────────────────────┬─────────────────┐ +// │ Table Name │ Columns │ Partitioning │ +// ├─────────────────────┼──────────────────────────────────────┼─────────────────┤ +// │ user_profiles │ user_id, username, email │ None │ +// │ product_catalog │ product_id, product_name, price │ None │ +// │ clickstream_events │ event_id, user_id, page_url, │ hours(event_ │ +// │ │ event_time │ time) │ +// │ sensor_readings │ sensor_id, temperature, humidity, │ hours(reading_ │ +// │ │ reading_time │ time) │ +// │ transaction_logs │ transaction_id, amount, status, │ hours(transac- │ +// │ │ transaction_time │ tion_time) │ +// └─────────────────────┴──────────────────────────────────────┴─────────────────┘ +// +// VERIFY RESULTS: +// After running, verify in spark-shell: +// spark.sql("SHOW DATABASES").show() +// spark.sql("SHOW TABLES IN openhouse.testdb").show() +// spark.sql("SELECT * FROM openhouse.testdb.user_profiles LIMIT 5").show() +// spark.sql("SELECT * FROM openhouse.testdb.user_profiles.snapshots").show() +// +// Or via OpenHouse API: +// curl http://localhost:8000/v1/databases +// curl http://localhost:8000/v1/databases/testdb/tables +// +// ============================================================================ + +import org.apache.spark.sql.SparkSession +import scala.util.Random +import java.sql.Timestamp +import java.time.Instant +import java.time.temporal.ChronoUnit + +println("=" * 80) +println("OpenHouse Catalog Population Script") +println("=" * 80) + +// ============================================================================ +// Helper Functions +// ============================================================================ + +// Generates a random alphanumeric string of specified length +// Used for creating realistic test data like usernames, product names, etc. +def randomString(length: Int): String = { + val chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + (1 to length).map(_ => chars(Random.nextInt(chars.length))).mkString +} + +// Generates a random timestamp within the past N hours +// Used to create realistic time-series data for partitioned tables +// The hoursBack parameter determines the time range (e.g., 24 = last 24 hours) +def randomTimestamp(hoursBack: Int): Timestamp = { + val now = Instant.now() + val randomHoursAgo = Random.nextInt(hoursBack) + new Timestamp(now.minus(randomHoursAgo, ChronoUnit.HOURS).toEpochMilli) +} + +// ============================================================================ +// Main Function: Populate Database +// ============================================================================ +// This function creates and populates all 5 tables in the specified database +// It drops existing tables first to ensure a clean state +// ============================================================================ + +def populateDatabase(dbName: String): Unit = { + println("\n" + "=" * 80) + println(s"Processing Database: openhouse.${dbName}") + println("=" * 80) + + // Create database if it doesn't exist + // OpenHouse doesn't support "IF NOT EXISTS" check, so we try to create and ignore errors + println(s"\nEnsuring database openhouse.${dbName} exists...") + try { + spark.sql(s"CREATE DATABASE openhouse.${dbName}") + println(s"✓ Created database openhouse.${dbName}") + } catch { + case e: Exception if e.getMessage.contains("already exists") || e.getMessage.contains("AlreadyExistsException") => + println(s"✓ Database openhouse.${dbName} already exists") + case e: Exception => + println(s"! Skipping database creation (OpenHouse may auto-create on first table): ${e.getMessage}") + } + + // ============================================================================ + // PART 1: UNPARTITIONED TABLES + // ============================================================================ + + println("\n" + "-" * 80) + println("PART 1: Creating 2 Unpartitioned Tables") + println("-" * 80) + + // ========================================================================== + // UNPARTITIONED TABLE 1: user_profiles + // ========================================================================== + // Schema: user_id (STRING), username (STRING), email (STRING) + // ========================================================================== + + println(s"\n--- Table: openhouse.${dbName}.user_profiles ---") + + // Drop table if it exists to ensure clean state + println("Dropping table if exists...") + spark.sql(s"DROP TABLE IF EXISTS openhouse.${dbName}.user_profiles") + + println("Creating table...") + spark.sql(s""" + CREATE TABLE openhouse.${dbName}.user_profiles ( + user_id STRING, + username STRING, + email STRING + ) USING iceberg + """).show() + + // INSERT #1: Initial batch (25 users) + println("Insert #1: Adding 25 users...") + for (i <- 1 to 25) { + spark.sql(s""" + INSERT INTO openhouse.${dbName}.user_profiles + VALUES ('user_${i}', 'username_${randomString(8)}', 'user${i}@example.com') + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // INSERT #2: Second batch (25 users) + println("Insert #2: Adding 25 users...") + for (i <- 26 to 50) { + spark.sql(s""" + INSERT INTO openhouse.${dbName}.user_profiles + VALUES ('user_${i}', 'username_${randomString(8)}', 'user${i}@example.com') + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // INSERT #3: Third batch (25 users) + println("Insert #3: Adding 25 users...") + for (i <- 51 to 75) { + spark.sql(s""" + INSERT INTO openhouse.${dbName}.user_profiles + VALUES ('user_${i}', 'username_${randomString(8)}', 'user${i}@example.com') + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // INSERT #4: Fourth batch (25 users) + println("Insert #4: Adding 25 users...") + for (i <- 76 to 100) { + spark.sql(s""" + INSERT INTO openhouse.${dbName}.user_profiles + VALUES ('user_${i}', 'username_${randomString(8)}', 'user${i}@example.com') + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // DELETE #1: Remove some users (creates delete snapshot) + println("Delete #1: Removing users 1-10...") + spark.sql(s"DELETE FROM openhouse.${dbName}.user_profiles WHERE user_id IN ('user_1', 'user_2', 'user_3', 'user_4', 'user_5', 'user_6', 'user_7', 'user_8', 'user_9', 'user_10')").show() + println(s"Count after delete: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // INSERT #5: Final batch (10 users) + println("Insert #5: Adding 10 users...") + for (i <- 101 to 110) { + spark.sql(s""" + INSERT INTO openhouse.${dbName}.user_profiles + VALUES ('user_${i}', 'username_${randomString(8)}', 'user${i}@example.com') + """) + } + println(s"Final count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.user_profiles").collect()(0)(0)}") + + // ========================================================================== + // UNPARTITIONED TABLE 2: product_catalog + // ========================================================================== + // Schema: product_id (STRING), product_name (STRING), price (DOUBLE) + // ========================================================================== + + println(s"\n--- Table: openhouse.${dbName}.product_catalog ---") + + println("Dropping table if exists...") + spark.sql(s"DROP TABLE IF EXISTS openhouse.${dbName}.product_catalog") + + println("Creating table...") + spark.sql(s""" + CREATE TABLE openhouse.${dbName}.product_catalog ( + product_id STRING, + product_name STRING, + price DOUBLE + ) USING iceberg + """).show() + + // INSERT #1: Initial products (20 products) + println("Insert #1: Adding 20 products...") + for (i <- 1 to 20) { + val price = 10.0 + Random.nextDouble() * 990.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.product_catalog + VALUES ('prod_${i}', 'Product_${randomString(6)}', ${price}) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // INSERT #2: Second batch (20 products) + println("Insert #2: Adding 20 products...") + for (i <- 21 to 40) { + val price = 10.0 + Random.nextDouble() * 990.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.product_catalog + VALUES ('prod_${i}', 'Product_${randomString(6)}', ${price}) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // INSERT #3: Third batch (20 products) + println("Insert #3: Adding 20 products...") + for (i <- 41 to 60) { + val price = 10.0 + Random.nextDouble() * 990.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.product_catalog + VALUES ('prod_${i}', 'Product_${randomString(6)}', ${price}) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // INSERT #4: Fourth batch (20 products) + println("Insert #4: Adding 20 products...") + for (i <- 61 to 80) { + val price = 10.0 + Random.nextDouble() * 990.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.product_catalog + VALUES ('prod_${i}', 'Product_${randomString(6)}', ${price}) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // DELETE #1: Remove some products + println("Delete #1: Removing products 1-5...") + spark.sql(s"DELETE FROM openhouse.${dbName}.product_catalog WHERE product_id IN ('prod_1', 'prod_2', 'prod_3', 'prod_4', 'prod_5')").show() + println(s"Count after delete: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // INSERT #5: Final batch (10 products) + println("Insert #5: Adding 10 products...") + for (i <- 81 to 90) { + val price = 10.0 + Random.nextDouble() * 990.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.product_catalog + VALUES ('prod_${i}', 'Product_${randomString(6)}', ${price}) + """) + } + println(s"Final count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.product_catalog").collect()(0)(0)}") + + // ============================================================================ + // PART 2: PARTITIONED TABLES (HOURLY PARTITIONING) + // ============================================================================ + + println("\n" + "-" * 80) + println("PART 2: Creating 3 Partitioned Tables (Hourly Partitioning)") + println("-" * 80) + + // ========================================================================== + // PARTITIONED TABLE 1: clickstream_events + // ========================================================================== + // Schema: event_id, user_id, page_url, event_time + // Partitioned by: hours(event_time) + // ========================================================================== + + println(s"\n--- Table: openhouse.${dbName}.clickstream_events ---") + + println("Dropping table if exists...") + spark.sql(s"DROP TABLE IF EXISTS openhouse.${dbName}.clickstream_events") + + println("Creating table...") + spark.sql(s""" + CREATE TABLE openhouse.${dbName}.clickstream_events ( + event_id STRING, + user_id STRING, + page_url STRING, + event_time TIMESTAMP + ) USING iceberg + PARTITIONED BY (hours(event_time)) + """).show() + + // INSERT #1: Events from last 24 hours (30 events) + println("Insert #1: Adding 30 events (last 24 hours)...") + for (i <- 1 to 30) { + val ts = randomTimestamp(24) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.clickstream_events + VALUES ('event_${i}', 'user_${Random.nextInt(100)}', 'https://example.com/${randomString(10)}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // INSERT #2: Events from last 48 hours (30 events) + println("Insert #2: Adding 30 events (last 48 hours)...") + for (i <- 31 to 60) { + val ts = randomTimestamp(48) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.clickstream_events + VALUES ('event_${i}', 'user_${Random.nextInt(100)}', 'https://example.com/${randomString(10)}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // INSERT #3: Events from last 72 hours (30 events) + println("Insert #3: Adding 30 events (last 72 hours)...") + for (i <- 61 to 90) { + val ts = randomTimestamp(72) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.clickstream_events + VALUES ('event_${i}', 'user_${Random.nextInt(100)}', 'https://example.com/${randomString(10)}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // INSERT #4: Events from last 96 hours (30 events) + println("Insert #4: Adding 30 events (last 96 hours)...") + for (i <- 91 to 120) { + val ts = randomTimestamp(96) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.clickstream_events + VALUES ('event_${i}', 'user_${Random.nextInt(100)}', 'https://example.com/${randomString(10)}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // DELETE #1: Remove some events + println("Delete #1: Removing events 1-15...") + spark.sql(s"DELETE FROM openhouse.${dbName}.clickstream_events WHERE event_id IN ('event_1', 'event_2', 'event_3', 'event_4', 'event_5', 'event_6', 'event_7', 'event_8', 'event_9', 'event_10', 'event_11', 'event_12', 'event_13', 'event_14', 'event_15')").show() + println(s"Count after delete: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // INSERT #5: Final batch (15 events) + println("Insert #5: Adding 15 events...") + for (i <- 121 to 135) { + val ts = randomTimestamp(24) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.clickstream_events + VALUES ('event_${i}', 'user_${Random.nextInt(100)}', 'https://example.com/${randomString(10)}', TIMESTAMP('${ts}')) + """) + } + println(s"Final count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.clickstream_events").collect()(0)(0)}") + + // ========================================================================== + // PARTITIONED TABLE 2: sensor_readings + // ========================================================================== + // Schema: sensor_id, temperature, humidity, reading_time + // Partitioned by: hours(reading_time) + // Temperature: 15-35°C, Humidity: 30-80% + // ========================================================================== + + println(s"\n--- Table: openhouse.${dbName}.sensor_readings ---") + + println("Dropping table if exists...") + spark.sql(s"DROP TABLE IF EXISTS openhouse.${dbName}.sensor_readings") + + println("Creating table...") + spark.sql(s""" + CREATE TABLE openhouse.${dbName}.sensor_readings ( + sensor_id STRING, + temperature DOUBLE, + humidity DOUBLE, + reading_time TIMESTAMP + ) USING iceberg + PARTITIONED BY (hours(reading_time)) + """).show() + + // INSERT #1: Readings from last 48 hours (25 readings) + println("Insert #1: Adding 25 sensor readings (last 48 hours)...") + for (i <- 1 to 25) { + val ts = randomTimestamp(48) + val temp = 15.0 + Random.nextDouble() * 20.0 + val humidity = 30.0 + Random.nextDouble() * 50.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.sensor_readings + VALUES ('sensor_${Random.nextInt(10)}', ${temp}, ${humidity}, TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // INSERT #2: Readings from last 72 hours (25 readings) + println("Insert #2: Adding 25 sensor readings (last 72 hours)...") + for (i <- 26 to 50) { + val ts = randomTimestamp(72) + val temp = 15.0 + Random.nextDouble() * 20.0 + val humidity = 30.0 + Random.nextDouble() * 50.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.sensor_readings + VALUES ('sensor_${Random.nextInt(10)}', ${temp}, ${humidity}, TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // INSERT #3: Readings from last 96 hours (25 readings) + println("Insert #3: Adding 25 sensor readings (last 96 hours)...") + for (i <- 51 to 75) { + val ts = randomTimestamp(96) + val temp = 15.0 + Random.nextDouble() * 20.0 + val humidity = 30.0 + Random.nextDouble() * 50.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.sensor_readings + VALUES ('sensor_${Random.nextInt(10)}', ${temp}, ${humidity}, TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // INSERT #4: Readings from last 120 hours (25 readings) + println("Insert #4: Adding 25 sensor readings (last 120 hours)...") + for (i <- 76 to 100) { + val ts = randomTimestamp(120) + val temp = 15.0 + Random.nextDouble() * 20.0 + val humidity = 30.0 + Random.nextDouble() * 50.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.sensor_readings + VALUES ('sensor_${Random.nextInt(10)}', ${temp}, ${humidity}, TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // DELETE #1: Remove all readings from sensor_0 + println("Delete #1: Removing readings from sensor_0...") + spark.sql(s"DELETE FROM openhouse.${dbName}.sensor_readings WHERE sensor_id = 'sensor_0'").show() + println(s"Count after delete: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // INSERT #5: Final batch (15 readings) + println("Insert #5: Adding 15 sensor readings...") + for (i <- 101 to 115) { + val ts = randomTimestamp(24) + val temp = 15.0 + Random.nextDouble() * 20.0 + val humidity = 30.0 + Random.nextDouble() * 50.0 + spark.sql(s""" + INSERT INTO openhouse.${dbName}.sensor_readings + VALUES ('sensor_${Random.nextInt(10)}', ${temp}, ${humidity}, TIMESTAMP('${ts}')) + """) + } + println(s"Final count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.sensor_readings").collect()(0)(0)}") + + // ========================================================================== + // PARTITIONED TABLE 3: transaction_logs + // ========================================================================== + // Schema: transaction_id, amount, status, transaction_time + // Partitioned by: hours(transaction_time) + // Status values: completed, pending, failed, cancelled + // Amount: $10 - $10,000 + // ========================================================================== + + println(s"\n--- Table: openhouse.${dbName}.transaction_logs ---") + + println("Dropping table if exists...") + spark.sql(s"DROP TABLE IF EXISTS openhouse.${dbName}.transaction_logs") + + println("Creating table...") + spark.sql(s""" + CREATE TABLE openhouse.${dbName}.transaction_logs ( + transaction_id STRING, + amount DOUBLE, + status STRING, + transaction_time TIMESTAMP + ) USING iceberg + PARTITIONED BY (hours(transaction_time)) + """).show() + + val statuses = Array("completed", "pending", "failed", "cancelled") + + // INSERT #1: Transactions from last 36 hours (30 transactions) + println("Insert #1: Adding 30 transactions (last 36 hours)...") + for (i <- 1 to 30) { + val ts = randomTimestamp(36) + val amount = 10.0 + Random.nextDouble() * 9990.0 + val status = statuses(Random.nextInt(statuses.length)) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.transaction_logs + VALUES ('txn_${i}', ${amount}, '${status}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // INSERT #2: Transactions from last 48 hours (30 transactions) + println("Insert #2: Adding 30 transactions (last 48 hours)...") + for (i <- 31 to 60) { + val ts = randomTimestamp(48) + val amount = 10.0 + Random.nextDouble() * 9990.0 + val status = statuses(Random.nextInt(statuses.length)) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.transaction_logs + VALUES ('txn_${i}', ${amount}, '${status}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // INSERT #3: Transactions from last 72 hours (30 transactions) + println("Insert #3: Adding 30 transactions (last 72 hours)...") + for (i <- 61 to 90) { + val ts = randomTimestamp(72) + val amount = 10.0 + Random.nextDouble() * 9990.0 + val status = statuses(Random.nextInt(statuses.length)) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.transaction_logs + VALUES ('txn_${i}', ${amount}, '${status}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // INSERT #4: Transactions from last 96 hours (30 transactions) + println("Insert #4: Adding 30 transactions (last 96 hours)...") + for (i <- 91 to 120) { + val ts = randomTimestamp(96) + val amount = 10.0 + Random.nextDouble() * 9990.0 + val status = statuses(Random.nextInt(statuses.length)) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.transaction_logs + VALUES ('txn_${i}', ${amount}, '${status}', TIMESTAMP('${ts}')) + """) + } + println(s"Count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // DELETE #1: Remove all failed transactions (conditional delete across partitions) + println("Delete #1: Removing failed transactions...") + spark.sql(s"DELETE FROM openhouse.${dbName}.transaction_logs WHERE status = 'failed'").show() + println(s"Count after delete: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // INSERT #5: Final batch (20 transactions) + println("Insert #5: Adding 20 transactions...") + for (i <- 121 to 140) { + val ts = randomTimestamp(24) + val amount = 10.0 + Random.nextDouble() * 9990.0 + val status = statuses(Random.nextInt(statuses.length)) + spark.sql(s""" + INSERT INTO openhouse.${dbName}.transaction_logs + VALUES ('txn_${i}', ${amount}, '${status}', TIMESTAMP('${ts}')) + """) + } + println(s"Final count: ${spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.transaction_logs").collect()(0)(0)}") + + // ========================================================================== + // Database Summary + // ========================================================================== + + println("\n" + "-" * 80) + println(s"Database openhouse.${dbName} - Summary") + println("-" * 80) + + println(s"\nAll tables in openhouse.${dbName}:") + spark.sql(s"SHOW TABLES IN openhouse.${dbName}").show(false) + + println(s"\nTable row counts in openhouse.${dbName}:") + val tables = Array("user_profiles", "product_catalog", "clickstream_events", "sensor_readings", "transaction_logs") + for (table <- tables) { + val count = spark.sql(s"SELECT COUNT(*) FROM openhouse.${dbName}.${table}").collect()(0)(0) + println(s" - ${table}: ${count} rows") + } +} + +// ============================================================================ +// Main Execution: Populate All Databases +// ============================================================================ +// Creates and populates tables in 3 databases: testdb, devdb, stagingdb +// ============================================================================ + +println("\nStarting population of 3 databases...") +println("This will create 5 tables in each database (15 tables total)") + +// List of databases to populate +val databases = Array("testdb", "devdb", "stagingdb") + +// Populate each database +for (dbName <- databases) { + try { + populateDatabase(dbName) + println(s"\n✓ Successfully populated openhouse.${dbName}") + } catch { + case e: Exception => + println(s"\n✗ Error populating openhouse.${dbName}: ${e.getMessage}") + e.printStackTrace() + } +} + +// ============================================================================ +// Final Summary +// ============================================================================ + +println("\n" + "=" * 80) +println("FINAL SUMMARY") +println("=" * 80) + +println("\nAll databases in openhouse catalog:") +try { + spark.sql("SHOW DATABASES").show(false) +} catch { + case e: Exception => + println("Unable to list databases (OpenHouse limitation)") + println(s"Populated databases: ${databases.mkString(", ")}") +} + +println("\nTotal tables created:") +for (dbName <- databases) { + try { + val tableCount = spark.sql(s"SHOW TABLES IN openhouse.${dbName}").count() + println(s" - openhouse.${dbName}: ${tableCount} tables") + } catch { + case e: Exception => + println(s" - openhouse.${dbName}: Unable to count tables") + } +} + +println("\n" + "=" * 80) +println("Population Script Complete!") +println("=" * 80) + +println("\nNext steps:") +println(" - List databases: spark.sql(\"SHOW DATABASES\").show()") +println(" - List tables: spark.sql(\"SHOW TABLES IN openhouse.testdb\").show()") +println(" - Query tables: spark.sql(\"SELECT * FROM openhouse.testdb.user_profiles LIMIT 10\").show()") +println(" - View snapshots: spark.sql(\"SELECT * FROM openhouse.testdb.user_profiles.snapshots\").show()") +println(" - View partitions: spark.sql(\"SELECT * FROM openhouse.testdb.clickstream_events.partitions\").show()") +println(" - Time travel: spark.sql(\"SELECT * FROM openhouse.testdb.user_profiles VERSION AS OF \").show()") +println("=" * 80) diff --git a/scripts/run_openhouse_auth_cli.sh b/scripts/run_openhouse_auth_cli.sh new file mode 100644 index 000000000..0a2a639b0 --- /dev/null +++ b/scripts/run_openhouse_auth_cli.sh @@ -0,0 +1,269 @@ +# Script to setup and execute Openhouse token CLI + +# Declare env variables for the current shell +ARTIFACTORY_URL=https://artifactory.corp.linkedin.com:8083/artifactory/ +GROUP_NAME=com.linkedin.li-openhouse +ARTIFACT=auth-token-cli +OPENHOUSE_CLI_HOME="${OPENHOUSE_CLI_HOME:-}" +OPENHOUSE_CLI_CONFIG_PATH="" + +# Download and extract latest openhouse cli distribution +function download_and_extract() { + # If OPENHOUSE_CLI_HOME is already set and valid, skip download + if [ -n "${OPENHOUSE_CLI_HOME}" ] && [ -d "${OPENHOUSE_CLI_HOME}" ]; then + echo_with_time "OPENHOUSE_CLI_HOME already set to: ${OPENHOUSE_CLI_HOME}" + echo_with_time "Skipping download, using local installation" + return 0 + fi + + local latest_version_url="${ARTIFACTORY_URL}api/search/latestVersion?g=${GROUP_NAME}&a=${ARTIFACT}" + echo_with_time $latest_version_url + local version=$(curl -XGET -s $latest_version_url) + echo_with_time "Latest distribution version: ${version}" + # Check if the openhouse cli directory with the latest version exists. If not download the latest distribution + if [ ! -d "${ARTIFACT}-${version}" ]; then + echo_with_time "Downloading the latest distribution version: ${version}" + local tar_distribution="${ARTIFACT}-${version}.tar" + local download_url="${ARTIFACTORY_URL}TOOLS/com/linkedin/li-openhouse/${ARTIFACT}/${version}/${tar_distribution}" + echo_with_time $download_url + wget -q --timeout=60 $download_url + if [ $? -ne 0 ]; then + echo_with_time "Download failed. Exiting." + exit 1 + fi + echo_with_time "Extracting the distribution: ${tar_distribution}" + tar -xf ${tar_distribution} + rm -rf ${tar_distribution} + fi + # Set openhouse token cli home dir + OPENHOUSE_CLI_HOME="${HOME}/${ARTIFACT}-${version}" +} + +# Function to print usage +function usage() { + echo "$0 is for fetching openhouse auth token" + echo "Usage:" + echo "sh $0 [-e|--executingIdentity headless/ldap] [-r|--realIdentity ldap/servicePrincipal] [-f|--tokenFile tokenFile] [-k|--keystore keystore]" + echo "" + echo " -e|--executingIdentity The executing identity i.e. Headless account/ldap user." + echo "" + echo " -r|--realIdentity Real identity urn i.e. service certificate principal or user principal. For example, urn:li:userPrincipal:user. Default: ldap user." + echo "" + echo " -f|--tokenFile Writes Openhouse auth token to this location. If not provided writes to a default tmp location i.e. /tmp/token_[uuid]" + echo "" + echo " -k|--keystore Grestin certificate location provided to KSudo as a proof of identity. If not provided user is + automatically prompted to provide LDAP user and password" + echo " -c|--cluster The openhouse cluster for which auth token will be generated. Default cluster is ltx1-holdem. + Specify lva1-war to generate token for War cluster and ltx1-faro for ei cluster" + echo "" + echo " -stacktrace Print stacktrace of errors." + echo "" + exit 1 +} + +# echo with current timestamp +echo_with_time() { + timestamp=$(date +"%Y-%m-%d %T") + echo "$timestamp" "$@" +} + +# This script is intended to run on gateway machines as well as openhouse jobs scheduler pods. +# The java installation location is /export/apps/jdk/ for both and so setting this as java base path. + +JAVA_BASE="/export/apps/jdk" # change this if needed + +# Function to pick latest matching Java version from a directory +pick_java_version() { + local version_pattern=$1 + ls -d $JAVA_BASE/*${version_pattern}* 2>/dev/null | sort -V | tail -1 +} + +# Function to check if JAVA_HOME points to Java 17 or 11 +is_valid_java_home() { + if [ -z "$JAVA_HOME" ]; then + return 1 + fi + + if [ ! -x "$JAVA_HOME/bin/java" ]; then + return 1 + fi + + local version_output + version_output=$("$JAVA_HOME/bin/java" -version 2>&1) + + if echo "$version_output" | grep -q 'version "17'; then + echo_with_time "Java 17 is already set to JAVA_HOME" + return 0 + fi + if echo "$version_output" | grep -q 'version "11'; then + echo_with_time "Java 11 is already set to JAVA_HOME" + return 0 + fi + + return 1 +} + +# Function to setup java home +setup_java_home() { + echo "Checking existing JAVA_HOME..." + + # If JAVA_HOME is already set AND points to Java 17 or 11, keep it + if is_valid_java_home; then + echo "Existing JAVA_HOME is valid: $JAVA_HOME" + "$JAVA_HOME/bin/java" -version + return 0 + else + echo "JAVA_HOME is not set or not Java 17/11. Selecting version..." + fi + + # Auto-select Java 17, then Java 11 + JAVA17_PATH=$(pick_java_version "17") + + if [ -n "$JAVA17_PATH" ]; then + JAVA_HOME="$JAVA17_PATH" + echo "Java 17 selected: $JAVA_HOME" + else + JAVA11_PATH=$(pick_java_version "11") + if [ -n "$JAVA11_PATH" ]; then + JAVA_HOME="$JAVA11_PATH" + echo "Java 11 selected: $JAVA_HOME" + else + echo "ERROR: No Java 17 or Java 11 found under $JAVA_BASE" + exit 1 + fi + fi + + # Export JAVA_HOME + export JAVA_HOME="$JAVA_HOME" + + echo "Printing JAVA_HOME = $JAVA_HOME" +} + +# This function works for reading root yaml config key, does not work for nested key. This is needed only to read +# zk url which is at the root level. +read_config_key() { + local config=$1 + local key=$2 + grep -E "^${key}:" "$config" \ + | cut -d: -f2- \ + | tr -d ' "' \ + | xargs # trims whitespace +} + +# Extract and export zk url as environment variable +export_zk_url() { + local config=$1 + d2ZkUri=$(read_config_key $config d2ZkUri) + export D2_ZK=$d2ZkUri + echo_with_time "Printing zk url: $D2_ZK" +} + + +# CLI main function +function main() { + + # Setup JAVA_HOME + setup_java_home + + # Download and extract distribution (will skip if OPENHOUSE_CLI_HOME already set) + download_and_extract + + CLI_ARGS="" + KEYSTORE_PROVIDED="false" + EXECUTING_IDENTITY_PROVIDED="false" + # Default cluster for which the token will be generated + CLUSTER="ltx1-holdem" + + # Parse user arguments + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) + usage + exit + ;; + -e|--executingIdentity) + EXECUTING_IDENTITY_PROVIDED="true" + CLI_ARGS="${CLI_ARGS} $1 $2" + shift + shift + ;; + -r|--realIdentity) + CLI_ARGS="${CLI_ARGS} $1 '$2'" + shift + shift + ;; + -f|--tokenFile) + # If user specifies cluster, use that + CLI_ARGS="${CLI_ARGS} $1 $2" + shift + shift + ;; + -k|--keystore) + KEYSTORE_PROVIDED="true" + CLI_ARGS="${CLI_ARGS} $1 $2" + shift + shift + ;; + -c|--cluster) + CLUSTER="$2" + shift + shift + ;; + -stacktrace) + CLI_ARGS="${CLI_ARGS} $1" + shift + shift + ;; + --) # End argument parsing + shift + break + ;; + *) + CLI_ARGS="${CLI_ARGS} '$1'" + shift + ;; + esac + done + + if [ ${EXECUTING_IDENTITY_PROVIDED} == "false" ]; then + CURRENT_USER="${USER:-$(whoami)}" + CLI_ARGS="${CLI_ARGS} -e ${CURRENT_USER}" + fi + + # Set classpath + if [ -z ${CLASSPATH} ]; then + CLASSPATH="${OPENHOUSE_CLI_HOME}/lib/*:/export/apps/hadoop/site/etc/hadoop" + else + CLASSPATH="${OPENHOUSE_CLI_HOME}/lib/*:/export/apps/hadoop/site/etc/hadoop:${CLASSPATH:-}" + fi + echo_with_time "Printing classpath: $CLASSPATH" + + # set log4j config + LOG4J_CONF="file:${OPENHOUSE_CLI_HOME}/conf/log4j.properties" + JAVA_OPTS="${JAVA_OPTS} -Djava.net.preferIPv4Stack=true" + + # set openhouse cli config path + if [[ ${CLUSTER} == "ltx1-holdem" || ${CLUSTER} == "lva1-war" || ${CLUSTER} == "ltx1-lasso" || ${CLUSTER} == "ltx1-yugioh" ]]; then + export OPENHOUSE_CLI_CONFIG_PATH="${OPENHOUSE_CLI_HOME}/conf/${CLUSTER}/ksudo_prod.yaml" + export_zk_url $OPENHOUSE_CLI_CONFIG_PATH + elif [[ ${CLUSTER} == "ltx1-faro" ]]; then + export OPENHOUSE_CLI_CONFIG_PATH="${OPENHOUSE_CLI_HOME}/conf/${CLUSTER}/ksudo_ei.yaml" + export_zk_url $OPENHOUSE_CLI_CONFIG_PATH + else + echo_with_time "Cluster: $CLUSTER is invalid. Specify a valid cluster from (ltx1-faro, ltx1-holdem, lva1-war, ltx1-lasso, ltx1-yugioh)" + exit 1 + fi + + + CMD="" + if [ ${KEYSTORE_PROVIDED} == "true" ]; then + CMD="$JAVA_HOME/bin/java ${JAVA_OPTS} -cp ${CLASSPATH} com.linkedin.openhouse.authentication.token.cli.OpenhouseTokenCli ${CLI_ARGS}" + else + CURRENT_USER="${USER:-$(whoami)}" + CMD="$JAVA_HOME/bin/java ${JAVA_OPTS} -cp ${CLASSPATH} com.linkedin.openhouse.authentication.token.cli.OpenhouseTokenCli ${CLI_ARGS} --ldap ${CURRENT_USER}" + fi + echo_with_time "Executing command: ${CMD}" + eval "${CMD}" +} + +main "$@" diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/JobsSpringApplication.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/JobsSpringApplication.java index 6b74f9f49..cecdddf4c 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/JobsSpringApplication.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/JobsSpringApplication.java @@ -5,6 +5,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.scheduling.annotation.EnableScheduling; @SuppressWarnings({"PMD", "checkstyle:hideutilityclassconstructor"}) @SpringBootApplication( @@ -17,6 +18,7 @@ }) @EntityScan(basePackages = {"com.linkedin.openhouse.jobs.model"}) @EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class}) +@EnableScheduling public class JobsSpringApplication { public static void main(String[] args) { diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/JobsApiHandler.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/JobsApiHandler.java index 89bb4331e..0bbaf302e 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/JobsApiHandler.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/JobsApiHandler.java @@ -3,6 +3,7 @@ import com.linkedin.openhouse.common.api.spec.ApiResponse; import com.linkedin.openhouse.jobs.api.spec.request.CreateJobRequestBody; import com.linkedin.openhouse.jobs.api.spec.response.JobResponseBody; +import com.linkedin.openhouse.jobs.api.spec.response.JobSearchResponseBody; /** * Interface layer between REST and Jobs backend. The implementation is injected into the Service @@ -32,4 +33,14 @@ public interface JobsApiHandler { * @return empty body on successful cancellation */ ApiResponse cancel(String jobId); + + /** + * Function to search for jobs by job name prefix with pagination + * + * @param jobNamePrefix prefix to search for in job names + * @param limit maximum number of results to return + * @param offset number of results to skip + * @return the search response body containing list of jobs and pagination metadata + */ + ApiResponse search(String jobNamePrefix, int limit, int offset); } diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/impl/OpenHouseJobsApiHandler.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/impl/OpenHouseJobsApiHandler.java index 8fad4c29b..2ca2bd012 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/impl/OpenHouseJobsApiHandler.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/handler/impl/OpenHouseJobsApiHandler.java @@ -4,10 +4,13 @@ import com.linkedin.openhouse.jobs.api.handler.JobsApiHandler; import com.linkedin.openhouse.jobs.api.spec.request.CreateJobRequestBody; import com.linkedin.openhouse.jobs.api.spec.response.JobResponseBody; +import com.linkedin.openhouse.jobs.api.spec.response.JobSearchResponseBody; import com.linkedin.openhouse.jobs.api.validator.JobsApiValidator; import com.linkedin.openhouse.jobs.dto.mapper.JobsMapper; import com.linkedin.openhouse.jobs.model.JobDto; import com.linkedin.openhouse.jobs.services.JobsService; +import java.util.List; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; @@ -41,4 +44,22 @@ public ApiResponse cancel(String jobId) { service.cancel(jobId); return ApiResponse.builder().httpStatus(HttpStatus.NO_CONTENT).build(); } + + @Override + public ApiResponse search(String jobNamePrefix, int limit, int offset) { + List jobs = service.search(jobNamePrefix, limit, offset); + long totalCount = service.count(jobNamePrefix); + List results = + jobs.stream().map(mapper::toGetJobResponseBody).collect(Collectors.toList()); + return ApiResponse.builder() + .httpStatus(HttpStatus.OK) + .responseBody( + JobSearchResponseBody.builder() + .results(results) + .totalCount(totalCount) + .offset(offset) + .limit(limit) + .build()) + .build(); + } } diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/spec/response/JobSearchResponseBody.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/spec/response/JobSearchResponseBody.java new file mode 100644 index 000000000..efdff312f --- /dev/null +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/api/spec/response/JobSearchResponseBody.java @@ -0,0 +1,29 @@ +package com.linkedin.openhouse.jobs.api.spec.response; + +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Builder +@Getter +@EqualsAndHashCode +@AllArgsConstructor(access = AccessLevel.PROTECTED) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class JobSearchResponseBody { + @Schema(description = "List of jobs matching the search criteria") + private List results; + + @Schema(description = "Total count of jobs matching the criteria") + private Long totalCount; + + @Schema(description = "Number of results skipped") + private Integer offset; + + @Schema(description = "Maximum number of results returned") + private Integer limit; +} diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/controller/JobsController.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/controller/JobsController.java index 6132a43ef..89591368d 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/controller/JobsController.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/controller/JobsController.java @@ -4,6 +4,7 @@ import com.linkedin.openhouse.jobs.api.handler.JobsApiHandler; import com.linkedin.openhouse.jobs.api.spec.request.CreateJobRequestBody; import com.linkedin.openhouse.jobs.api.spec.response.JobResponseBody; +import com.linkedin.openhouse.jobs.api.spec.response.JobSearchResponseBody; import io.micrometer.core.annotation.Timed; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** Class defining all the REST Endpoints for /jobs endpoint. */ @@ -24,6 +26,37 @@ public class JobsController { @Autowired private JobsApiHandler jobsApiHandler; + @Operation( + summary = "Search Jobs", + description = "Search for jobs by job name prefix.", + tags = {"Job"}) + @ApiResponses( + value = { + @ApiResponse(responseCode = "200", description = "Job Search: OK"), + @ApiResponse(responseCode = "400", description = "Job Search: BAD_REQUEST") + }) + @GetMapping( + value = "/jobs", + produces = {"application/json"}) + @Timed( + value = MetricsConstant.REQUEST, + extraTags = {"service", MetricsConstant.JOBS_SERVICE, "action", "search"}) + public ResponseEntity searchJobs( + @Parameter(description = "Job name prefix", required = true) @RequestParam + String jobNamePrefix, + @Parameter(description = "Maximum number of results", required = false) + @RequestParam(defaultValue = "10") + int limit, + @Parameter(description = "Number of results to skip", required = false) + @RequestParam(defaultValue = "0") + int offset) { + + com.linkedin.openhouse.common.api.spec.ApiResponse apiResponse = + jobsApiHandler.search(jobNamePrefix, limit, offset); + return new ResponseEntity<>( + apiResponse.getResponseBody(), apiResponse.getHttpHeaders(), apiResponse.getHttpStatus()); + } + @Operation( summary = "Get Job", description = "Returns a Job resource identified by jobId.", diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepository.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepository.java index c774e8d62..86174f12d 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepository.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepository.java @@ -2,8 +2,14 @@ import com.linkedin.openhouse.jobs.model.JobDto; import com.linkedin.openhouse.jobs.model.JobDtoPrimaryKey; +import java.util.List; +import org.springframework.data.domain.Pageable; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; @Repository -public interface JobsInternalRepository extends CrudRepository {} +public interface JobsInternalRepository extends CrudRepository { + List findByJobNameStartingWith(String prefix, Pageable pageable); + + long countByJobNameStartingWith(String prefix); +} diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepositoryImpl.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepositoryImpl.java index 84156d0be..d87e96619 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepositoryImpl.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/repository/JobsInternalRepositoryImpl.java @@ -16,9 +16,12 @@ import com.linkedin.openhouse.jobs.repository.exception.JobsTableConcurrentUpdateException; import io.netty.resolver.dns.DnsNameResolverTimeoutException; import java.time.Duration; +import java.util.Comparator; +import java.util.List; import java.util.Optional; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Pageable; import org.springframework.retry.support.RetryTemplate; import org.springframework.retry.support.RetryTemplateBuilder; import org.springframework.stereotype.Component; @@ -119,6 +122,43 @@ public Iterable findAll() { .get()); } + @Override + public List findByJobNameStartingWith(String prefix, Pageable pageable) { + return getHtsRetryTemplate() + .execute( + context -> + jobApi + .getAllJobs(ImmutableMap.of()) + .map(GetAllEntityResponseBodyJob::getResults) + .flatMapMany(Flux::fromIterable) + .map(jobsMapper::toJobDto) + .filter(job -> job.getJobName().startsWith(prefix)) + .sort(Comparator.comparing(JobDto::getCreationTimeMs).reversed()) + .skip((long) pageable.getPageNumber() * pageable.getPageSize()) + .take(pageable.getPageSize()) + .onErrorResume(this::handleHtsHttpError) + .collectList() + .blockOptional(Duration.ofSeconds(REQUEST_TIMEOUT_SECONDS)) + .get()); + } + + @Override + public long countByJobNameStartingWith(String prefix) { + return getHtsRetryTemplate() + .execute( + context -> + jobApi + .getAllJobs(ImmutableMap.of()) + .map(GetAllEntityResponseBodyJob::getResults) + .flatMapMany(Flux::fromIterable) + .map(jobsMapper::toJobDto) + .filter(job -> job.getJobName().startsWith(prefix)) + .count() + .onErrorResume(e -> Mono.just(0L)) + .blockOptional(Duration.ofSeconds(REQUEST_TIMEOUT_SECONDS)) + .orElse(0L)); + } + private Optional getCurrentVersion(String jobId) { return getHtsRetryTemplate() .execute( diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/scheduler/OrphanJobCleanupTask.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/scheduler/OrphanJobCleanupTask.java new file mode 100644 index 000000000..7d5c9f3ea --- /dev/null +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/scheduler/OrphanJobCleanupTask.java @@ -0,0 +1,151 @@ +package com.linkedin.openhouse.jobs.scheduler; + +import com.linkedin.openhouse.common.JobState; +import com.linkedin.openhouse.jobs.model.JobDto; +import com.linkedin.openhouse.jobs.repository.JobsInternalRepository; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Background task that runs periodically to detect and mark orphaned jobs as FAILED. + * + *

An orphaned job is one that: + * + *

    + *
  • Has a non-terminal state (not SUCCEEDED, FAILED, or CANCELLED) + *
  • Has not received a heartbeat/update for more than 10 minutes + *
+ * + *

These jobs typically occur when a job fails to start in the execution engine (e.g., Livy) and + * therefore never sends state updates or heartbeats. + */ +@Slf4j +@Component +@ConditionalOnProperty( + value = "scheduler.orphan-job-cleanup.enabled", + havingValue = "true", + matchIfMissing = true) +public class OrphanJobCleanupTask { + + private static final long ORPHAN_THRESHOLD_MS = 10 * 60 * 1000; // 10 minutes in milliseconds + + @Autowired private JobsInternalRepository jobsRepository; + + public OrphanJobCleanupTask() { + System.out.println("========================================"); + System.out.println("OrphanJobCleanupTask bean created - scheduler will start in 5 minutes"); + System.out.println("========================================"); + } + + /** + * Scheduled task that runs every 5 minutes to detect and mark orphaned jobs as FAILED. + * + *

Execution schedule: - Initial delay: 5 minutes (to allow service to fully start) - Fixed + * rate: 5 minutes + */ + @Scheduled(initialDelay = 5 * 60 * 1000, fixedRate = 5 * 60 * 1000) + public void cleanupOrphanedJobs() { + System.out.println("========================================"); + System.out.println("OrphanJobCleanupTask: Starting orphaned job cleanup task"); + System.out.println("========================================"); + log.info("Starting orphaned job cleanup task"); + long startTime = System.currentTimeMillis(); + + try { + // Fetch all jobs from housetables + Iterable allJobs = jobsRepository.findAll(); + long currentTimeMs = Instant.now().toEpochMilli(); + long orphanThresholdTimeMs = currentTimeMs - ORPHAN_THRESHOLD_MS; + + // Filter for orphaned jobs: non-terminal state and stale lastUpdateTimeMs + List orphanedJobs = + StreamSupport.stream(allJobs.spliterator(), false) + .filter(job -> !job.getState().isTerminal()) + .filter(job -> job.getLastUpdateTimeMs() < orphanThresholdTimeMs) + .collect(Collectors.toList()); + + if (orphanedJobs.isEmpty()) { + System.out.println("OrphanJobCleanupTask: No orphaned jobs found"); + log.info("No orphaned jobs found"); + return; + } + + System.out.println( + "OrphanJobCleanupTask: Found " + + orphanedJobs.size() + + " orphaned jobs to mark as FAILED"); + log.info("Found {} orphaned jobs to mark as FAILED", orphanedJobs.size()); + + // Update each orphaned job to FAILED state + int successCount = 0; + int failureCount = 0; + + for (JobDto orphanedJob : orphanedJobs) { + try { + long minutesSinceUpdate = + (currentTimeMs - orphanedJob.getLastUpdateTimeMs()) / (60 * 1000); + + log.info( + "Marking orphaned job as FAILED: jobId={}, jobName={}, state={}, " + + "minutesSinceLastUpdate={}, executionId={}", + orphanedJob.getJobId(), + orphanedJob.getJobName(), + orphanedJob.getState(), + minutesSinceUpdate, + orphanedJob.getExecutionId()); + + // Create updated job with FAILED state + JobDto updatedJob = + orphanedJob + .toBuilder() + .state(JobState.FAILED) + .lastUpdateTimeMs(currentTimeMs) + .finishTimeMs(currentTimeMs) + .build(); + + jobsRepository.save(updatedJob); + successCount++; + + log.info("Successfully marked job {} as FAILED", orphanedJob.getJobId()); + + } catch (Exception e) { + failureCount++; + log.error( + "Failed to mark orphaned job {} as FAILED: {}", + orphanedJob.getJobId(), + e.getMessage(), + e); + } + } + + long duration = System.currentTimeMillis() - startTime; + System.out.println( + "OrphanJobCleanupTask: Completed in " + + duration + + "ms. Success: " + + successCount + + ", Failures: " + + failureCount + + ", Total: " + + orphanedJobs.size()); + log.info( + "Orphaned job cleanup completed in {}ms. Success: {}, Failures: {}, Total: {}", + duration, + successCount, + failureCount, + orphanedJobs.size()); + + } catch (Exception e) { + System.err.println("OrphanJobCleanupTask ERROR: " + e.getMessage()); + e.printStackTrace(); + log.error("Error during orphaned job cleanup task: {}", e.getMessage(), e); + } + } +} diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsService.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsService.java index 4d2a565a8..499254be8 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsService.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsService.java @@ -2,6 +2,7 @@ import com.linkedin.openhouse.jobs.api.spec.request.CreateJobRequestBody; import com.linkedin.openhouse.jobs.model.JobDto; +import java.util.List; /** Service interface for implementing /jobs endpoint. */ public interface JobsService { @@ -28,4 +29,22 @@ public interface JobsService { * @param jobId unique job identifier */ void cancel(String jobId); + + /** + * Search for jobs by job name prefix with pagination. + * + * @param jobNamePrefix prefix to search for in job names + * @param limit maximum number of results to return + * @param offset number of results to skip + * @return List of JobDto objects matching the prefix + */ + List search(String jobNamePrefix, int limit, int offset); + + /** + * Count total jobs matching the job name prefix. + * + * @param jobNamePrefix prefix to search for in job names + * @return total count of matching jobs + */ + long count(String jobNamePrefix); } diff --git a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsServiceImpl.java b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsServiceImpl.java index 626ed3126..1bff761ee 100644 --- a/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsServiceImpl.java +++ b/services/jobs/src/main/java/com/linkedin/openhouse/jobs/services/JobsServiceImpl.java @@ -12,9 +12,11 @@ import com.linkedin.openhouse.jobs.model.JobDtoPrimaryKey; import com.linkedin.openhouse.jobs.repository.JobsInternalRepository; import java.time.Instant; +import java.util.List; import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; @Slf4j @@ -95,4 +97,17 @@ private boolean shouldCancel(JobDto job) { private String generateJobId(String jobName) { return jobName + "_" + UUID.randomUUID(); } + + @Override + public List search(String jobNamePrefix, int limit, int offset) { + METRICS_REPORTER.count(MetricsConstant.REQUEST_COUNT, MetricsConstant.ACTION_TAG, "search"); + return repository.findByJobNameStartingWith( + jobNamePrefix, PageRequest.of(offset / limit, limit)); + } + + @Override + public long count(String jobNamePrefix) { + METRICS_REPORTER.count(MetricsConstant.REQUEST_COUNT, MetricsConstant.ACTION_TAG, "count"); + return repository.countByJobNameStartingWith(jobNamePrefix); + } } diff --git a/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/MockJobsApiHandler.java b/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/MockJobsApiHandler.java index 6fd0c06b4..d827a532d 100644 --- a/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/MockJobsApiHandler.java +++ b/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/MockJobsApiHandler.java @@ -7,6 +7,8 @@ import com.linkedin.openhouse.jobs.api.handler.JobsApiHandler; import com.linkedin.openhouse.jobs.api.spec.request.CreateJobRequestBody; import com.linkedin.openhouse.jobs.api.spec.response.JobResponseBody; +import com.linkedin.openhouse.jobs.api.spec.response.JobSearchResponseBody; +import java.util.ArrayList; import org.springframework.context.annotation.Primary; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; @@ -62,4 +64,19 @@ public ApiResponse cancel(String jobId) { return null; } } + + @Override + public ApiResponse search(String jobNamePrefix, int limit, int offset) { + JobSearchResponseBody responseBody = + JobSearchResponseBody.builder() + .results(new ArrayList<>()) + .totalCount(0L) + .offset(offset) + .limit(limit) + .build(); + return ApiResponse.builder() + .httpStatus(HttpStatus.OK) + .responseBody(responseBody) + .build(); + } } diff --git a/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/OrphanJobCleanupTaskTest.java b/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/OrphanJobCleanupTaskTest.java new file mode 100644 index 000000000..890fd1c57 --- /dev/null +++ b/services/jobs/src/test/java/com/linkedin/openhouse/jobs/mock/OrphanJobCleanupTaskTest.java @@ -0,0 +1,240 @@ +package com.linkedin.openhouse.jobs.mock; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.linkedin.openhouse.common.JobState; +import com.linkedin.openhouse.jobs.model.JobDto; +import com.linkedin.openhouse.jobs.repository.JobsInternalRepository; +import com.linkedin.openhouse.jobs.scheduler.OrphanJobCleanupTask; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class OrphanJobCleanupTaskTest { + + @Mock private JobsInternalRepository jobsRepository; + + @InjectMocks private OrphanJobCleanupTask cleanupTask; + + private static final long CURRENT_TIME_MS = Instant.now().toEpochMilli(); + private static final long ELEVEN_MINUTES_AGO = CURRENT_TIME_MS - (11 * 60 * 1000); + private static final long FIVE_MINUTES_AGO = CURRENT_TIME_MS - (5 * 60 * 1000); + + @BeforeEach + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testCleanupOrphanedJobsOrphanedQueuedJob() { + // Given: A job in QUEUED state that hasn't been updated for 11 minutes (orphaned) + JobDto orphanedJob = + JobDto.builder() + .jobId("orphaned-job-1") + .jobName("test-job") + .state(JobState.QUEUED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .creationTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build(); + + when(jobsRepository.findAll()).thenReturn(Arrays.asList(orphanedJob)); + when(jobsRepository.save(any(JobDto.class))).thenAnswer(i -> i.getArguments()[0]); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: Job should be marked as FAILED + ArgumentCaptor captor = ArgumentCaptor.forClass(JobDto.class); + verify(jobsRepository, times(1)).save(captor.capture()); + + JobDto savedJob = captor.getValue(); + assert savedJob.getState() == JobState.FAILED; + assert savedJob.getJobId().equals("orphaned-job-1"); + } + + @Test + public void testCleanupOrphanedJobsOrphanedRunningJob() { + // Given: A job in RUNNING state that hasn't been updated for 15 minutes (orphaned) + JobDto orphanedJob = + JobDto.builder() + .jobId("orphaned-job-2") + .jobName("test-job") + .state(JobState.RUNNING) + .lastUpdateTimeMs(CURRENT_TIME_MS - (15 * 60 * 1000)) + .creationTimeMs(CURRENT_TIME_MS - (20 * 60 * 1000)) + .clusterId("cluster1") + .build(); + + when(jobsRepository.findAll()).thenReturn(Arrays.asList(orphanedJob)); + when(jobsRepository.save(any(JobDto.class))).thenAnswer(i -> i.getArguments()[0]); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: Job should be marked as FAILED + ArgumentCaptor captor = ArgumentCaptor.forClass(JobDto.class); + verify(jobsRepository, times(1)).save(captor.capture()); + + JobDto savedJob = captor.getValue(); + assert savedJob.getState() == JobState.FAILED; + } + + @Test + public void testCleanupOrphanedJobsRecentJobNotOrphaned() { + // Given: A job in QUEUED state that was updated 5 minutes ago (NOT orphaned) + JobDto recentJob = + JobDto.builder() + .jobId("recent-job") + .jobName("test-job") + .state(JobState.QUEUED) + .lastUpdateTimeMs(FIVE_MINUTES_AGO) + .creationTimeMs(FIVE_MINUTES_AGO) + .clusterId("cluster1") + .build(); + + when(jobsRepository.findAll()).thenReturn(Arrays.asList(recentJob)); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: Job should NOT be updated + verify(jobsRepository, never()).save(any(JobDto.class)); + } + + @Test + public void testCleanupOrphanedJobsTerminalStateNotOrphaned() { + // Given: Jobs in terminal states (SUCCEEDED, FAILED, CANCELLED) with old timestamps + List terminalJobs = + Arrays.asList( + JobDto.builder() + .jobId("succeeded-job") + .state(JobState.SUCCEEDED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build(), + JobDto.builder() + .jobId("failed-job") + .state(JobState.FAILED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build(), + JobDto.builder() + .jobId("cancelled-job") + .state(JobState.CANCELLED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build()); + + when(jobsRepository.findAll()).thenReturn(terminalJobs); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: No jobs should be updated (terminal states are not orphans) + verify(jobsRepository, never()).save(any(JobDto.class)); + } + + @Test + public void testCleanupOrphanedJobsMixedJobs() { + // Given: Mix of orphaned and non-orphaned jobs + List jobs = + Arrays.asList( + // Orphaned jobs (should be marked as FAILED) + JobDto.builder() + .jobId("orphan-1") + .state(JobState.QUEUED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build(), + JobDto.builder() + .jobId("orphan-2") + .state(JobState.RUNNING) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build(), + // Non-orphaned jobs (should NOT be updated) + JobDto.builder() + .jobId("recent") + .state(JobState.QUEUED) + .lastUpdateTimeMs(FIVE_MINUTES_AGO) + .clusterId("cluster1") + .build(), + JobDto.builder() + .jobId("terminal") + .state(JobState.SUCCEEDED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .clusterId("cluster1") + .build()); + + when(jobsRepository.findAll()).thenReturn(jobs); + when(jobsRepository.save(any(JobDto.class))).thenAnswer(i -> i.getArguments()[0]); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: Only 2 orphaned jobs should be marked as FAILED + ArgumentCaptor captor = ArgumentCaptor.forClass(JobDto.class); + verify(jobsRepository, times(2)).save(captor.capture()); + + List savedJobs = captor.getAllValues(); + assert savedJobs.size() == 2; + assert savedJobs.stream().allMatch(job -> job.getState() == JobState.FAILED); + assert savedJobs.stream().anyMatch(job -> job.getJobId().equals("orphan-1")); + assert savedJobs.stream().anyMatch(job -> job.getJobId().equals("orphan-2")); + } + + @Test + public void testCleanupOrphanedJobsEmptyJobList() { + // Given: No jobs in the system + when(jobsRepository.findAll()).thenReturn(Arrays.asList()); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: No updates should be made + verify(jobsRepository, never()).save(any(JobDto.class)); + } + + @Test + public void testCleanupOrphanedJobsUpdatedJobHasCorrectTimestamps() { + // Given: An orphaned job + JobDto orphanedJob = + JobDto.builder() + .jobId("orphan-with-timestamps") + .state(JobState.QUEUED) + .lastUpdateTimeMs(ELEVEN_MINUTES_AGO) + .creationTimeMs(CURRENT_TIME_MS - (30 * 60 * 1000)) + .startTimeMs(0L) + .finishTimeMs(0L) + .clusterId("cluster1") + .build(); + + when(jobsRepository.findAll()).thenReturn(Arrays.asList(orphanedJob)); + when(jobsRepository.save(any(JobDto.class))).thenAnswer(i -> i.getArguments()[0]); + + // When: Cleanup task runs + cleanupTask.cleanupOrphanedJobs(); + + // Then: Updated job should have current timestamps for lastUpdateTimeMs and finishTimeMs + ArgumentCaptor captor = ArgumentCaptor.forClass(JobDto.class); + verify(jobsRepository, times(1)).save(captor.capture()); + + JobDto savedJob = captor.getValue(); + assert savedJob.getState() == JobState.FAILED; + assert savedJob.getLastUpdateTimeMs() > ELEVEN_MINUTES_AGO; + assert savedJob.getFinishTimeMs() > ELEVEN_MINUTES_AGO; + // Creation and start times should be preserved + assert savedJob.getCreationTimeMs() == orphanedJob.getCreationTimeMs(); + } +} diff --git a/services/jobs/src/test/resources/application.properties b/services/jobs/src/test/resources/application.properties new file mode 100644 index 000000000..3545deec5 --- /dev/null +++ b/services/jobs/src/test/resources/application.properties @@ -0,0 +1,2 @@ +# Disable orphan job cleanup scheduler in tests +scheduler.orphan-job-cleanup.enabled=false diff --git a/services/spectacle/.gitignore b/services/spectacle/.gitignore new file mode 100644 index 000000000..892067bc7 --- /dev/null +++ b/services/spectacle/.gitignore @@ -0,0 +1,33 @@ +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/services/spectacle/README.md b/services/spectacle/README.md new file mode 100644 index 000000000..03cb730d7 --- /dev/null +++ b/services/spectacle/README.md @@ -0,0 +1,106 @@ +# OpenHouse Spectacle Application + +A Next.js web application for searching and exploring OpenHouse tables. + +## Features + +- Built with Next.js 14 and React 18 +- TypeScript support +- Standalone output for Docker deployment +- Modern, responsive design +- Search tables by database ID +- Filter results in real-time +- Display table metadata (ID, type, creator, last modified time) + +## Development + +### Prerequisites + +- Node.js 20 or higher +- npm + +### Local Development + +1. Install dependencies: +```bash +npm install +``` + +2. Run the development server: +```bash +npm run dev +``` + +3. Open [http://localhost:3000](http://localhost:3000) in your browser. + +### Build + +```bash +npm run build +``` + +### Production + +```bash +npm start +``` + +## Docker Deployment + +### Build and Run with Docker Compose + +From the root of the OpenHouse repository: + +```bash +docker-compose -f infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.spectacle.yml up --build +``` + +This will: +1. Build and start the tables-service on port 8000 +2. Build and start the jobs-service on port 8002 +3. Build and start the spectacle-service on port 3000 (depends on the above services) + +### Access the Application + +Once all services are running, access the web application at: +- **Spectacle App**: http://localhost:3000 +- **Tables Service**: http://localhost:8000 +- **Jobs Service**: http://localhost:8002 + +### Stop Services + +```bash +docker-compose -f infra/recipes/docker-compose/oh-hadoop-spark/docker-compose.spectacle.yml down +``` + +## Project Structure + +``` +services/spectacle/ +├── src/ +│ └── app/ +│ ├── layout.tsx # Root layout component +│ └── page.tsx # Home page with "Hello World" +├── package.json # Dependencies and scripts +├── next.config.js # Next.js configuration +├── tsconfig.json # TypeScript configuration +└── README.md # This file +``` + +## Environment Variables + +The following environment variables are available: + +- `NEXT_PUBLIC_TABLES_SERVICE_URL`: URL for the tables service API (default: http://localhost:8000) +- `NEXT_PUBLIC_JOBS_SERVICE_URL`: URL for the jobs service API (default: http://localhost:8002) + +**Note:** Environment variables prefixed with `NEXT_PUBLIC_` are exposed to the browser and can be used in client-side code. + +## Usage + +1. **Start the services** using Docker Compose (see Docker Deployment section above) +2. **Open the web app** at http://localhost:3000 +3. **Enter a database ID** in the search box (e.g., "my_database") +4. **Click Search** to fetch all tables in that database +5. **Use the filter box** to narrow down results by table name or database ID +6. **View table details** including ID, type, creator, and last modified time diff --git a/services/spectacle/next.config.js b/services/spectacle/next.config.js new file mode 100644 index 000000000..c89f28a8b --- /dev/null +++ b/services/spectacle/next.config.js @@ -0,0 +1,7 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + output: 'standalone', + reactStrictMode: true, +} + +module.exports = nextConfig diff --git a/services/spectacle/package-lock.json b/services/spectacle/package-lock.json new file mode 100644 index 000000000..397fdcf8c --- /dev/null +++ b/services/spectacle/package-lock.json @@ -0,0 +1,747 @@ +{ + "name": "openhouse-spectacle", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "openhouse-spectacle", + "version": "1.0.0", + "dependencies": { + "next": "^14.2.35", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "20.10.6", + "@types/react": "18.2.46", + "@types/react-dom": "18.2.18", + "typescript": "5.3.3" + } + }, + "node_modules/@next/env": { + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.35.tgz", + "integrity": "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==", + "license": "MIT" + }, + "node_modules/@next/swc-darwin-arm64": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.33.tgz", + "integrity": "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.33.tgz", + "integrity": "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.33.tgz", + "integrity": "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.33.tgz", + "integrity": "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.33.tgz", + "integrity": "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.33.tgz", + "integrity": "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.33.tgz", + "integrity": "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", + "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.33.tgz", + "integrity": "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", + "license": "Apache-2.0" + }, + "node_modules/@swc/helpers": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz", + "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==", + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@types/node": { + "version": "20.10.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.6.tgz", + "integrity": "sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true + }, + "node_modules/@types/react": { + "version": "18.2.46", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.46.tgz", + "integrity": "sha512-nNCvVBcZlvX4NU1nRRNV/mFl1nNRuTuslAJglQsq+8ldXe5Xv0Wd2f7WTE3jOxhLH2BFfiZGC6GCp+kHQbgG+w==", + "dev": true, + "dependencies": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.2.18", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.18.tgz", + "integrity": "sha512-TJxDm6OfAX2KJWJdMEVTwWke5Sc/E/RlnPGvGfS0W7+6ocy2xhDVQVh/KvC2Uf7kACs+gDytdusDSdWfWkaNzw==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/scheduler": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", + "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", + "dev": true + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001764", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001764.tgz", + "integrity": "sha512-9JGuzl2M+vPL+pz70gtMF9sHdMFbY9FJaQBi186cHKH3pSzDvzoUJUPV6fqiKIMyXbud9ZLg4F3Yza1vJ1+93g==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/next": { + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.35.tgz", + "integrity": "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==", + "license": "MIT", + "dependencies": { + "@next/env": "14.2.35", + "@swc/helpers": "0.5.5", + "busboy": "1.6.0", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", + "postcss": "8.4.31", + "styled-jsx": "5.1.1" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": ">=18.17.0" + }, + "optionalDependencies": { + "@next/swc-darwin-arm64": "14.2.33", + "@next/swc-darwin-x64": "14.2.33", + "@next/swc-linux-arm64-gnu": "14.2.33", + "@next/swc-linux-arm64-musl": "14.2.33", + "@next/swc-linux-x64-gnu": "14.2.33", + "@next/swc-linux-x64-musl": "14.2.33", + "@next/swc-win32-arm64-msvc": "14.2.33", + "@next/swc-win32-ia32-msvc": "14.2.33", + "@next/swc-win32-x64-msvc": "14.2.33" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.41.2", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@playwright/test": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "node_modules/postcss": { + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", + "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.0" + }, + "peerDependencies": { + "react": "^18.2.0" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/styled-jsx": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz", + "integrity": "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==", + "dependencies": { + "client-only": "0.0.1" + }, + "engines": { + "node": ">= 12.0.0" + }, + "peerDependencies": { + "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + }, + "dependencies": { + "@next/env": { + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.35.tgz", + "integrity": "sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==" + }, + "@next/swc-darwin-arm64": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.33.tgz", + "integrity": "sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA==", + "optional": true + }, + "@next/swc-darwin-x64": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.33.tgz", + "integrity": "sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA==", + "optional": true + }, + "@next/swc-linux-arm64-gnu": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.33.tgz", + "integrity": "sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw==", + "optional": true + }, + "@next/swc-linux-arm64-musl": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.33.tgz", + "integrity": "sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==", + "optional": true + }, + "@next/swc-linux-x64-gnu": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.33.tgz", + "integrity": "sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==", + "optional": true + }, + "@next/swc-linux-x64-musl": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.33.tgz", + "integrity": "sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==", + "optional": true + }, + "@next/swc-win32-arm64-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.33.tgz", + "integrity": "sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==", + "optional": true + }, + "@next/swc-win32-ia32-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", + "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", + "optional": true + }, + "@next/swc-win32-x64-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.33.tgz", + "integrity": "sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg==", + "optional": true + }, + "@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==" + }, + "@swc/helpers": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz", + "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==", + "requires": { + "@swc/counter": "^0.1.3", + "tslib": "^2.4.0" + } + }, + "@types/node": { + "version": "20.10.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.6.tgz", + "integrity": "sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } + }, + "@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true + }, + "@types/react": { + "version": "18.2.46", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.46.tgz", + "integrity": "sha512-nNCvVBcZlvX4NU1nRRNV/mFl1nNRuTuslAJglQsq+8ldXe5Xv0Wd2f7WTE3jOxhLH2BFfiZGC6GCp+kHQbgG+w==", + "dev": true, + "requires": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "@types/react-dom": { + "version": "18.2.18", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.18.tgz", + "integrity": "sha512-TJxDm6OfAX2KJWJdMEVTwWke5Sc/E/RlnPGvGfS0W7+6ocy2xhDVQVh/KvC2Uf7kACs+gDytdusDSdWfWkaNzw==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, + "@types/scheduler": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", + "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", + "dev": true + }, + "busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "requires": { + "streamsearch": "^1.1.0" + } + }, + "caniuse-lite": { + "version": "1.0.30001764", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001764.tgz", + "integrity": "sha512-9JGuzl2M+vPL+pz70gtMF9sHdMFbY9FJaQBi186cHKH3pSzDvzoUJUPV6fqiKIMyXbud9ZLg4F3Yza1vJ1+93g==" + }, + "client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" + }, + "csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true + }, + "graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==" + }, + "next": { + "version": "14.2.35", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.35.tgz", + "integrity": "sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==", + "requires": { + "@next/env": "14.2.35", + "@next/swc-darwin-arm64": "14.2.33", + "@next/swc-darwin-x64": "14.2.33", + "@next/swc-linux-arm64-gnu": "14.2.33", + "@next/swc-linux-arm64-musl": "14.2.33", + "@next/swc-linux-x64-gnu": "14.2.33", + "@next/swc-linux-x64-musl": "14.2.33", + "@next/swc-win32-arm64-msvc": "14.2.33", + "@next/swc-win32-ia32-msvc": "14.2.33", + "@next/swc-win32-x64-msvc": "14.2.33", + "@swc/helpers": "0.5.5", + "busboy": "1.6.0", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", + "postcss": "8.4.31", + "styled-jsx": "5.1.1" + } + }, + "picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "postcss": { + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "requires": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + } + }, + "react": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "requires": { + "loose-envify": "^1.1.0" + } + }, + "react-dom": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", + "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "requires": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.0" + } + }, + "scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "requires": { + "loose-envify": "^1.1.0" + } + }, + "source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==" + }, + "streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" + }, + "styled-jsx": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz", + "integrity": "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==", + "requires": { + "client-only": "0.0.1" + } + }, + "tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true + }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/services/spectacle/package.json b/services/spectacle/package.json new file mode 100644 index 000000000..a38876194 --- /dev/null +++ b/services/spectacle/package.json @@ -0,0 +1,25 @@ +{ + "name": "openhouse-spectacle", + "version": "1.0.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint" + }, + "dependencies": { + "next": "^14.2.35", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "20.10.6", + "@types/react": "18.2.46", + "@types/react-dom": "18.2.18", + "typescript": "5.3.3" + }, + "volta": { + "node": "18.20.3" + } +} diff --git a/services/spectacle/public/iceberg.ico b/services/spectacle/public/iceberg.ico new file mode 100644 index 000000000..b803d1ce6 Binary files /dev/null and b/services/spectacle/public/iceberg.ico differ diff --git a/services/spectacle/public/iceberg_bg.png b/services/spectacle/public/iceberg_bg.png new file mode 100644 index 000000000..982e487e4 Binary files /dev/null and b/services/spectacle/public/iceberg_bg.png differ diff --git a/services/spectacle/public/logo.png b/services/spectacle/public/logo.png new file mode 100644 index 000000000..efbf9227d Binary files /dev/null and b/services/spectacle/public/logo.png differ diff --git a/services/spectacle/src/app/api/databases/acl-policies/route.ts b/services/spectacle/src/app/api/databases/acl-policies/route.ts new file mode 100644 index 000000000..cdae9ac65 --- /dev/null +++ b/services/spectacle/src/app/api/databases/acl-policies/route.ts @@ -0,0 +1,41 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + + if (bearerToken) { + headers['Authorization'] = `Bearer ${bearerToken}`; + } + + const response = await fetch(`${tablesServiceUrl}/v1/databases/${databaseId}/aclPolicies`, { + method: 'GET', + headers, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch database ACL policies', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/databases/route.ts b/services/spectacle/src/app/api/databases/route.ts new file mode 100644 index 000000000..4a373b24a --- /dev/null +++ b/services/spectacle/src/app/api/databases/route.ts @@ -0,0 +1,34 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function GET(request: NextRequest) { + try { + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const response = await fetch(`${tablesServiceUrl}/v1/databases`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch databases', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/jobs/[jobId]/route.ts b/services/spectacle/src/app/api/jobs/[jobId]/route.ts new file mode 100644 index 000000000..5218205aa --- /dev/null +++ b/services/spectacle/src/app/api/jobs/[jobId]/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ jobId: string }> } +) { + try { + const { jobId } = await params; + + const jobsServiceUrl = process.env.NEXT_PUBLIC_JOBS_SERVICE_URL || 'http://localhost:8002'; + const bearerToken = getBearerToken(); + + const response = await fetch(`${jobsServiceUrl}/jobs/${jobId}`, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('Job status API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch job status', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/jobs/route.ts b/services/spectacle/src/app/api/jobs/route.ts new file mode 100644 index 000000000..d2f4153cd --- /dev/null +++ b/services/spectacle/src/app/api/jobs/route.ts @@ -0,0 +1,85 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function GET(request: NextRequest) { + try { + const searchParams = request.nextUrl.searchParams; + const jobNamePrefix = searchParams.get('jobNamePrefix'); + const limit = searchParams.get('limit') || '10'; + const offset = searchParams.get('offset') || '0'; + + if (!jobNamePrefix) { + return NextResponse.json( + { error: 'jobNamePrefix query parameter is required' }, + { status: 400 } + ); + } + + const jobsServiceUrl = process.env.NEXT_PUBLIC_JOBS_SERVICE_URL || 'http://localhost:8002'; + const bearerToken = getBearerToken(); + + const url = new URL(`${jobsServiceUrl}/jobs`); + url.searchParams.set('jobNamePrefix', jobNamePrefix); + url.searchParams.set('limit', limit); + url.searchParams.set('offset', offset); + + const response = await fetch(url.toString(), { + method: 'GET', + headers: { + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('Jobs search API route error:', error); + return NextResponse.json( + { error: 'Failed to search jobs', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + + const jobsServiceUrl = process.env.NEXT_PUBLIC_JOBS_SERVICE_URL || 'http://localhost:8002'; + const bearerToken = getBearerToken(); + + const response = await fetch(`${jobsServiceUrl}/jobs`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${bearerToken}`, + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('Jobs API route error:', error); + return NextResponse.json( + { error: 'Failed to submit job', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/data/route.ts b/services/spectacle/src/app/api/tables/data/route.ts new file mode 100644 index 000000000..eff3fc3da --- /dev/null +++ b/services/spectacle/src/app/api/tables/data/route.ts @@ -0,0 +1,49 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId, tableId, limit = 10 } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const url = `${tablesServiceUrl}/internal/tables/${databaseId}/${tableId}/data?limit=${limit}`; + console.log(`[DataPreview] Fetching: ${url}`); + + const response = await fetch(url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + console.log(`[DataPreview] Response status: ${response.status}`); + + if (!response.ok) { + const errorText = await response.text(); + console.error(`[DataPreview] Error response: ${errorText}`); + if (response.status === 400) { + console.warn(`Table data unavailable for ${databaseId}.${tableId}: ${errorText}`); + return NextResponse.json( + { error: 'Table data unavailable', details: 'Could not read data from the table. The table may be empty or have access restrictions.' }, + { status: 400 } + ); + } + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('[DataPreview] API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch table data', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/details/route.ts b/services/spectacle/src/app/api/tables/details/route.ts new file mode 100644 index 000000000..6b91bfb0a --- /dev/null +++ b/services/spectacle/src/app/api/tables/details/route.ts @@ -0,0 +1,54 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId, tableId } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + + if (bearerToken) { + headers['Authorization'] = `Bearer ${bearerToken}`; + } + + const response = await fetch(`${tablesServiceUrl}/v1/databases/${databaseId}/tables/${tableId}`, { + method: 'GET', + headers, + }); + + if (!response.ok) { + // If backend returns 400 (bad request, often due to Iceberg metadata issues), + // return partial table info instead of failing completely + if (response.status === 400) { + console.warn(`Table details returned 400 for ${databaseId}.${tableId}, returning partial info`); + return NextResponse.json({ + databaseId, + tableId, + clusterId: 'unknown', + _partial: true, + _error: 'Iceberg metadata unavailable', + }); + } + + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch table details', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/metadata-diff/route.ts b/services/spectacle/src/app/api/tables/metadata-diff/route.ts new file mode 100644 index 000000000..e93adf043 --- /dev/null +++ b/services/spectacle/src/app/api/tables/metadata-diff/route.ts @@ -0,0 +1,64 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function GET(request: NextRequest) { + try { + const searchParams = request.nextUrl.searchParams; + const databaseId = searchParams.get('databaseId'); + const tableId = searchParams.get('tableId'); + const metadataFile = searchParams.get('metadataFile'); + + console.log('[metadata-diff] Request params:', { databaseId, tableId, metadataFile }); + + if (!databaseId || !tableId || !metadataFile) { + console.error('[metadata-diff] Missing parameters'); + return NextResponse.json( + { error: 'Missing required parameters: databaseId, tableId, metadataFile' }, + { status: 400 } + ); + } + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + const backendUrl = `${tablesServiceUrl}/internal/tables/${databaseId}/${tableId}/metadata/diff?metadataFile=${encodeURIComponent(metadataFile)}`; + + console.log('[metadata-diff] Calling backend:', backendUrl); + + const response = await fetch(backendUrl, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + console.log('[metadata-diff] Backend response status:', response.status); + + if (!response.ok) { + const errorText = await response.text(); + console.error('[metadata-diff] Backend error:', errorText); + return NextResponse.json( + { + error: `API Error: ${response.status} ${response.statusText}`, + details: errorText, + backendUrl + }, + { status: response.status } + ); + } + + const data = await response.json(); + console.log('[metadata-diff] Success, returning data'); + return NextResponse.json(data); + } catch (error) { + console.error('[metadata-diff] API route error:', error); + return NextResponse.json( + { + error: 'Failed to fetch metadata diff', + details: error instanceof Error ? error.message : 'Unknown error', + stack: error instanceof Error ? error.stack : undefined + }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/metadata/route.ts b/services/spectacle/src/app/api/tables/metadata/route.ts new file mode 100644 index 000000000..96bb29f9a --- /dev/null +++ b/services/spectacle/src/app/api/tables/metadata/route.ts @@ -0,0 +1,44 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId, tableId } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const response = await fetch(`${tablesServiceUrl}/internal/tables/${databaseId}/${tableId}/metadata`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${bearerToken}`, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + // Provide more descriptive error message for 400 (Iceberg metadata issues) + if (response.status === 400) { + console.warn(`Iceberg metadata unavailable for ${databaseId}.${tableId}: ${errorText}`); + return NextResponse.json( + { error: 'Iceberg metadata unavailable', details: 'The table exists but Iceberg metadata could not be loaded. This may be due to metadata file corruption or access issues.' }, + { status: 400 } + ); + } + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch table metadata', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/permissions/route.ts b/services/spectacle/src/app/api/tables/permissions/route.ts new file mode 100644 index 000000000..025424ddd --- /dev/null +++ b/services/spectacle/src/app/api/tables/permissions/route.ts @@ -0,0 +1,41 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId, tableId } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + + if (bearerToken) { + headers['Authorization'] = `Bearer ${bearerToken}`; + } + + const response = await fetch(`${tablesServiceUrl}/v1/databases/${databaseId}/tables/${tableId}/aclPolicies`, { + method: 'GET', + headers, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch ACL policies', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/api/tables/search/route.ts b/services/spectacle/src/app/api/tables/search/route.ts new file mode 100644 index 000000000..8311ad099 --- /dev/null +++ b/services/spectacle/src/app/api/tables/search/route.ts @@ -0,0 +1,41 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getBearerToken } from '@/lib/auth'; + +export async function POST(request: NextRequest) { + try { + const { databaseId } = await request.json(); + + const tablesServiceUrl = process.env.NEXT_PUBLIC_TABLES_SERVICE_URL || 'http://localhost:8000'; + const bearerToken = getBearerToken(); + + const headers: Record = { + 'Content-Type': 'application/json', + }; + + if (bearerToken) { + headers['Authorization'] = `Bearer ${bearerToken}`; + } + + const response = await fetch(`${tablesServiceUrl}/v1/databases/${databaseId}/tables/search`, { + method: 'POST', + headers, + }); + + if (!response.ok) { + const errorText = await response.text(); + return NextResponse.json( + { error: `API Error: ${response.status} ${response.statusText}`, details: errorText }, + { status: response.status } + ); + } + + const data = await response.json(); + return NextResponse.json(data); + } catch (error) { + console.error('API route error:', error); + return NextResponse.json( + { error: 'Failed to fetch tables', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ); + } +} diff --git a/services/spectacle/src/app/databases/[databaseId]/page.tsx b/services/spectacle/src/app/databases/[databaseId]/page.tsx new file mode 100644 index 000000000..26aad5469 --- /dev/null +++ b/services/spectacle/src/app/databases/[databaseId]/page.tsx @@ -0,0 +1,474 @@ +'use client'; + +import { useParams, useRouter } from 'next/navigation'; +import { useEffect, useState } from 'react'; +import Link from 'next/link'; + +interface Table { + tableId: string; + databaseId: string; + clusterId?: string; + tableUri: string; + tableUUID?: string; + tableLocation?: string; + tableVersion?: string; + tableType: string; + tableCreator?: string; + creationTime?: number; + lastModifiedTime?: number; +} + +interface AclPolicy { + principal: string; + role: string; + operation: string; +} + +export default function DatabasePage() { + const params = useParams(); + const router = useRouter(); + const databaseId = params.databaseId as string; + + const [tables, setTables] = useState([]); + const [aclPolicies, setAclPolicies] = useState([]); + const [loading, setLoading] = useState(true); + const [aclLoading, setAclLoading] = useState(true); + const [error, setError] = useState(''); + const [aclError, setAclError] = useState(''); + + useEffect(() => { + fetchTables(); + fetchAclPolicies(); + }, [databaseId]); + + const fetchTables = async () => { + setLoading(true); + setError(''); + + try { + const response = await fetch('/api/tables/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ databaseId }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to fetch tables'); + } + + const data = await response.json(); + const baseTables = data.results || []; + + // Fetch metadata for all tables in parallel to get UUID and timestamps + const metadataPromises = baseTables.map(async (table: any) => { + try { + const metaResponse = await fetch('/api/tables/metadata', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + databaseId: table.databaseId, + tableId: table.tableId + }), + }); + + if (metaResponse.ok) { + const metadata = await metaResponse.json(); + // Parse the currentMetadata JSON to extract UUID and timestamps + if (metadata.currentMetadata) { + try { + const parsed = JSON.parse(metadata.currentMetadata); + return { + ...table, + tableUUID: parsed['table-uuid'] || table.tableId, + lastModifiedTime: parsed['last-updated-ms'], + creationTime: parsed['last-updated-ms'], // Iceberg doesn't have creation time, use last-updated + }; + } catch (e) { + console.error('Error parsing metadata for', table.tableId, e); + } + } + } + } catch (e) { + console.error('Error fetching metadata for', table.tableId, e); + } + return table; + }); + + const enrichedTables = await Promise.all(metadataPromises); + setTables(enrichedTables); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load tables'); + console.error('Tables error:', err); + } finally { + setLoading(false); + } + }; + + const fetchAclPolicies = async () => { + setAclLoading(true); + setAclError(''); + + try { + const response = await fetch('/api/databases/acl-policies', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ databaseId }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to fetch ACL policies'); + } + + const data = await response.json(); + setAclPolicies(data.results || []); + } catch (err) { + setAclError(err instanceof Error ? err.message : 'Failed to load ACL policies'); + console.error('ACL error:', err); + } finally { + setAclLoading(false); + } + }; + + const formatDate = (timestamp: number | undefined) => { + if (timestamp === undefined || timestamp === null) { + return 'Not available'; + } + if (timestamp === 0) { + return 'Not set'; + } + // If timestamp is in seconds (less than year 2000 in milliseconds), convert to milliseconds + const timestampMs = timestamp < 10000000000 ? timestamp * 1000 : timestamp; + return new Date(timestampMs).toLocaleString(); + }; + + return ( +

+
+ {/* Breadcrumb Navigation */} +
+ + / + + {databaseId} + +
+ + {/* Header */} +
+ +

+ Database: {databaseId} +

+ +
+ {tables.length} {tables.length === 1 ? 'table' : 'tables'} +
+
+ + {/* Two Column Layout */} +
+ {/* Tables List - Left Column */} +
+

+ Tables +

+ + {loading && ( +
+ Loading tables... +
+ )} + + {error && ( +
+ {error} +
+ )} + + {!loading && !error && tables.length === 0 && ( +
+ No tables found in this database +
+ )} + + {!loading && tables.length > 0 && ( +
+ {tables.map((table) => ( + +
{ + e.currentTarget.style.borderColor = '#3b82f6'; + e.currentTarget.style.backgroundColor = '#eff6ff'; + }} + onMouseLeave={(e) => { + e.currentTarget.style.borderColor = '#e5e7eb'; + e.currentTarget.style.backgroundColor = '#f9fafb'; + }}> +
+
+ {table.tableId} +
+ + {table.tableType} + +
+ +
+ UUID: {table.tableUUID || 'Loading...'} +
+ +
+ Last Updated: {formatDate(table.lastModifiedTime)} +
+
+ + ))} +
+ )} +
+ + {/* Database ACL Policies - Right Column */} +
+

+ Database Permissions +

+ + {aclLoading && ( +
+ Loading permissions... +
+ )} + + {aclError && ( +
+ {aclError} +
+ )} + + {!aclLoading && !aclError && aclPolicies.length === 0 && ( +
+ No ACL policies configured for this database +
+ )} + + {!aclLoading && aclPolicies.length > 0 && ( +
+ + + + + + + + + {aclPolicies.map((policy, index) => ( + + + + + ))} + +
+ Principal + + Role +
+ {policy.principal} + + + {policy.role} + +
+
+ )} +
+
+
+
+ ); +} diff --git a/services/spectacle/src/app/layout.tsx b/services/spectacle/src/app/layout.tsx new file mode 100644 index 000000000..07d2d5a23 --- /dev/null +++ b/services/spectacle/src/app/layout.tsx @@ -0,0 +1,59 @@ +import type { Metadata } from 'next' +import { fonts, fontSizes, lineHeights, colors } from '@/lib/theme' + +export const metadata: Metadata = { + title: 'OpenHouse Spectacle', + description: 'OpenHouse Spectacle Application', + icons: { + icon: '/iceberg.ico', + shortcut: '/iceberg.ico', + apple: '/apple-touch-icon.png', + }, +} + +export default function RootLayout({ + children, +}: { + children: React.ReactNode +}) { + return ( + + + + + {children} + + ) +} diff --git a/services/spectacle/src/app/page.tsx b/services/spectacle/src/app/page.tsx new file mode 100644 index 000000000..67c951849 --- /dev/null +++ b/services/spectacle/src/app/page.tsx @@ -0,0 +1,164 @@ +'use client'; + +import { Suspense, useEffect, useRef } from 'react'; +import { useTableSearch } from '@/hooks/useTableSearch'; +import PageHeader from '@/components/PageHeader'; +import SearchBar from '@/components/SearchBar'; +import TableFilter from '@/components/TableFilter'; +import ErrorMessage from '@/components/ErrorMessage'; +import TablesTable from '@/components/TablesTable'; +import EmptyState from '@/components/EmptyState'; + +function HomeContent() { + const { + databaseId, + setDatabaseId, + tables, + loading, + loadingMore, + hasMore, + error, + searchFilter, + setSearchFilter, + searchTables, + loadMoreTables, + filteredTables, + } = useTableSearch(); + + const observerTarget = useRef(null); + + useEffect(() => { + const observer = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting && hasMore && !loadingMore) { + loadMoreTables(); + } + }, + { threshold: 0.1 } + ); + + const currentTarget = observerTarget.current; + if (currentTarget) { + observer.observe(currentTarget); + } + + return () => { + if (currentTarget) { + observer.unobserve(currentTarget); + } + }; + }, [hasMore, loadingMore, loadMoreTables]); + + const handleDatabaseIdChange = (value: string) => { + setDatabaseId(value); + }; + + return ( +
+ {/* Iceberg Background Overlay */} +
+ +
+ + + + + {tables.length > 0 && ( + + )} + + + + + + {!loading && tables.length === 0 && !error && } + + {/* Infinite scroll observer target */} + {tables.length > 0 && ( +
+ {loadingMore && ( +
+
+

Loading more tables...

+
+ )} + {!hasMore && tables.length > 20 && ( +
+ All tables loaded ({tables.length} total) +
+ )} +
+ )} +
+
+ ); +} + +export default function Home() { + return ( + +
+
+

Loading...

+
+ + }> + + + ); +} diff --git a/services/spectacle/src/app/tables/[databaseId]/[tableId]/page.tsx b/services/spectacle/src/app/tables/[databaseId]/[tableId]/page.tsx new file mode 100644 index 000000000..65b7191ad --- /dev/null +++ b/services/spectacle/src/app/tables/[databaseId]/[tableId]/page.tsx @@ -0,0 +1,1577 @@ +'use client'; + +import { useParams, useRouter, useSearchParams } from 'next/navigation'; +import { useEffect, useState, Suspense } from 'react'; +import { Table } from '@/types/table'; +import Maintenance from '@/components/Maintenance'; +import Permissions from '@/components/Permissions'; +import MetadataDiffModal from '@/components/MetadataDiffModal'; +import DataPreview from '@/components/DataPreview'; +import { fonts, fontSizes, fontWeights, colors } from '@/lib/theme'; + +interface IcebergMetadata { + tableId: string; + databaseId: string; + currentMetadata: string; + metadataLog: string | null; + metadataHistory: Array<{ + version: number; + file: string; + timestamp: number; + location: string; + }> | null; + metadataLocation: string; + snapshots: string | null; + partitions: string | null; + currentSnapshotId: number | string | null; +} + +interface SnapshotCardProps { + snapshotId: number | string; + operation: string; + timestamp: string; + summary: Record; + isCurrent: boolean; +} + +function SnapshotCard({ snapshotId, operation, timestamp, summary, isCurrent }: SnapshotCardProps) { + const [isExpanded, setIsExpanded] = useState(false); + + // Defensive checks + if (snapshotId === undefined || snapshotId === null) { + return null; + } + + const formatValue = (value: string | number) => { + if (value === 'N/A' || value === undefined || value === null) return 'N/A'; + return Number(value).toLocaleString(); + }; + + const formatBytes = (bytes: string | number) => { + if (bytes === 'N/A' || bytes === undefined || bytes === null) return 'N/A'; + const num = Number(bytes); + if (num === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(num) / Math.log(k)); + return Math.round((num / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]; + }; + + // Extract metadata fields from summary + const sparkAppId = summary['spark.app.id'] || summary['spark-app-id'] || 'N/A'; + const addedRecords = summary['added-records'] || 'N/A'; + const addedFiles = summary['added-data-files'] || 'N/A'; + const addedFilesSize = summary['added-files-size'] || 'N/A'; + const changedPartitionCount = summary['changed-partition-count'] || 'N/A'; + const totalRecords = summary['total-records'] || 'N/A'; + const totalFiles = summary['total-data-files'] || 'N/A'; + const totalFilesSize = summary['total-files-size'] || 'N/A'; + const totalDeleteFiles = summary['total-delete-files'] || 'N/A'; + const totalPositionDeletes = summary['total-position-deletes'] || 'N/A'; + const totalEqualityDeletes = summary['total-equality-deletes'] || 'N/A'; + + const MetricItem = ({ label, value }: { label: string; value: string }) => ( +
+ {label} + {value} +
+ ); + + return ( +
+ {/* Card Header - Always Visible */} +
setIsExpanded(!isExpanded)} + style={{ + padding: '0.7rem', + cursor: 'pointer', + display: 'grid', + gridTemplateColumns: '1fr 110px 180px 30px', + gap: '0.5rem', + alignItems: 'center', + backgroundColor: isCurrent ? '#dbeafe' : '#f9fafb', + borderRadius: isExpanded ? '8px 8px 0 0' : '8px', + minHeight: '39px' + }} + > + {/* Left: Snapshot ID with CURRENT badge */} +
+ + {snapshotId || 'Unknown ID'} + + {isCurrent && ( + + CURRENT + + )} +
+ + {/* Operation Badge - Fixed Width */} + + {operation || 'unknown'} + + + {/* Timestamp */} + + {timestamp || 'Unknown time'} + + + {/* Expand Arrow */} + + ▼ + +
+ + {/* Expandable Content */} + {isExpanded && ( +
+ {/* Data Changes Section */} +
+

+ Data Changes +

+ + + + +
+ + {/* Total Statistics Section */} +
+

+ Total Statistics +

+ + + +
+ + {/* Delete Operations Section */} +
+

+ Delete Operations +

+ + + +
+ + {/* Execution Details Section */} +
+

+ Execution Details +

+ +
+
+ )} +
+ ); +} + +interface SchemaViewerProps { + currentSchema: string; + metadata: IcebergMetadata; +} + +function SchemaViewer({ currentSchema, metadata }: SchemaViewerProps) { + const [currentSchemaIndex, setCurrentSchemaIndex] = useState(0); + const [schemas, setSchemas] = useState([]); + + useEffect(() => { + try { + const metadataJson = JSON.parse(metadata.currentMetadata); + const schemasArray = metadataJson.schemas || []; + + // Sort schemas by schema-id in descending order (newest first) + const sortedSchemas = [...schemasArray].sort((a, b) => (b['schema-id'] || 0) - (a['schema-id'] || 0)); + setSchemas(sortedSchemas); + } catch (e) { + console.error('Error parsing schemas:', e); + } + }, [metadata]); + + const getFieldChanges = (currentSchema: any, previousSchema: any | null) => { + if (!previousSchema) return { newFieldIds: new Set() }; + + const currentFields = currentSchema.fields || []; + const previousFields = previousSchema.fields || []; + const previousFieldIds = new Set(previousFields.map((f: any) => f.id)); + + const newFieldIds = new Set( + currentFields + .filter((f: any) => !previousFieldIds.has(f.id)) + .map((f: any) => f.id as number) + ); + + return { newFieldIds }; + }; + + const highlightSchemaJSON = (schema: any, newFieldIds: Set) => { + const schemaString = JSON.stringify(schema, null, 2); + const lines = schemaString.split('\n'); + + // Track which lines belong to new fields + const highlightedLines = new Set(); + let currentFieldId: number | null = null; + let braceDepth = 0; + let inFieldObject = false; + let fieldStartLine = -1; + + // First pass: identify which lines belong to new fields + lines.forEach((line, index) => { + // Check if this line contains a field ID + const idMatch = line.match(/"id":\s*(\d+)/); + if (idMatch) { + const fieldId = parseInt(idMatch[1]); + if (newFieldIds.has(fieldId)) { + currentFieldId = fieldId; + inFieldObject = true; + fieldStartLine = index; + // Find the line where this field object starts (the opening brace) + for (let i = index - 1; i >= 0; i--) { + if (lines[i].trim().endsWith('{')) { + fieldStartLine = i; + break; + } + } + braceDepth = 0; + } + } + + // Track brace depth to know when field object ends + if (inFieldObject) { + const openBraces = (line.match(/{/g) || []).length; + const closeBraces = (line.match(/}/g) || []).length; + braceDepth += openBraces - closeBraces; + + // Mark all lines from field start to current as highlighted + for (let i = fieldStartLine; i <= index; i++) { + highlightedLines.add(i); + } + + // When braces are balanced, we've exited the field object + if (braceDepth <= 0 && line.includes('}')) { + inFieldObject = false; + currentFieldId = null; + } + } + }); + + return lines.map((line, index) => { + const isHighlighted = highlightedLines.has(index); + + return ( +
+ {line} +
+ ); + }); + }; + + const handlePrevious = () => { + if (currentSchemaIndex < schemas.length - 1) { + setCurrentSchemaIndex(currentSchemaIndex + 1); + } + }; + + const handleNext = () => { + if (currentSchemaIndex > 0) { + setCurrentSchemaIndex(currentSchemaIndex - 1); + } + }; + + if (schemas.length === 0) { + return ( + 
+        {JSON.stringify(JSON.parse(currentSchema), null, 2)}
+
+ ); + } + + const currentDisplaySchema = schemas[currentSchemaIndex]; + const previousDisplaySchema = currentSchemaIndex < schemas.length - 1 ? schemas[currentSchemaIndex + 1] : null; + const { newFieldIds } = getFieldChanges(currentDisplaySchema, previousDisplaySchema); + + return ( +
+ {/* Navigation Controls */} +
+ + +
+
+ Schema Version {currentDisplaySchema['schema-id']} + + {' '}({currentSchemaIndex + 1} of {schemas.length}) + +
+ {newFieldIds.size > 0 && ( +
+ {newFieldIds.size} New Field{newFieldIds.size !== 1 ? 's' : ''} Added +
+ )} +
+ + +
+ + {/* Schema Display */} + 
+        {highlightSchemaJSON(currentDisplaySchema, newFieldIds)}
+
+
+ ); +} + +function TableDetailContent() { + const params = useParams(); + const router = useRouter(); + const searchParams = useSearchParams(); + const databaseId = params.databaseId as string; + const tableId = params.tableId as string; + const searchDatabaseId = searchParams.get('db') || databaseId; + + const [table, setTable] = useState(null); + const [icebergMetadata, setIcebergMetadata] = useState(null); + const [loading, setLoading] = useState(true); + const [metadataLoading, setMetadataLoading] = useState(false); + const [error, setError] = useState(''); + const [metadataError, setMetadataError] = useState(''); + const [diffModalOpen, setDiffModalOpen] = useState(false); + const [selectedMetadataFile, setSelectedMetadataFile] = useState(null); + + const handleViewDiff = (metadataFile: string) => { + setSelectedMetadataFile(metadataFile); + setDiffModalOpen(true); + }; + + useEffect(() => { + fetchTableDetails(); + }, [databaseId, tableId]); + + const fetchTableDetails = async () => { + setLoading(true); + setError(''); + + try { + const response = await fetch('/api/tables/details', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ databaseId, tableId }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to fetch table details'); + } + + const data = await response.json(); + setTable(data); + + // Fetch Iceberg metadata after table details load + fetchIcebergMetadata(); + } catch (err) { + setError(err instanceof Error ? err.message : 'An error occurred'); + } finally { + setLoading(false); + } + }; + + const fetchIcebergMetadata = async () => { + setMetadataLoading(true); + setMetadataError(''); + + try { + const response = await fetch('/api/tables/metadata', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ databaseId, tableId }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to fetch Iceberg metadata'); + } + + // Backend now sends all Long values as strings, so we can safely parse + const data = await response.json(); + console.log('Fetched metadata with currentSnapshotId:', data.currentSnapshotId, typeof data.currentSnapshotId); + setIcebergMetadata(data); + } catch (err) { + setMetadataError(err instanceof Error ? err.message : 'Failed to load Iceberg metadata'); + console.error('Iceberg metadata error:', err); + } finally { + setMetadataLoading(false); + } + }; + + const formatDate = (timestamp: number | string) => { + // Convert string to number if needed + const ts = typeof timestamp === 'string' ? parseInt(timestamp) : timestamp; + // If timestamp is in seconds (less than year 2000 in milliseconds), convert to milliseconds + const timestampMs = ts < 10000000000 ? ts * 1000 : ts; + return new Date(timestampMs).toLocaleString(); + }; + + const removeNullValues = (obj: any): any => { + if (typeof obj !== 'object' || obj === null) return obj; + return Object.fromEntries( + Object.entries(obj).filter(([_, value]) => value !== null) + ); + }; + + if (loading) { + return ( +
+
+

Loading table details...

+
+
+ ); + } + + if (error) { + return ( +
+
+ +
+ Error: {error} +
+
+
+ ); + } + + if (!table) { + return ( +
+
+ +

Table not found

+
+
+ ); + } + + // Check if this is partial data (Iceberg metadata unavailable) + const isPartialData = (table as any)._partial === true; + + return ( +
+
+ {/* Breadcrumb Navigation */} +
+ + / + + / + + {tableId} + +
+ + {/* Warning Banner for Partial Data */} + {isPartialData && ( +
+ ⚠️ +
+
+ Limited Table Information +
+
+ {(table as any)._error || 'Iceberg metadata could not be loaded for this table. Showing basic information only.'} +
+
+
+ )} + + {/* Header */} +
+
+ {/* Database Name */} +
+
+ + Database Name + +
+

+ {table.databaseId} +

+
+ + {/* Table Name */} +
+
+ + Table Name + +
+

+ {table.tableId} +

+
+
+
+ + {/* Two Column Layout: Basic Info (2/5) and Iceberg Metadata (3/5) */} +
+ {/* Basic Information - Left Column */} +
+

+ Basic Information +

+ +
+ + + + + + + {table.tableType || 'Unknown'} + + } /> + + + + + {/* Table URI */} + {table.tableUri && ( +
+

+ Table URI +

+
+ {table.tableUri} +
+
+ )} + + {/* Storage Location */} + {table.tableLocation && ( +
+

+ Storage Location +

+
+ {table.tableLocation} +
+
+ )} +
+
+ + {/* Iceberg Metadata - Right Column */} +
+

+ Iceberg Metadata +

+ + {metadataLoading && ( +
+ Loading Iceberg metadata... +
+ )} + + {metadataError && ( +
+ {metadataError} +
+ )} + + {icebergMetadata && !metadataLoading && ( +
+ {/* Current Snapshot ID */} + {icebergMetadata.currentSnapshotId && (() => { + // Find the current snapshot to get its summary data + let currentSnapshotSummary = null; + try { + if (icebergMetadata.snapshots) { + const snapshots = JSON.parse(icebergMetadata.snapshots); + const currentSnapshot = snapshots.find((s: any) => + s['snapshot-id'] === icebergMetadata.currentSnapshotId + ); + if (currentSnapshot && currentSnapshot.summary) { + currentSnapshotSummary = currentSnapshot.summary; + } + } + } catch (e) { + console.error('Error parsing current snapshot:', e); + } + + const formatValue = (value: string | number) => { + if (value === 'N/A' || value === undefined || value === null) return 'N/A'; + return Number(value).toLocaleString(); + }; + + const formatBytes = (bytes: string | number) => { + if (bytes === 'N/A' || bytes === undefined || bytes === null) return 'N/A'; + const num = Number(bytes); + if (num === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(num) / Math.log(k)); + return Math.round((num / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]; + }; + + const totalRecords = currentSnapshotSummary?.['total-records'] || 'N/A'; + const totalFiles = currentSnapshotSummary?.['total-data-files'] || 'N/A'; + const totalSize = currentSnapshotSummary?.['total-files-size'] || 'N/A'; + + return ( +
+

+ Current Snapshot +

+
+
+ {icebergMetadata.currentSnapshotId} +
+ {currentSnapshotSummary && ( +
+
+
Records
+
+ {formatValue(totalRecords)} +
+
+
+
Files
+
+ {formatValue(totalFiles)} +
+
+
+
Size
+
+ {formatBytes(totalSize)} +
+
+
+ )} +
+
+ ); + })()} + + {/* Snapshots */} + {(() => { + // Check if snapshots data exists + if (!icebergMetadata.snapshots) { + console.warn('No snapshots data available in metadata'); + return ( +
+

+ Snapshots +

+
+ No snapshot data available for this table. +
+
+ ); + } + + try { + // Backend now sends all Long values as strings, so we can safely parse + const snapshots = JSON.parse(icebergMetadata.snapshots); + console.log('Parsed snapshots:', snapshots); + console.log('Current snapshot ID from backend:', icebergMetadata.currentSnapshotId, typeof icebergMetadata.currentSnapshotId); + + // Sort by timestamp (convert string timestamps to numbers for comparison) + const sortedSnapshots = snapshots.sort((a: any, b: any) => { + const timeA = typeof a['timestamp-ms'] === 'string' ? parseInt(a['timestamp-ms']) : a['timestamp-ms']; + const timeB = typeof b['timestamp-ms'] === 'string' ? parseInt(b['timestamp-ms']) : b['timestamp-ms']; + return timeB - timeA; + }); + if (sortedSnapshots.length > 0) { + console.log('Most recent snapshot by timestamp:', sortedSnapshots[0]['snapshot-id'], typeof sortedSnapshots[0]['snapshot-id']); + } + + // Check if snapshots array is empty + if (!Array.isArray(snapshots) || snapshots.length === 0) { + console.warn('Snapshots array is empty'); + return ( +
+

+ Snapshots +

+
+ This table has no snapshots yet. +
+
+ ); + } + + return ( +
+

+ Snapshots ({snapshots.length}) +

+
+ {sortedSnapshots + .map((snapshot: any, index: number) => { + // Extract operation from various possible locations + const operation = snapshot.operation + || snapshot.summary?.operation + || (snapshot.summary && Object.keys(snapshot.summary).length > 0 ? 'append' : 'unknown'); + + const isCurrent = snapshot['snapshot-id'] === icebergMetadata.currentSnapshotId; + + console.log(`Snapshot ${snapshot['snapshot-id']}: isCurrent=${isCurrent} (current=${icebergMetadata.currentSnapshotId})`); + + return ( + + ); + })} +
+
+ ); + } catch (e) { + console.error('Error parsing snapshots:', e); + return ( +
+

+ Snapshots +

+
+
Failed to parse snapshots data
+
+ Error: {e instanceof Error ? e.message : 'Unknown error'} +
+
+ Check browser console for more details. +
+
+
+ ); + } + })()} + + {/* Metadata History */} + {(() => { + try { + const metadataLog = icebergMetadata.metadataLog ? JSON.parse(icebergMetadata.metadataLog) : []; + + if (metadataLog.length > 0) { + return ( +
+

+ Metadata History ({metadataLog.length}) +

+
+
+ + + + + + + + + {[...metadataLog].reverse().map((entry: any, index: number) => ( + + + + + + ))} + +
+ Timestamp + + Metadata File + + Actions +
+ {formatDate(entry['timestamp-ms'])} + + {entry['metadata-file']} + + +
+ + + ); + } + } catch (e) { + console.error('Error parsing metadata log:', e); + } + return null; + })()} + + {/* Partitions */} + {icebergMetadata.partitions && ( +
+

+ Partition Specs +

+ 
+                      {JSON.stringify(JSON.parse(icebergMetadata.partitions), null, 2)}
+
+
+ )} + + {/* Full Metadata JSON (Collapsible) */} +
+ + View Full Metadata JSON + + 
+                    {JSON.stringify(JSON.parse(icebergMetadata.currentMetadata), (key, value) =>
+                      typeof value === 'bigint' ? value.toString() : value
+                    , 2)}
+
+
+ + )} + + + + {/* Schema - Full Width */} + {table.schema && icebergMetadata && ( +
+

+ Schema +

+ +
+ )} + + {/* Data Preview - Full Width */} + + + {/* Table Properties with Policies Section */} + {(table.tableProperties && Object.keys(table.tableProperties).length > 0) || table.policies ? ( +
+

+ Properties & Policies +

+ + {/* Policies Section */} + {table.policies && ( +
+

+ Policies +

+
+ + {table.policies.sharingEnabled ? 'Yes' : 'No'} + + } + /> + {table.policies.retention && ( + + )} + {table.policies.replication && ( + + )} + {table.policies.history && ( + + )} +
+
+ )} + + {/* Table Properties */} + {table.tableProperties && Object.keys(table.tableProperties).length > 0 && ( +
+

+ Table Properties ({Object.keys(table.tableProperties).length}) +

+
+ + + + + + + + + {Object.entries(table.tableProperties).map(([key, value]) => ( + + + + + ))} + +
+ Property + + Value +
+ {key} + + {value} +
+
+
+ )} +
+ ) : null} + + {/* Maintenance Operations */} + + + {/* Permissions (ACL Policies) */} + + + + {/* Metadata Diff Modal */} + {selectedMetadataFile && ( + setDiffModalOpen(false)} + databaseId={databaseId} + tableId={tableId} + metadataFile={selectedMetadataFile} + /> + )} + + ); +} + +function DetailRow({ label, value }: { label: string; value: React.ReactNode }) { + return ( +
+
+ {label} +
+
+ {value} +
+
+ ); +} + +function CompactDetailRow({ label, value }: { label: string; value: React.ReactNode }) { + return ( +
+
+ {label} +
+
+ {value} +
+
+ ); +} + +export default function TableDetailPage() { + return ( + +
+
+

Loading table details...

+
+ + }> + + + ); +} diff --git a/services/spectacle/src/components/DataPreview.tsx b/services/spectacle/src/components/DataPreview.tsx new file mode 100644 index 000000000..ad6d0c64d --- /dev/null +++ b/services/spectacle/src/components/DataPreview.tsx @@ -0,0 +1,558 @@ +'use client'; + +import { useState, useEffect, useRef } from 'react'; +import { fonts, fontSizes, fontWeights, lineHeights, colors } from '@/lib/theme'; + +interface TableDataResponse { + tableId: string; + databaseId: string; + schema: string; + rows: Array>; + totalRowsFetched: number; + hasMore: boolean; +} + +interface DataPreviewProps { + databaseId: string; + tableId: string; +} + +type ViewMode = 'table' | 'json'; +type ExportFormat = 'csv' | 'tsv' | 'json'; + +export default function DataPreview({ databaseId, tableId }: DataPreviewProps) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [viewMode, setViewMode] = useState('table'); + const [exportDropdownOpen, setExportDropdownOpen] = useState(false); + const exportDropdownRef = useRef(null); + + // Close dropdown when clicking outside + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (exportDropdownRef.current && !exportDropdownRef.current.contains(event.target as Node)) { + setExportDropdownOpen(false); + } + }; + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, []); + + useEffect(() => { + fetchTableData(); + }, [databaseId, tableId]); + + const fetchTableData = async () => { + setLoading(true); + setError(null); + + try { + const response = await fetch('/api/tables/data', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ databaseId, tableId, limit: 10 }), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to fetch table data'); + } + + const result = await response.json(); + setData(result); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load data preview'); + console.error('Data preview error:', err); + } finally { + setLoading(false); + } + }; + + // Extract column names from schema or first row + const getColumns = (): string[] => { + if (!data) return []; + + // Try to get columns from schema + if (data.schema) { + try { + const schema = JSON.parse(data.schema); + if (schema.fields && Array.isArray(schema.fields)) { + return schema.fields.map((field: any) => field.name); + } + } catch (e) { + console.error('Error parsing schema:', e); + } + } + + // Fallback: get columns from first row + if (data.rows && data.rows.length > 0) { + return Object.keys(data.rows[0]); + } + + return []; + }; + + const formatCellValue = (value: any): string => { + if (value === null || value === undefined) { + return 'NULL'; + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + return String(value); + }; + + const escapeForCSV = (value: any): string => { + if (value === null || value === undefined) { + return ''; + } + const str = typeof value === 'object' ? JSON.stringify(value) : String(value); + // Escape quotes and wrap in quotes if contains comma, quote, or newline + if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\t')) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; + }; + + const downloadFile = (content: string, filename: string, mimeType: string) => { + const blob = new Blob([content], { type: mimeType }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + }; + + const handleExport = (format: ExportFormat) => { + if (!data || !data.rows || data.rows.length === 0) return; + + const columns = getColumns(); + const filename = `${databaseId}_${tableId}`; + + switch (format) { + case 'json': { + const jsonContent = JSON.stringify(data.rows, null, 2); + downloadFile(jsonContent, `${filename}.json`, 'application/json'); + break; + } + case 'csv': { + const header = columns.map(col => escapeForCSV(col)).join(','); + const rows = data.rows.map(row => + columns.map(col => escapeForCSV(row[col])).join(',') + ); + const csvContent = [header, ...rows].join('\n'); + downloadFile(csvContent, `${filename}.csv`, 'text/csv'); + break; + } + case 'tsv': { + const header = columns.join('\t'); + const rows = data.rows.map(row => + columns.map(col => { + const value = row[col]; + if (value === null || value === undefined) return ''; + const str = typeof value === 'object' ? JSON.stringify(value) : String(value); + return str.replace(/\t/g, ' ').replace(/\n/g, ' '); + }).join('\t') + ); + const tsvContent = [header, ...rows].join('\n'); + downloadFile(tsvContent, `${filename}.tsv`, 'text/tab-separated-values'); + break; + } + } + + setExportDropdownOpen(false); + }; + + if (loading) { + return ( +
+

+ Data Preview +

+
+ Loading data preview... +
+
+ ); + } + + if (error) { + return ( +
+

+ Data Preview +

+
+ ⚠️ +
+
Unable to load data preview
+
+ {error} +
+
+
+
+ ); + } + + if (!data || !data.rows || data.rows.length === 0) { + return ( +
+

+ Data Preview +

+
+
📭
+
This table has no data yet.
+
+
+ ); + } + + const columns = getColumns(); + + return ( +
+
+

+ Data Preview +

+
+ + Showing {data.totalRowsFetched} row{data.totalRowsFetched !== 1 ? 's' : ''} + {data.hasMore && ' (more available)'} + + + {/* View Mode Toggle */} +
+ + +
+ + {/* Export Dropdown */} +
+ + {exportDropdownOpen && ( +
+ {(['csv', 'tsv', 'json'] as ExportFormat[]).map((format) => ( + + ))} +
+ )} +
+ + +
+
+ + {viewMode === 'table' ? ( +
+ 5 ? `${columns.length * 150}px` : 'auto' + }}> + + + {columns.map((column, index) => ( + + ))} + + + + {data.rows.map((row, rowIndex) => ( + + {columns.map((column, colIndex) => { + const value = row[column]; + const isNull = value === null || value === undefined; + return ( + + ); + })} + + ))} + +
+ {column} +
+ {formatCellValue(value)} +
+
+ ) : ( +
+ 
+            {JSON.stringify(data.rows, null, 2)}
+
+
+ )} + + {data.hasMore && ( +
+ Table contains more rows. This preview shows the first {data.totalRowsFetched} rows. +
+ )} +
+ ); +} diff --git a/services/spectacle/src/components/EmptyState.tsx b/services/spectacle/src/components/EmptyState.tsx new file mode 100644 index 000000000..38d0116a5 --- /dev/null +++ b/services/spectacle/src/components/EmptyState.tsx @@ -0,0 +1,17 @@ +import { fontSizes, colors } from '@/lib/theme'; + +export default function EmptyState() { + return ( +
+

No tables found

+

Enter a database ID and click Search to get started

+
+ ); +} diff --git a/services/spectacle/src/components/ErrorMessage.tsx b/services/spectacle/src/components/ErrorMessage.tsx new file mode 100644 index 000000000..c438c648a --- /dev/null +++ b/services/spectacle/src/components/ErrorMessage.tsx @@ -0,0 +1,19 @@ +interface ErrorMessageProps { + message: string; +} + +export default function ErrorMessage({ message }: ErrorMessageProps) { + if (!message) return null; + + return ( +
+ Error: {message} +
+ ); +} diff --git a/services/spectacle/src/components/Maintenance.tsx b/services/spectacle/src/components/Maintenance.tsx new file mode 100644 index 000000000..afdf7794c --- /dev/null +++ b/services/spectacle/src/components/Maintenance.tsx @@ -0,0 +1,791 @@ +'use client'; + +import { useState, useEffect, useRef } from 'react'; +import { Table } from '@/types/table'; + +interface MaintenanceJob { + type: string; + args: string[]; +} + +interface MaintenanceProps { + databaseId: string; + tableId: string; + table: Table | null; +} + +const MAINTENANCE_JOBS: MaintenanceJob[] = [ + { type: 'NO_OP', args: [] }, + { type: 'SQL_TEST', args: [] }, + { type: 'RETENTION', args: ['--backupDir', '.backup'] }, + { type: 'DATA_COMPACTION', args: [] }, + { type: 'SNAPSHOTS_EXPIRATION', args: [] }, + { type: 'ORPHAN_FILES_DELETION', args: ['--backupDir', '.backup'] }, + { type: 'ORPHAN_DIRECTORY_DELETION', args: ['--trashDir', '.trash'] }, + { type: 'TABLE_STATS_COLLECTION', args: [] }, + { type: 'DATA_LAYOUT_STRATEGY_GENERATION', args: [] }, + { type: 'DATA_LAYOUT_STRATEGY_EXECUTION', args: [] }, +]; + +interface PaginationControlsProps { + currentPage: number; + totalCount: number; + pageSize: number; + onPageChange: (page: number) => void; +} + +function PaginationControls({ currentPage, totalCount, pageSize, onPageChange }: PaginationControlsProps) { + const totalPages = Math.ceil(totalCount / pageSize); + + // Don't show pagination if there's only one page or less + if (totalPages <= 1) { + return null; + } + + const canGoPrevious = currentPage > 1; + const canGoNext = currentPage < totalPages; + + // Generate page numbers to display (show current, +/- 2 pages, first and last) + const getPageNumbers = () => { + const pages: (number | string)[] = []; + const maxPagesToShow = 7; + + if (totalPages <= maxPagesToShow) { + // Show all pages + for (let i = 1; i <= totalPages; i++) { + pages.push(i); + } + } else { + // Always show first page + pages.push(1); + + // Show pages around current page + const startPage = Math.max(2, currentPage - 1); + const endPage = Math.min(totalPages - 1, currentPage + 1); + + if (startPage > 2) { + pages.push('...'); + } + + for (let i = startPage; i <= endPage; i++) { + pages.push(i); + } + + if (endPage < totalPages - 1) { + pages.push('...'); + } + + // Always show last page + pages.push(totalPages); + } + + return pages; + }; + + const buttonBaseStyle: React.CSSProperties = { + padding: '0.5rem 0.75rem', + fontSize: '0.875rem', + fontWeight: '500', + border: '1px solid #e5e7eb', + borderRadius: '6px', + cursor: 'pointer', + backgroundColor: 'white', + color: '#374151', + transition: 'all 0.2s', + minWidth: '2.5rem', + }; + + const activeButtonStyle: React.CSSProperties = { + ...buttonBaseStyle, + backgroundColor: '#3b82f6', + color: 'white', + borderColor: '#3b82f6', + }; + + const disabledButtonStyle: React.CSSProperties = { + ...buttonBaseStyle, + cursor: 'not-allowed', + opacity: 0.5, + backgroundColor: '#f3f4f6', + }; + + return ( +
+ {/* Page info */} +
+ Showing {(currentPage - 1) * pageSize + 1} to {Math.min(currentPage * pageSize, totalCount)} of {totalCount} jobs +
+ + {/* Page controls */} +
+ {/* Previous button */} + + + {/* Page numbers */} + {getPageNumbers().map((page, idx) => { + if (page === '...') { + return ( + + ... + + ); + } + + const pageNum = page as number; + const isActive = pageNum === currentPage; + + return ( + + ); + })} + + {/* Next button */} + +
+
+ ); +} + +export default function Maintenance({ databaseId, tableId, table }: MaintenanceProps) { + const fqtn = `${databaseId}.${tableId}`; + + // Jobs that don't need tableName argument + const NO_TABLE_JOBS = ['NO_OP', 'SQL_TEST']; + // Jobs that need tableDirectoryPath instead of tableName + const DIRECTORY_JOBS = ['ORPHAN_DIRECTORY_DELETION']; + + const [jobRequests, setJobRequests] = useState<{ [key: string]: string }>( + MAINTENANCE_JOBS.reduce((acc, job) => { + let args: string[]; + + if (NO_TABLE_JOBS.includes(job.type)) { + // Jobs that don't need table argument at all + args = [...job.args]; + } else if (DIRECTORY_JOBS.includes(job.type)) { + // Jobs that need tableDirectoryPath instead + // For now, we'll leave this empty as it's not table-specific + args = [...job.args]; + } else if (job.type === 'RETENTION' && table?.policies?.retention) { + // Populate RETENTION args from policies + const retention = table.policies.retention; + args = ['--tableName', fqtn]; + + if (retention.columnPattern?.columnName) { + args.push('--columnName', retention.columnPattern.columnName); + } + if (retention.columnPattern?.pattern) { + args.push('--columnPattern', retention.columnPattern.pattern); + } + if (retention.granularity) { + args.push('--granularity', retention.granularity.toLowerCase()); + } + if (retention.count) { + args.push('--count', String(retention.count)); + } + args.push(...job.args); // Add --backupDir + } else if (job.type === 'SNAPSHOTS_EXPIRATION' && table?.policies?.history) { + // Populate SNAPSHOTS_EXPIRATION args from policies + const history = table.policies.history; + args = ['--tableName', fqtn]; + + if (history.maxAge) { + args.push('--maxAge', String(history.maxAge)); + } + if (history.granularity) { + args.push('--granularity', history.granularity.toLowerCase()); + } + if (history.versions) { + args.push('--versions', String(history.versions)); + } + } else { + // Most jobs need --tableName with fqtn + args = ['--tableName', fqtn, ...job.args]; + } + + const defaultRequest = { + jobName: `${job.type.toLowerCase()}_${tableId}`, + clusterId: 'LocalHadoopCluster', + jobConf: { + jobType: job.type, + args: args, + }, + }; + acc[job.type] = JSON.stringify(defaultRequest, null, 2); + return acc; + }, {} as { [key: string]: string }) + ); + const [activeTab, setActiveTab] = useState(MAINTENANCE_JOBS[0].type); + const [loading, setLoading] = useState<{ [key: string]: boolean }>({}); + const [results, setResults] = useState<{ [key: string]: { success: boolean; message: string } }>({}); + const [jobIds, setJobIds] = useState<{ [key: string]: string }>({}); + const [jobStates, setJobStates] = useState<{ [key: string]: string }>({}); + const [recentJobs, setRecentJobs] = useState<{ [key: string]: any[] }>({}); + const pollIntervalRefs = useRef<{ [key: string]: NodeJS.Timeout }>({}); + + // Pagination state + const [currentPages, setCurrentPages] = useState<{ [key: string]: number }>({}); + const [totalCounts, setTotalCounts] = useState<{ [key: string]: number }>({}); + const [pageSize] = useState(10); + + // Use ref to always get the latest page number (avoids stale closure in polling intervals) + const currentPagesRef = useRef<{ [key: string]: number }>({}); + + // Keep ref in sync with state + useEffect(() => { + currentPagesRef.current = currentPages; + }, [currentPages]); + + const getCurrentPage = (jobType: string) => currentPagesRef.current[jobType] || 1; + const getTotalCount = (jobType: string) => totalCounts[jobType] || 0; + + const handleRequestChange = (jobType: string, value: string) => { + setJobRequests((prev) => ({ ...prev, [jobType]: value })); + }; + + const fetchRecentJobs = async (jobType: string, page?: number) => { + try { + const currentPage = page || getCurrentPage(jobType); + const offset = (currentPage - 1) * pageSize; + const jobNamePrefix = `${jobType.toLowerCase()}_${tableId}`; + const response = await fetch( + `/api/jobs?jobNamePrefix=${encodeURIComponent(jobNamePrefix)}&limit=${pageSize}&offset=${offset}` + ); + + if (!response.ok) { + return; + } + + const data = await response.json(); + setRecentJobs((prev) => ({ ...prev, [jobType]: data.results || [] })); + if (data.totalCount !== undefined) { + setTotalCounts((prev) => ({ ...prev, [jobType]: data.totalCount })); + } + } catch (err) { + // Silently handle errors + } + }; + + const handlePageChange = (jobType: string, page: number) => { + setCurrentPages((prev) => ({ ...prev, [jobType]: page })); + fetchRecentJobs(jobType, page); + }; + + const TERMINAL_STATES = ['CANCELLED', 'FAILED', 'SUCCEEDED']; + + const pollJobStatus = async (jobType: string, jobId: string) => { + try { + const response = await fetch(`/api/jobs/${jobId}`, { + cache: 'no-store', + headers: { + 'Cache-Control': 'no-cache', + }, + }); + + if (!response.ok) { + throw new Error('Failed to fetch job status'); + } + + const data = await response.json(); + const state = data.state; + + setJobStates((prev) => ({ ...prev, [jobType]: state })); + + // Refresh recent jobs list on each poll to show state changes, maintain current page + fetchRecentJobs(jobType, getCurrentPage(jobType)); + + // Stop polling if terminal state is reached + if (TERMINAL_STATES.includes(state)) { + if (pollIntervalRefs.current[jobType]) { + clearInterval(pollIntervalRefs.current[jobType]); + delete pollIntervalRefs.current[jobType]; + } + setLoading((prev) => ({ ...prev, [jobType]: false })); + + // Update result message with final status + const success = state === 'SUCCEEDED'; + setResults((prev) => ({ + ...prev, + [jobType]: { + success, + message: `Job ${state.toLowerCase()}! Job ID: ${jobId}`, + }, + })); + } + } catch (err) { + // Silently handle polling errors + } + }; + + // Cleanup polling intervals on unmount + useEffect(() => { + return () => { + Object.values(pollIntervalRefs.current).forEach((interval) => { + clearInterval(interval); + }); + }; + }, []); + + // Fetch recent jobs when active tab changes + useEffect(() => { + // Initialize page 1 for new tabs if not set + if (!currentPages[activeTab]) { + setCurrentPages((prev) => ({ ...prev, [activeTab]: 1 })); + } + fetchRecentJobs(activeTab); + }, [activeTab, tableId]); + + const handleTriggerJob = async (jobType: string) => { + // Clear any existing polling interval for this job type + if (pollIntervalRefs.current[jobType]) { + clearInterval(pollIntervalRefs.current[jobType]); + delete pollIntervalRefs.current[jobType]; + } + + setLoading((prev) => ({ ...prev, [jobType]: true })); + setResults((prev) => ({ ...prev, [jobType]: { success: false, message: '' } })); + setJobStates((prev) => ({ ...prev, [jobType]: 'SUBMITTING' })); + + try { + const requestBody = JSON.parse(jobRequests[jobType]); + + const response = await fetch('/api/jobs', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error || 'Failed to submit job'); + } + + const data = await response.json(); + const jobId = data.jobId; + + setJobIds((prev) => ({ ...prev, [jobType]: jobId })); + setJobStates((prev) => ({ ...prev, [jobType]: data.state || 'QUEUED' })); + + // Refresh recent jobs list immediately, maintaining current page + fetchRecentJobs(jobType, getCurrentPage(jobType)); + + // Start polling for job status every 2 seconds + pollIntervalRefs.current[jobType] = setInterval(() => { + pollJobStatus(jobType, jobId); + }, 2000); + + // Poll immediately once + pollJobStatus(jobType, jobId); + + setResults((prev) => ({ + ...prev, + [jobType]: { + success: true, + message: `Job submitted! Job ID: ${jobId}`, + }, + })); + } catch (err) { + setResults((prev) => ({ + ...prev, + [jobType]: { + success: false, + message: err instanceof Error ? err.message : 'An error occurred', + }, + })); + setLoading((prev) => ({ ...prev, [jobType]: false })); + setJobStates((prev) => { + const newState = { ...prev }; + delete newState[jobType]; + return newState; + }); + } + }; + + return ( +
+

+ Maintenance Operations +

+ + {/* Tab Navigation */} +
+ {MAINTENANCE_JOBS.map((job) => ( + + ))} +
+ + {/* Active Tab Content */} + {MAINTENANCE_JOBS.filter((job) => job.type === activeTab).map((job) => ( +
+

+ Edit the request body as needed before submitting the job. +

+ + {/* Request Body Editor */} +