From 24ba1131bc4c3142b05d6cdc08bd3dddcab3331e Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 23 Jun 2025 17:10:03 +0200 Subject: [PATCH 1/4] Removed types = ["TABLE"] argument from readAllSqlTables In other databases like DuckDB, this type is actually "BASE TABLE" instead of "TABLE", making the function return 0 results. Setting it to `null` makes the integration do its default behavior, which usually is no filtering at all. --- .../main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index 7bd110ba73..8630df9bd0 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -643,7 +643,7 @@ public fun DataFrame.Companion.readAllSqlTables( val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) // exclude a system and other tables without data, but it looks like it is supported badly for many databases - val tables = metaData.getTables(catalogue, null, null, arrayOf("TABLE")) + val tables = metaData.getTables(catalogue, null, null, null) val dataFrames = mutableMapOf() From 99ebe3d6917a2e9702f67df3e7ce1ff2cb1efc0a Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 23 Jun 2025 18:26:48 +0200 Subject: [PATCH 2/4] added tableTypes parameter to readAllSqlTables instead, with a helpful error when no tables are found, hinting at the tableTypes argument and what the database supports --- dataframe-jdbc/api/dataframe-jdbc.api | 8 ++--- .../kotlinx/dataframe/io/readJdbc.kt | 33 +++++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/dataframe-jdbc/api/dataframe-jdbc.api b/dataframe-jdbc/api/dataframe-jdbc.api index 552c069316..68493581a3 100644 --- a/dataframe-jdbc/api/dataframe-jdbc.api +++ b/dataframe-jdbc/api/dataframe-jdbc.api @@ -50,10 +50,10 @@ public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt { public static final fun getSchemaForSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map; + public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map; public static final fun readDataFrame (Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame; public static final fun readDataFrame (Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; public static final fun readDataFrame (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZ)Lorg/jetbrains/kotlinx/dataframe/DataFrame; diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index 8630df9bd0..f7ada0fcd7 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -604,6 +604,8 @@ public fun ResultSet.readDataFrame( * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [dbConfig]. + * @param [tableTypes] an optional list of table types, which must be from the list of table types + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE"]`. * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. */ public fun DataFrame.Companion.readAllSqlTables( @@ -612,9 +614,10 @@ public fun DataFrame.Companion.readAllSqlTables( limit: Int = DEFAULT_LIMIT, inferNullability: Boolean = true, dbType: DbType? = null, + tableTypes: List? = listOf("TABLE"), ): Map { DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password).use { connection -> - return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType) + return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType, tableTypes) } } @@ -628,6 +631,8 @@ public fun DataFrame.Companion.readAllSqlTables( * @param [inferNullability] indicates how the column nullability should be inferred. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. + * @param [tableTypes] an optional list of table types, which must be from the list of table types + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE"]`. * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. * * @see DriverManager.getConnection @@ -638,12 +643,13 @@ public fun DataFrame.Companion.readAllSqlTables( limit: Int = DEFAULT_LIMIT, inferNullability: Boolean = true, dbType: DbType? = null, + tableTypes: List? = listOf("TABLE"), ): Map { val metaData = connection.metaData val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - // exclude a system and other tables without data, but it looks like it is supported badly for many databases - val tables = metaData.getTables(catalogue, null, null, null) + // exclude system- and other tables without data, but it looks like it is supported badly for many databases + val tables = metaData.getTables(catalogue, null, null, tableTypes?.toTypedArray()) val dataFrames = mutableMapOf() @@ -668,6 +674,27 @@ public fun DataFrame.Companion.readAllSqlTables( } } + // We may have no tables or filtered for the wrong table type, let's give a helpful error message + if (dataFrames.isEmpty()) { + val supportedTableTypes = metaData.tableTypes.let { + buildList { while (it.next()) add(it.getString("TABLE_TYPE")) } + } + if (tableTypes?.any { it !in supportedTableTypes } == true) { + val unsupportedTypes = tableTypes.filter { it !in supportedTableTypes } + throw IllegalArgumentException( + buildString { + appendLine("Found no tables with type(s) $tableTypes.") + if (unsupportedTypes.isNotEmpty()) { + appendLine( + "Table type(s) $unsupportedTypes are unsupported for ${determinedDbType::class.simpleName}.", + ) + } + appendLine("If this is unexpected, try adjusting `tableTypes=` to any of $supportedTableTypes.") + }, + ) + } + } + return dataFrames } From 97db01f5f9125552573ba61b4674f84d941b176c Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 23 Jun 2025 18:42:26 +0200 Subject: [PATCH 3/4] added BASE TABLE as default tableType too --- .../kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index f7ada0fcd7..2eba355475 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -605,7 +605,7 @@ public fun ResultSet.readDataFrame( * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [dbConfig]. * @param [tableTypes] an optional list of table types, which must be from the list of table types - * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE"]`. + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`. * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. */ public fun DataFrame.Companion.readAllSqlTables( @@ -614,7 +614,7 @@ public fun DataFrame.Companion.readAllSqlTables( limit: Int = DEFAULT_LIMIT, inferNullability: Boolean = true, dbType: DbType? = null, - tableTypes: List? = listOf("TABLE"), + tableTypes: List? = listOf("TABLE", "BASE TABLE"), ): Map { DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password).use { connection -> return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType, tableTypes) @@ -632,7 +632,7 @@ public fun DataFrame.Companion.readAllSqlTables( * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. * @param [tableTypes] an optional list of table types, which must be from the list of table types - * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE"]`. + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`. * @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database. * * @see DriverManager.getConnection @@ -643,7 +643,7 @@ public fun DataFrame.Companion.readAllSqlTables( limit: Int = DEFAULT_LIMIT, inferNullability: Boolean = true, dbType: DbType? = null, - tableTypes: List? = listOf("TABLE"), + tableTypes: List? = listOf("TABLE", "BASE TABLE"), ): Map { val metaData = connection.metaData val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) From dc029b5164afbd0b32e8ea2bd55456145ae3487c Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 23 Jun 2025 21:12:07 +0200 Subject: [PATCH 4/4] adjusted getSchemaForAllSqlTables too --- dataframe-jdbc/api/dataframe-jdbc.api | 8 ++--- .../kotlinx/dataframe/io/readJdbc.kt | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/dataframe-jdbc/api/dataframe-jdbc.api b/dataframe-jdbc/api/dataframe-jdbc.api index 68493581a3..b8c66a1f5d 100644 --- a/dataframe-jdbc/api/dataframe-jdbc.api +++ b/dataframe-jdbc/api/dataframe-jdbc.api @@ -37,10 +37,10 @@ public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt { public static final fun getDataFrameSchema (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static synthetic fun getDataFrameSchema$default (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static synthetic fun getDataFrameSchema$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; - public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map; - public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; - public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map; + public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map; + public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map; + public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map; + public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map; public static final fun getSchemaForResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema; diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index 2eba355475..bc8bb2c772 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -862,14 +862,17 @@ public fun ResultSet.getDataFrameSchema(dbType: DbType): DataFrameSchema = DataF * @param [dbConfig] the database configuration to connect to the database, including URL, user, and password. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [dbConfig]. + * @param [tableTypes] an optional list of table types, which must be from the list of table types + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`. * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. */ public fun DataFrame.Companion.getSchemaForAllSqlTables( dbConfig: DbConnectionConfig, dbType: DbType? = null, + tableTypes: List? = listOf("TABLE", "BASE TABLE"), ): Map { DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password).use { connection -> - return getSchemaForAllSqlTables(connection, dbType) + return getSchemaForAllSqlTables(connection, dbType, tableTypes) } } @@ -879,18 +882,20 @@ public fun DataFrame.Companion.getSchemaForAllSqlTables( * @param [connection] the database connection. * @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`, * in that case the [dbType] will be recognized from the [connection]. + * @param [tableTypes] an optional list of table types, which must be from the list of table types + * returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`. * @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table. */ public fun DataFrame.Companion.getSchemaForAllSqlTables( connection: Connection, dbType: DbType? = null, + tableTypes: List? = listOf("TABLE", "BASE TABLE"), ): Map { val metaData = connection.metaData val determinedDbType = dbType ?: extractDBTypeFromConnection(connection) - val tableTypes = arrayOf("TABLE") - // exclude a system and other tables without data - val tables = metaData.getTables(null, null, null, tableTypes) + // exclude system- and other tables without data + val tables = metaData.getTables(null, null, null, tableTypes?.toTypedArray()) val dataFrameSchemas = mutableMapOf() @@ -904,6 +909,27 @@ public fun DataFrame.Companion.getSchemaForAllSqlTables( } } + // We may have no tables or filtered for the wrong table type, let's give a helpful error message + if (dataFrameSchemas.isEmpty()) { + val supportedTableTypes = metaData.tableTypes.let { + buildList { while (it.next()) add(it.getString("TABLE_TYPE")) } + } + if (tableTypes?.any { it !in supportedTableTypes } == true) { + val unsupportedTypes = tableTypes.filter { it !in supportedTableTypes } + throw IllegalArgumentException( + buildString { + appendLine("Found no tables with type(s) $tableTypes.") + if (unsupportedTypes.isNotEmpty()) { + appendLine( + "Table type(s) $unsupportedTypes are unsupported for ${determinedDbType::class.simpleName}.", + ) + } + appendLine("If this is unexpected, try adjusting `tableTypes=` to any of $supportedTableTypes.") + }, + ) + } + } + return dataFrameSchemas }