diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java index 69adc0d1c..65b6a8158 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java @@ -23,11 +23,16 @@ import java.sql.SQLType; import java.sql.Statement; import java.sql.Types; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wrapper { private static final Logger log = LoggerFactory.getLogger(DatabaseMetaDataImpl.class); @@ -38,12 +43,13 @@ public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wr public enum TableType { DICTIONARY("DICTIONARY"), LOG_TABLE("LOG TABLE"), + MATERIALIZED_VIEW("MATERIALIZED VIEW"), MEMORY_TABLE("MEMORY TABLE"), REMOTE_TABLE("REMOTE TABLE"), - TABLE("TABLE"), - VIEW("VIEW"), SYSTEM_TABLE("SYSTEM TABLE"), - TEMPORARY_TABLE("TEMPORARY TABLE"); + TABLE("TABLE"), + TEMPORARY_TABLE("TEMPORARY TABLE"), + VIEW("VIEW"); private final String typeName; @@ -55,8 +61,9 @@ public String getTypeName() { return typeName; } } - public static final String[] TABLE_TYPES = new String[] { "DICTIONARY", "LOG TABLE", "MEMORY TABLE", - "REMOTE TABLE", "TABLE", "VIEW", "SYSTEM TABLE", "TEMPORARY TABLE" }; + + static final Set TABLE_TYPES = Arrays.stream(TableType.values()).map(TableType::getTypeName).collect(Collectors.toSet()); + private static final String TEMPORARY_ENGINE_PREFIX = "Temporary"; private static final String DATABASE_PRODUCT_NAME = "ClickHouse"; private static final String DRIVER_NAME = DATABASE_PRODUCT_NAME + " JDBC Driver"; @@ -764,6 +771,151 @@ public ResultSet getProcedureColumns(String catalog, String schemaPattern, Strin } } + // Map of ClickHouse engine names to JDBC table types + static final Map ENGINE_TO_TABLE_TYPE; + static { + Map map = new java.util.HashMap<>(); + + // Log tables + map.put("Log", TableType.LOG_TABLE.getTypeName()); + map.put("StripeLog", TableType.LOG_TABLE.getTypeName()); + map.put("TinyLog", TableType.LOG_TABLE.getTypeName()); + + // Memory tables + map.put("Buffer", TableType.MEMORY_TABLE.getTypeName()); + map.put("Memory", TableType.MEMORY_TABLE.getTypeName()); + map.put("Set", TableType.MEMORY_TABLE.getTypeName()); + + // Views + map.put("View", TableType.VIEW.getTypeName()); + map.put("LiveView", TableType.VIEW.getTypeName()); + map.put("MaterializedView", TableType.MATERIALIZED_VIEW.getTypeName()); + map.put("WindowView", TableType.VIEW.getTypeName()); + + // Dictionary + map.put("Dictionary", TableType.DICTIONARY.getTypeName()); + + // Remote/External tables + map.put("AzureBlobStorage", TableType.REMOTE_TABLE.getTypeName()); + map.put("AzureQueue", TableType.REMOTE_TABLE.getTypeName()); + map.put("COSN", TableType.REMOTE_TABLE.getTypeName()); + map.put("ArrowFlight", TableType.REMOTE_TABLE.getTypeName()); + map.put("DeltaLake", TableType.REMOTE_TABLE.getTypeName()); + map.put("DeltaLakeAzure", TableType.REMOTE_TABLE.getTypeName()); + map.put("DeltaLakeLocal", TableType.REMOTE_TABLE.getTypeName()); + map.put("DeltaLakeS3", TableType.REMOTE_TABLE.getTypeName()); + map.put("Distributed", TableType.REMOTE_TABLE.getTypeName()); + map.put("FuzzJSON", TableType.REMOTE_TABLE.getTypeName()); + map.put("FuzzQuery", TableType.REMOTE_TABLE.getTypeName()); + map.put("GCS", TableType.REMOTE_TABLE.getTypeName()); + map.put("GenerateRandom", TableType.REMOTE_TABLE.getTypeName()); + map.put("HDFS", TableType.REMOTE_TABLE.getTypeName()); + map.put("Hive", TableType.REMOTE_TABLE.getTypeName()); + map.put("Hudi", TableType.REMOTE_TABLE.getTypeName()); + map.put("Iceberg", TableType.REMOTE_TABLE.getTypeName()); + map.put("IcebergAzure", TableType.REMOTE_TABLE.getTypeName()); + map.put("IcebergHDFS", TableType.REMOTE_TABLE.getTypeName()); + map.put("IcebergLocal", TableType.REMOTE_TABLE.getTypeName()); + map.put("IcebergS3", TableType.REMOTE_TABLE.getTypeName()); + map.put("JDBC", TableType.REMOTE_TABLE.getTypeName()); + map.put("Kafka", TableType.REMOTE_TABLE.getTypeName()); + map.put("MaterializedPostgreSQL", TableType.REMOTE_TABLE.getTypeName()); + map.put("MongoDB", TableType.REMOTE_TABLE.getTypeName()); + map.put("MySQL", TableType.REMOTE_TABLE.getTypeName()); + map.put("NATS", TableType.REMOTE_TABLE.getTypeName()); + map.put("ODBC", TableType.REMOTE_TABLE.getTypeName()); + map.put("OSS", TableType.REMOTE_TABLE.getTypeName()); + map.put("PostgreSQL", TableType.REMOTE_TABLE.getTypeName()); + map.put("RabbitMQ", TableType.REMOTE_TABLE.getTypeName()); + map.put("Redis", TableType.REMOTE_TABLE.getTypeName()); + map.put("S3", TableType.REMOTE_TABLE.getTypeName()); + map.put("S3Queue", TableType.REMOTE_TABLE.getTypeName()); + map.put("URL", TableType.REMOTE_TABLE.getTypeName()); + map.put("YTsaurus", TableType.REMOTE_TABLE.getTypeName()); + + // Regular tables (MergeTree family and others) + map.put("AggregatingMergeTree", TableType.TABLE.getTypeName()); + map.put("Alias", TableType.TABLE.getTypeName()); + map.put("CoalescingMergeTree", TableType.TABLE.getTypeName()); + map.put("CollapsingMergeTree", TableType.TABLE.getTypeName()); + map.put("EmbeddedRocksDB", TableType.TABLE.getTypeName()); + map.put("Executable", TableType.TABLE.getTypeName()); + map.put("ExecutablePool", TableType.TABLE.getTypeName()); + map.put("GraphiteMergeTree", TableType.TABLE.getTypeName()); + map.put("Join", TableType.TABLE.getTypeName()); + map.put("KeeperMap", TableType.TABLE.getTypeName()); + map.put("Merge", TableType.TABLE.getTypeName()); + map.put("MergeTree", TableType.TABLE.getTypeName()); + map.put("ReplacingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedAggregatingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedCoalescingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedCollapsingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedGraphiteMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedReplacingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedSummingMergeTree", TableType.TABLE.getTypeName()); + map.put("ReplicatedVersionedCollapsingMergeTree", TableType.TABLE.getTypeName()); + map.put("SummingMergeTree", TableType.TABLE.getTypeName()); + map.put("VersionedCollapsingMergeTree", TableType.TABLE.getTypeName()); + + // Special + map.put("TimeSeries", TableType.TABLE.getTypeName()); + map.put("Null", TableType.TABLE.getTypeName()); + map.put("Loop", TableType.TABLE.getTypeName()); + map.put("SQLite", TableType.TABLE.getTypeName()); + map.put("File", TableType.TABLE.getTypeName()); + map.put("FileLog", TableType.TABLE.getTypeName()); + + ENGINE_TO_TABLE_TYPE = Collections.unmodifiableMap(map); + } + + /** + * Converts engine name to table type. Returns TABLE as default for unknown engines. + */ + private static String engineToTableType(String engine) { + if (engine == null) { + return TableType.TABLE.getTypeName(); + } + // Check for system tables (engines starting with "System" or "Async") + if (engine.startsWith("System") || engine.startsWith("Async")) { + return TableType.SYSTEM_TABLE.getTypeName(); + } + return ENGINE_TO_TABLE_TYPE.getOrDefault(engine, TableType.TABLE.getTypeName()); + } + + /** + * Returns set of engines that map to any of the given table types. + */ + private static Set getEnginesForTableTypes(Set requestedTypes) { + Set engines = new HashSet<>(); + + for (Map.Entry entry : ENGINE_TO_TABLE_TYPE.entrySet()) { + if (requestedTypes.contains(entry.getValue())) { + engines.add(entry.getKey()); + } + } + + return engines; + } + + private static final Consumer> TABLE_TYPE_MUTATOR = row -> { + String engine = (String) row.get("TABLE_TYPE"); + + String tableType; + if (engine != null && engine.startsWith(TEMPORARY_ENGINE_PREFIX)) { + tableType = TableType.TEMPORARY_TABLE.getTypeName(); + } else if (engine != null && (engine.startsWith("System") || engine.startsWith("Async"))) { + tableType = TableType.SYSTEM_TABLE.getTypeName(); + } else { + tableType = engineToTableType(engine); + } + + row.put("TABLE_TYPE", tableType); + }; + + private static final Collection>> GET_TABLES_MUTATORS = Collections.singletonList(TABLE_TYPE_MUTATOR); + + /** * Returns tables defined for a schema. Parameter {@code catalog} is ignored * @@ -774,24 +926,47 @@ public ResultSet getProcedureColumns(String catalog, String schemaPattern, Strin public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { log.debug("getTables: catalog={}, schemaPattern={}, tableNamePattern={}, types={}", catalog, schemaPattern, tableNamePattern, types); // TODO: when switch between catalog and schema is implemented, then TABLE_SCHEMA and TABLE_CAT should be populated accordingly -// String commentColumn = connection.getServerVersion().check("[21.6,)") ? "t.comment" : "''"; // TODO: handle useCatalogs == true and return schema catalog name - if (types == null || types.length == 0) { - types = TABLE_TYPES; + + // Get engines that map to the requested table types + Set requestedTypes = (types == null || types.length == 0) ? TABLE_TYPES : Arrays.stream(types).collect(Collectors.toSet()) ; + Set engines = getEnginesForTableTypes(requestedTypes); + + // Build engine filter conditions + List filterConditions = new ArrayList<>(); + + // Add condition for engines that map to requested types + if (!engines.isEmpty()) { + filterConditions.add("(t.engine IN ('" + String.join("','", engines) + "'))"); + } + + // If TABLE type is requested, also include engines not in our map (they default to TABLE) + if (requestedTypes.contains(TableType.TABLE.getTypeName())) { + filterConditions.add("(t.engine NOT IN ('" + String.join("','", ENGINE_TO_TABLE_TYPE.keySet()) + + "') AND NOT t.engine LIKE 'System%' AND NOT t.engine LIKE 'Async%' AND t.is_temporary = 0)"); + } + + // If SYSTEM TABLE is requested, include system engines (System* and Async*) + if (requestedTypes.contains(TableType.SYSTEM_TABLE.getTypeName())) { + filterConditions.add("(t.engine LIKE 'System%' OR t.engine LIKE 'Async%')"); + } + + // If TEMPORARY TABLE is requested, include temporary tables + if (requestedTypes.contains(TableType.TEMPORARY_TABLE.getTypeName())) { + filterConditions.add("(t.is_temporary = 1)"); + } + + String engineFilter = filterConditions.isEmpty() ? "" : " AND ( " + String.join(" OR ", filterConditions) + ")"; + // Exclude temporary tables when not requested (they would otherwise match engine-based conditions) + if (!requestedTypes.contains(TableType.TEMPORARY_TABLE.getTypeName())) { + engineFilter += " AND (t.is_temporary = 0)"; } String sql = "SELECT " + catalogPlaceholder + " AS TABLE_CAT, " + "t.database AS TABLE_SCHEM, " + "t.name AS TABLE_NAME, " + - "CASE WHEN t.engine LIKE '%Log' THEN 'LOG TABLE' " + - "WHEN t.engine in ('Buffer', 'Memory', 'Set') THEN 'MEMORY TABLE' " + - "WHEN t.is_temporary != 0 THEN 'TEMPORARY TABLE' " + - "WHEN t.engine like '%View' THEN 'VIEW'" + - "WHEN t.engine = 'Dictionary' THEN 'DICTIONARY' " + - "WHEN t.engine LIKE 'Async%' OR t.engine LIKE 'System%' THEN 'SYSTEM TABLE' " + - "WHEN empty(t.data_paths) THEN 'REMOTE TABLE' " + - "ELSE 'TABLE' END AS TABLE_TYPE, " + + "if(t.is_temporary = 1, concat('Temporary', t.engine), t.engine) AS TABLE_TYPE, " + "t.comment AS REMARKS, " + "CAST(null as Nullable(String)) AS TYPE_CAT, " + // no types catalog "d.engine AS TYPE_SCHEM, " + // no types schema @@ -802,10 +977,10 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam " JOIN system.databases d ON system.tables.database = system.databases.name" + " WHERE t.database LIKE '" + (schemaPattern == null ? "%" : schemaPattern) + "'" + " AND t.name LIKE '" + (tableNamePattern == null ? "%" : tableNamePattern) + "'" + - " AND TABLE_TYPE IN ('" + String.join("','", types) + "')"; + engineFilter; - try { - return connection.createStatement().executeQuery(sql); + try (Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery(sql)) { + return DetachedResultSet.createFromResultSet(rs, connection.getDefaultCalendar(), GET_TABLES_MUTATORS); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } @@ -844,6 +1019,8 @@ public ResultSet getCatalogs() throws SQLException { } } + + static final String TABLE_TYPES_SQL_ARRAY = Arrays.stream(TableType.values()).map(TableType::getTypeName).collect(Collectors.joining("','")); /** * Returns name of the ClickHouse table types as the broad category (rather than engine name). * @return - ResultSet with one column TABLE_TYPE @@ -852,7 +1029,7 @@ public ResultSet getCatalogs() throws SQLException { @Override public ResultSet getTableTypes() throws SQLException { try { - return connection.createStatement().executeQuery("SELECT arrayJoin(['" + String.join("','", TABLE_TYPES) + "']) AS TABLE_TYPE ORDER BY TABLE_TYPE"); + return connection.createStatement().executeQuery("SELECT arrayJoin(['" + TABLE_TYPES_SQL_ARRAY + "']) AS TABLE_TYPE ORDER BY TABLE_TYPE"); } catch (Exception e) { throw ExceptionUtils.toSqlState(e); } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index 6c80df583..7fdddf505 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -17,8 +17,10 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; @@ -481,8 +483,7 @@ public void testGetTableTypes() throws Exception { try (Connection conn = getJdbcConnection()) { DatabaseMetaData dbmd = conn.getMetaData(); ResultSet rs = dbmd.getTableTypes(); - List sortedTypes = Arrays.asList(DatabaseMetaDataImpl.TABLE_TYPES); - Collections.sort(sortedTypes); + List sortedTypes = Arrays.stream(DatabaseMetaDataImpl.TableType.values()).map(DatabaseMetaDataImpl.TableType::getTypeName).sorted().collect(Collectors.toList()); for (String type: sortedTypes) { assertTrue(rs.next()); assertEquals(rs.getString("TABLE_TYPE"), type); @@ -515,6 +516,27 @@ public void testGetTablesReturnKnownTableTypes() throws Exception { } } + @Test(groups = { "integration" }) + public void testAllTableEnginesFromSystemTableEnginesAreMapped() throws Exception { + try (Connection conn = getJdbcConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT name FROM system.table_engines ORDER BY name")) { + Set mappedEngines = DatabaseMetaDataImpl.ENGINE_TO_TABLE_TYPE.keySet(); + List unmapped = new ArrayList<>(); + while (rs.next()) { + String engine = rs.getString("name"); + boolean isMapped = mappedEngines.contains(engine) + || (engine != null && (engine.startsWith("System") || engine.startsWith("Async"))); + if (!isMapped) { + unmapped.add(engine); + } + } + assertTrue(unmapped.isEmpty(), + "All table engines from system.table_engines must be mapped in DatabaseMetaDataImpl.ENGINE_TO_TABLE_TYPE " + + "or handled by System/Async prefix. Unmapped engines: " + unmapped); + } + } + @Test(groups = { "integration" }, enabled = false) public void testGetColumnsWithEmptyCatalog() throws Exception { // test not relevant until catalogs are implemented @@ -1389,4 +1411,109 @@ public void testGetDatabaseMinorVersion() throws Exception { assertEquals(minorVersion, minorVersionOfServer, "Minor version"); } } + + @Test(groups = {"integration"}) + public void testTableTypes() throws Exception { + final String database = getDatabase(); + + // Map of table name -> (schema, expected type), ordered alphabetically by type + java.util.Map knownTables = new java.util.LinkedHashMap<>(); + knownTables.put("test_table_types_dict", new String[]{database, "DICTIONARY"}); + knownTables.put("test_table_types_log", new String[]{database, "LOG TABLE"}); + knownTables.put("test_table_types_mat_view", new String[]{database, "MATERIALIZED VIEW"}); + knownTables.put("test_table_types_memory", new String[]{database, "MEMORY TABLE"}); + knownTables.put("test_table_types_remote", new String[]{database, "REMOTE TABLE"}); + knownTables.put("numbers", new String[]{"system", "SYSTEM TABLE"}); + knownTables.put("test_table_types_regular", new String[]{database, "TABLE"}); + knownTables.put("test_table_types_view", new String[]{database, "VIEW"}); + + try (Connection conn = getJdbcConnection()) { + final DatabaseMetaData dbmd = conn.getMetaData(); + + try (Statement stmt = conn.createStatement()) { + // Regular MergeTree table + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_regular"); + stmt.executeUpdate("CREATE TABLE test_table_types_regular (id Int32) ENGINE = MergeTree ORDER BY id"); + + // Source table for views + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_source"); + stmt.executeUpdate("CREATE TABLE test_table_types_source (id Int32) ENGINE = MergeTree ORDER BY id"); + + // Normal view + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_view"); + stmt.executeUpdate("CREATE VIEW test_table_types_view AS SELECT id FROM test_table_types_source"); + + // Materialized view + stmt.executeUpdate("DROP VIEW IF EXISTS test_table_types_mat_view"); + stmt.executeUpdate("CREATE MATERIALIZED VIEW test_table_types_mat_view ENGINE = MergeTree ORDER BY id AS SELECT id FROM test_table_types_source"); + + // Remote table (URL engine has empty data_paths) + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_remote"); + stmt.executeUpdate("CREATE TABLE test_table_types_remote (id Int32) ENGINE = URL('http://localhost:8123/?query=SELECT+1', CSV)"); + + // Log table + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_log"); + stmt.executeUpdate("CREATE TABLE test_table_types_log (id Int32) ENGINE = Log"); + + // Memory table + stmt.executeUpdate("DROP TABLE IF EXISTS test_table_types_memory"); + stmt.executeUpdate("CREATE TABLE test_table_types_memory (id Int32) ENGINE = Memory"); + + // Dictionary (using source table as source) + stmt.executeUpdate("DROP DICTIONARY IF EXISTS test_table_types_dict"); + stmt.executeUpdate("CREATE DICTIONARY test_table_types_dict (id Int32) " + + "PRIMARY KEY id SOURCE(CLICKHOUSE(TABLE 'test_table_types_source' DB '" + database + "')) " + + "LAYOUT(FLAT()) LIFETIME(0)"); + + } + + // Get all types from DatabaseMetaData and verify against expected list + Set allTypes = new HashSet<>(); + try (ResultSet rs = dbmd.getTableTypes()) { + while (rs.next()) { + allTypes.add(rs.getString("TABLE_TYPE")); + } + } + final Set expectedTypes = new HashSet<>(Arrays.asList( + "DICTIONARY", + "LOG TABLE", + "MATERIALIZED VIEW", + "MEMORY TABLE", + "REMOTE TABLE", + "SYSTEM TABLE", + "TABLE", + "TEMPORARY TABLE", // Temporary tables are visible only in the session where they created. + "VIEW" + )); + assertEquals(allTypes, expectedTypes, "Table types from getTableTypes() should match expected types"); + + for (java.util.Map.Entry entry : knownTables.entrySet()) { + String tableName = entry.getKey(); + String schema = entry.getValue()[0]; + String expectedType = entry.getValue()[1]; + + // Test with no filter - table should be returned + try (ResultSet rs = dbmd.getTables(null, schema, tableName, null)) { + assertTrue(rs.next(), tableName + " should be found with no filter"); + assertEquals(rs.getString("TABLE_NAME"), tableName); + assertEquals(rs.getString("TABLE_TYPE"), expectedType); + assertFalse(rs.next()); + } + + // Test with each type filter - table should be returned only when filter matches + for (String filterType : allTypes) { + try (ResultSet rs = dbmd.getTables(null, schema, tableName, new String[]{filterType})) { + if (filterType.equals(expectedType)) { + assertTrue(rs.next(), tableName + " should be found with matching filter " + filterType); + assertEquals(rs.getString("TABLE_NAME"), tableName); + assertEquals(rs.getString("TABLE_TYPE"), expectedType); + assertFalse(rs.next()); + } else { + assertFalse(rs.next(), tableName + " should NOT be found with filter " + filterType); + } + } + } + } + } + } } diff --git a/performance/pom.xml b/performance/pom.xml index b0505ca0a..52f9c41b0 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -17,7 +17,7 @@ 2.0.17 0.9.6-SNAPSHOT 1.37 - 1.20.6 + 2.0.2 3.1.0 3.6.0 @@ -82,7 +82,7 @@ org.testcontainers - clickhouse + testcontainers-clickhouse ${testcontainers.version}