From 9cc7d160ae5499a6bb1a1d18c95f34a5d7e50c46 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 20 Oct 2026 22:14:34 +0200 Subject: [PATCH] fix(calcite): quote SQL identifiers with double quotes, as SQL tools generate them IntelliJ opens a table with ORDER BY "date" (it quotes a reserved-word column with the standard double quote), which a back-tick-only connection could not parse. getIdentifierQuoteString now reports the quote the parser accepts. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../dfa1/vortex/calcite/VortexCalcite.java | 14 ++++--- .../vortex/calcite/ColumnTypesSqlTest.java | 2 +- .../dfa1/vortex/calcite/OhlcSqlDemoTest.java | 2 +- .../vortex/calcite/VortexCalciteTest.java | 40 +++++++++++++++++-- docs/how-to.md | 13 +++--- docs/reference.md | 5 ++- .../CalciteInteropIntegrationTest.java | 6 +-- 8 files changed, 62 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e20fd52..e58c24c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed +- **Breaking:** SQL identifiers are quoted with double quotes, as IntelliJ and DBeaver generate them, instead of back-ticks, so opening a table or sorting its grid works; write `"date"` where you wrote `` `date` ``. ([#588](https://github.com/dfa1/vortex-java/pull/588)) - **Breaking:** the `ScanIterator` constructor and the reader's Pco helpers (`PcoTansDecoder`, `PcoBin`, `LeBitReader`) are no longer public, and the built-in extension codecs are reached only through their `INSTANCE`. ([4ba6ea12](https://github.com/dfa1/vortex-java/commit/4ba6ea12)) - SQL `ORDER BY ... LIMIT` on a string column is up to 3 times faster: rows that cannot make the top are ranked by their bytes without building a `String`, and a chunk stored in the opposite order is read backward. ([#575](https://github.com/dfa1/vortex-java/pull/575)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java index 8f9799dd3..fc7ad5d09 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java @@ -26,15 +26,15 @@ /// } /// ``` /// -/// The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, back-tick -/// quoting, unquoted case preserved) together with the **Babel** SQL parser, which accepts most SQL +/// The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, unquoted case +/// preserved) with standard double-quote identifier quoting, together with the **Babel** SQL parser, which accepts most SQL /// reserved words as identifiers. A column whose name is a reserved word — `close`, `open`, `value`, /// `year`, … — can therefore be queried **unquoted**: `select close, open from vtx.ohlc`, which /// columnar files routinely need. /// /// The exception is the handful of keywords that begin a typed literal — `date`, `time`, /// `timestamp`, `interval` — which stay reserved even under Babel (`date` opens `DATE '2020-01-01'`). -/// A column with one of those names must still be back-tick quoted: `select \`date\` from vtx.ohlc`. +/// A column with one of those names must still be double-quoted: `select "date" from vtx.ohlc`. public final class VortexCalcite { // One instance: building a driver loads a resource and sets up Avatica's factory by reflection. @@ -81,9 +81,13 @@ public static Connection connect(Path directory) throws SQLException { public static Connection connect(VortexCalciteOptions options) throws SQLException { Objects.requireNonNull(options, "options"); Properties info = new Properties(); - // JAVA lex: case-sensitive identifiers, back-tick quoting, unquoted case preserved — so an - // unquoted `ohlc` matches the registered table name exactly. + // JAVA lex: case-sensitive identifiers, unquoted case preserved — so an unquoted `ohlc` + // matches the registered table name exactly. info.setProperty("lex", "JAVA"); + // Standard double-quote identifiers, not JAVA's back-ticks: SQL tools (IntelliJ, DBeaver) + // generate "name" for a reserved-word column such as "date" in their own statements, e.g. + // a table opened sorted, whatever quote the driver reports. + info.setProperty("quoting", "DOUBLE_QUOTE"); // Babel parser: accept SQL reserved words (date, close, open, value, …) as identifiers, so // columnar files whose column names collide with keywords are queryable unquoted. info.setProperty("parserFactory", SqlBabelParserImpl.class.getName() + "#FACTORY"); diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/ColumnTypesSqlTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/ColumnTypesSqlTest.java index 511e954d3..c450f2945 100644 --- a/calcite/src/test/java/io/github/dfa1/vortex/calcite/ColumnTypesSqlTest.java +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/ColumnTypesSqlTest.java @@ -83,7 +83,7 @@ static void write() throws Exception { @Test void everyScalarType_readsThroughSql() throws Exception { // Given - String sql = "select amount, blob, `day`, at, wall, `instant`, `uuid` from vtx.t"; + String sql = "select amount, blob, \"day\", at, wall, \"instant\", \"uuid\" from vtx.t"; // When try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file)); diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/OhlcSqlDemoTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/OhlcSqlDemoTest.java index f39822322..e7ffa6eb4 100644 --- a/calcite/src/test/java/io/github/dfa1/vortex/calcite/OhlcSqlDemoTest.java +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/OhlcSqlDemoTest.java @@ -107,7 +107,7 @@ void variousQueryShapes() throws Exception { long upDays; try (Statement st = conn.createStatement(); // `close` and `open` are SQL reserved words (CLOSE/OPEN cursor) — must be quoted. - ResultSet rs = st.executeQuery("select count(*) c from vtx.ohlc where `close` > `open`")) { + ResultSet rs = st.executeQuery("select count(*) c from vtx.ohlc where \"close\" > \"open\"")) { rs.next(); upDays = rs.getLong("c"); } diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteTest.java index b28317e66..114b71089 100644 --- a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteTest.java +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteTest.java @@ -70,13 +70,13 @@ void connect_reservedWordColumnsAreQueryableUnquoted() throws Exception { @Test void connect_typeKeywordColumnStillNeedsQuoting() throws Exception { // Given the `date` column: even Babel keeps DATE reserved (it begins a date literal, - // DATE '2020-01-01'), so it is the residual case that still needs back-ticks + // DATE '2020-01-01'), so it is the residual case that still needs quoting try (Connection conn = VortexCalcite.connect("vtx", Map.of("ohlc", file))) { - // When `date` is back-tick quoted + // When `date` is double-quoted long rows = 0; try (Statement st = conn.createStatement(); - ResultSet rs = st.executeQuery("select `date` from vtx.ohlc")) { + ResultSet rs = st.executeQuery("select \"date\" from vtx.ohlc")) { while (rs.next()) { rows++; } @@ -87,6 +87,40 @@ void connect_typeKeywordColumnStillNeedsQuoting() throws Exception { } } + @Test + void connect_sqlToolSortOnTypeKeywordColumn_runs() throws Exception { + // Given the query IntelliJ's table editor sends when a table is opened sorted by `date`: + // it quotes with the standard double quote, whatever the driver's own quoting is, so a + // connection that only understood back-ticks failed to parse it + try (Connection conn = VortexCalcite.connect("vtx", Map.of("ohlc", file))) { + + // When it runs + long rows = 0; + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT t.*\nFROM vtx.ohlc t\nORDER BY \"date\" DESC")) { + while (rs.next()) { + rows++; + } + } + + // Then it parses and returns every row + assertThat(rows).isEqualTo(20_000); + } + } + + @Test + void connect_identifierQuoteString_isTheDoubleQuoteTheParserAccepts() throws Exception { + // Given a connection + try (Connection conn = VortexCalcite.connect("vtx", Map.of("ohlc", file))) { + + // When a tool asks which quote to wrap identifiers in + String result = conn.getMetaData().getIdentifierQuoteString(); + + // Then it is the one the parser accepts + assertThat(result).isEqualTo("\""); + } + } + @Test void connect_nullArguments_throw() { // Given / When / Then — both arguments are required diff --git a/docs/how-to.md b/docs/how-to.md index d509df861..8a98de9e5 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -586,7 +586,7 @@ segment is decoded — on files with the legacy `vortex.stats` zone map (written `WriteOptions.defaults().withEdition(Editions.CORE_2025_10_0)`). Files with the default `vortex.zoned` zone map answer them through a scan. Column names that collide with SQL reserved words (`close`, `open`, `value`, `year`, …) work unquoted; the handful that open a typed literal (`date`, `time`, `timestamp`, -`interval`) still need back-ticks: `` select `date` from vtx.ohlc ``. See +`interval`) still need double quotes: `select "date" from vtx.ohlc`. See [reference.md#calcite-sql-adapter](reference.md#calcite-sql-adapter) for the full lexical/parser policy. @@ -635,12 +635,13 @@ The directory is scanned when the connection opens, so a file added later needs ```sql select symbol, count(*), max(high) from vtx.ohlc group by symbol order by 2 desc; -select close, open from vtx.ohlc where `date` > 20000 limit 100; +select close, open from vtx.ohlc where "date" > 20000 limit 100; ``` -- **Quote with back-ticks, not double quotes.** Identifiers are case-sensitive, and `"x"` is not an - identifier. `date`, `time`, `timestamp` and `interval` still need back-ticks even as column names; - other reserved words such as `close`, `open` and `value` work unquoted. +- **Quote with double quotes, not back-ticks.** Identifiers are case-sensitive, and `` `x` `` is not + an identifier. `date`, `time`, `timestamp` and `interval` still need double quotes even as column + names; other reserved words such as `close`, `open` and `value` work unquoted. IntelliJ and + DBeaver generate `"name"` themselves, so opening a table or sorting its grid works as is. - **Filters and `LIMIT` are cheap.** A `WHERE` on a clustered column skips whole chunks using the files' zone maps, and the data grid only reads the page it shows. - **Low-cardinality string columns filter fastest.** A `WHERE symbol = 'AAPL'` (or `IN`, `<>`, a @@ -659,7 +660,7 @@ select close, open from vtx.ohlc where `date` > 20000 limit 100; | `class file has wrong version` or `UnsupportedClassVersionError` | The JDBC process runs on a JDK older than 25. Set the project or IDE SDK to 25+. | | `invalid Vortex URL ... unknown key` | A misspelled option; the message lists the keys that exist. | | `Object 'OHLC' not found ... did you mean 'ohlc'?` | Names are case-sensitive; write `ohlc` as the file is named. | -| `parse failed: Encountered ". \""` | A double-quoted identifier; use back-ticks. | +| ``Lexical error ... Encountered: "`"`` | A back-ticked identifier, accepted before 0.19; use double quotes. | On JDK 25 the first query prints a `sun.misc.Unsafe` warning from protobuf, a Calcite dependency; it is harmless. diff --git a/docs/reference.md b/docs/reference.md index 1f5dc20ff..80d81b4c4 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -627,10 +627,11 @@ unquoted, also add `parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelPa [`VortexCalcite`] sets both for you. The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, unquoted case -preserved) and the **Babel** SQL parser, so a column named after a reserved word — `close`, `open`, +preserved) with standard double-quote identifiers, and the **Babel** SQL parser, so a column named after a reserved word — `close`, `open`, `value`, `year`, … — is queryable unquoted (`select close, open from vtx.ohlc`). The exception is keywords that open a typed literal (`date`, `time`, `timestamp`, `interval`), which stay reserved -even under Babel and need back-ticks: `` select `date` from vtx.ohlc ``. +even under Babel and need double quotes: `select "date" from vtx.ohlc`. Back-ticks, which the +connection accepted before, are no longer identifier quotes. A `WHERE` of column-vs-literal comparisons, `IN` lists, `IS [NOT] NULL` and prefix `LIKE 'abc%'` patterns, joined by `AND` (an `OR` only within one column), prunes chunks from zone maps and is diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java index e493ece78..1d05b9a70 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java @@ -152,7 +152,7 @@ void jniWriter_temporalDecimalAndBinaryColumns_readThroughSql(@TempDir Path tmp) // Given — a file with a timestamp could not be queried at all: the row type threw Path file = tmp.resolve("jni_types.vortex"); writeTypesWithJni(file); - String sql = "select `day`, at, wall, `instant`, amount, blob from vtx.t"; + String sql = "select \"day\", at, wall, \"instant\", amount, blob from vtx.t"; // When try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file)); @@ -417,9 +417,9 @@ void jniWriter_likePrefix_returnsTheRowsCalciteReturns(String pattern, @TempDir @ParameterizedTest(name = "{0}") @CsvSource(delimiter = '|', value = { - "min(`day`), max(`day`) | min(`day` + interval '0' day), max(`day` + interval '0' day)", + "min(\"day\"), max(\"day\") | min(\"day\" + interval '0' day), max(\"day\" + interval '0' day)", "min(at), max(at) | min(at + interval '0' second), max(at + interval '0' second)", - "min(wall), max(`instant`) | min(wall + interval '0' second), max(`instant` + interval '0' second)"}) + "min(wall), max(\"instant\") | min(wall + interval '0' second), max(\"instant\" + interval '0' second)"}) void jniWriter_temporalMinMax_answeredFromStatsEqualsTheScan(String aggregates, String oracle, @TempDir Path tmp) throws Exception { // Given — Rust's zone statistics hold the raw storage integer (days, millis, micros before