diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 0ab501d2e0..acb6686b9c 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -6,6 +6,8 @@ priorities and future plans. ## Work in progress +- Preserve arbitrary bytes in SQLite text columns across DBI fetches (#1923). + - Run MakeMaker `MY::post_initialize` hooks so generated files join `pm_to_blib`, and parse Perl's `$]` version variable. - Preserve hash-reference constant calls after typeglob undefinition, dereference glob-backed hash values during Morpheus configuration traversal, parse Error.pm's dynamic `catch $class with { ... }` form on both backends, and retain integer scalar identity through `int()` for decimal stringification; preserve regex referent types through blessed scalar references and cover Morpheus relational overload dispatch and caller filename normalization (#1342, #1376, #1427, #1428). diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/DBI.java b/src/main/java/org/perlonjava/runtime/perlmodule/DBI.java index baf255dc2a..86d29576d3 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/DBI.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/DBI.java @@ -6,6 +6,10 @@ import org.perlonjava.runtime.regex.RuntimeRegex; import org.sqlite.Function; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.Enumeration; @@ -599,17 +603,9 @@ public static RuntimeList fetchrow_arrayref(RuntimeArray args, int ctx) { ResultSetMetaData metaData = rs.getMetaData(); int colCount = metaData.getColumnCount(); boolean sqliteUnicode = usesSqliteUnicodeStrings(dbh); - // Convert each column value to string and add to row array. - // Perl 5's DBD::SQLite (without sqlite_unicode) returns byte strings - // (no UTF-8 flag). JDBC returns Java Strings which are decoded Unicode. - // To match Perl 5 behavior, we must UTF-8 encode the JDBC string and - // return it as BYTE_STRING. This is equivalent to sqlite_unicode=0. - // - // Why: In Perl 5, DBD::SQLite works at the byte level — strings go in - // as raw bytes (UTF-8 encoded for STRING, raw for BYTE_STRING) and come - // back as raw bytes without the UTF-8 flag. JDBC works at the character - // level — it always decodes UTF-8 on fetch. Re-encoding to UTF-8 bytes - // here restores the byte-level behavior that Perl code expects. + // DBD::SQLite normally returns byte strings. SQLite TEXT can also + // contain arbitrary bytes, so sqliteTextScalar reads the raw bytes + // before JDBC's String conversion can replace invalid UTF-8 sequences. for (int i = 1; i <= colCount; i++) { Object obj = rs.getObject(i); RuntimeScalar val; @@ -618,6 +614,8 @@ public static RuntimeList fetchrow_arrayref(RuntimeArray args, int ctx) { // a Java String via ISO-8859-1 and tagging as BYTE_STRING. val = new RuntimeScalar(new String(bytes, StandardCharsets.ISO_8859_1)); val.type = RuntimeScalarType.BYTE_STRING; + } else if (obj instanceof String && isSqliteConnection(dbh)) { + val = sqliteTextScalar(rs, i, sqliteUnicode); } else { val = RuntimeScalar.newScalarOrString(obj); if (!sqliteUnicode && val.type == RuntimeScalarType.STRING && val.value instanceof String s) { @@ -674,6 +672,33 @@ private static boolean usesSqliteUnicodeStrings(RuntimeHash dbh) { return stringMode != null && stringMode.getInt() == 5; } + private static boolean isSqliteConnection(RuntimeHash dbh) { + RuntimeScalar name = dbh.get("Name"); + return name != null && name.toString().toLowerCase().startsWith("jdbc:sqlite:"); + } + + /** + * SQLite TEXT can contain arbitrary bytes despite its storage class. JDBC's + * getObject() decodes those bytes as UTF-8 and replaces invalid sequences; + * fetch the raw column bytes so DBD::SQLite's byte-string behavior is kept. + */ + private static RuntimeScalar sqliteTextScalar(ResultSet rs, int column, boolean sqliteUnicode) + throws SQLException { + byte[] bytes = rs.getBytes(column); + if (!sqliteUnicode) { + return new RuntimeScalar(bytes); + } + try { + CharBuffer decoded = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)); + return new RuntimeScalar(decoded.toString()); + } catch (CharacterCodingException invalidUtf8) { + return new RuntimeScalar(bytes); + } + } + private static RuntimeArray columnNamesAttribute(RuntimeHash sth, String key) { if (sth == null) { return null; @@ -758,7 +783,7 @@ public static RuntimeList fetchrow_hashref(RuntimeArray args, int ctx) { } // For each column, add column name -> value pair to hash. - // See fetchrow_arrayref for rationale on UTF-8 encode to BYTE_STRING. + // Use the same raw-byte handling as fetchrow_arrayref for SQLite TEXT. for (int i = 1; i <= colCount; i++) { String columnName = columnNames.get(i - 1).toString(); Object value = rs.getObject(i); @@ -766,6 +791,8 @@ public static RuntimeList fetchrow_hashref(RuntimeArray args, int ctx) { if (value instanceof byte[] bytes) { val = new RuntimeScalar(new String(bytes, StandardCharsets.ISO_8859_1)); val.type = RuntimeScalarType.BYTE_STRING; + } else if (value instanceof String && isSqliteConnection(dbh)) { + val = sqliteTextScalar(rs, i, sqliteUnicode); } else { val = RuntimeScalar.newScalarOrString(value); if (!sqliteUnicode && val.type == RuntimeScalarType.STRING && val.value instanceof String s) { diff --git a/src/test/resources/unit/dbi_sqlite_invalid_text_bytes.t b/src/test/resources/unit/dbi_sqlite_invalid_text_bytes.t new file mode 100644 index 0000000000..9c9661a00c --- /dev/null +++ b/src/test/resources/unit/dbi_sqlite_invalid_text_bytes.t @@ -0,0 +1,35 @@ +use strict; +use warnings; +use Test::More; +use DBI; +use DBD::SQLite; + +my $dbh = DBI->connect( + 'dbi:SQLite:dbname=:memory:', + '', + '', + { RaiseError => 1, PrintError => 0 }, +); + +$dbh->do('CREATE TABLE cookies (encrypted_value BLOB)'); + +# Chrome cookie databases can contain arbitrary ciphertext bytes in a TEXT +# storage value, even when the declared column type is BLOB. +my $encrypted = "v10" . ("\x83" x 64); +my $hex = unpack 'H*', $encrypted; +$dbh->do("INSERT INTO cookies VALUES (CAST(X'$hex' AS TEXT))"); + +my $array_row = $dbh->selectrow_arrayref('SELECT encrypted_value FROM cookies'); +is(length($array_row->[0]), length($encrypted), + 'array fetch preserves the byte length of invalid UTF-8 TEXT'); +is(unpack('H*', $array_row->[0]), $hex, + 'array fetch preserves invalid UTF-8 TEXT bytes'); + +my $hash_row = $dbh->selectrow_hashref('SELECT encrypted_value FROM cookies'); +is(length($hash_row->{encrypted_value}), length($encrypted), + 'hash fetch preserves the byte length of invalid UTF-8 TEXT'); +is(unpack('H*', $hash_row->{encrypted_value}), $hex, + 'hash fetch preserves invalid UTF-8 TEXT bytes'); + +$dbh->disconnect; +done_testing;