diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java index 6944c6ca5..679129f41 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java @@ -94,6 +94,8 @@ import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -900,18 +902,54 @@ private static boolean isEmptyArray(Operand operand) { && values.isEmpty(); } + private static final Pattern SEMI_JOIN = + Pattern.compile("^A\\.(\\w+) IN \\((SELECT .*)\\)$", Pattern.DOTALL); + + // The negation of a predicate on a joined property, i.e. of a semi-join + // `A. IN (SELECT . FROM ...)`. As `NOT (A. IN (...))` the planner can only + // evaluate the subquery as a SubPlan per candidate (or hash it, if it fits into work_mem), which + // for frequent values takes minutes. `NOT EXISTS` over the unchanged subquery is the same + // predicate (the outer key is never null), and the planner turns it into an anti-join that probes + // the joined table per candidate. Anything but a single semi-join is wrapped in `NOT (...)`. The + // derived table is aliased without AS, which Oracle does not accept there. + static String negate(String predicate) { + Matcher matcher = SEMI_JOIN.matcher(predicate); + if (matcher.matches() && closesAtEnd(predicate, predicate.indexOf(" IN (") + 4)) { + return String.format( + "NOT EXISTS (SELECT 1 FROM (%2$s) NE_ WHERE NE_.%1$s = A.%1$s)", + matcher.group(1), matcher.group(2)); + } + return String.format("NOT (%s)", predicate); + } + + // whether the parenthesis at openIndex is closed by the last character, ignoring parentheses in + // string literals + private static boolean closesAtEnd(String expression, int openIndex) { + int depth = 0; + boolean inLiteral = false; + for (int i = openIndex; i < expression.length(); i++) { + char c = expression.charAt(i); + if (c == '\'') { + inLiteral = !inLiteral; + } else if (!inLiteral && c == '(') { + depth++; + } else if (!inLiteral && c == ')') { + depth--; + if (depth == 0) { + return i == expression.length() - 1; + } + } + } + return false; + } + // As in the other array predicates, a property without values (no row in the joined table) does // not match. So only a property with values contains the empty array, and no property overlaps // with it, is contained by it or is equal to it. The SQL `IN ()` would be invalid. private static String encodeEmptyArray( - ArrayFunction operator, - boolean notInverse, - String mainExpression, - String alias, - String sortKey) { + ArrayFunction operator, boolean notInverse, String mainExpression) { if (notInverse ? operator == A_CONTAINS : operator == A_CONTAINEDBY) { - return String.format( - mainExpression, "", String.format(" IS NOT NULL GROUP BY %s.%s", alias, sortKey)); + return String.format(mainExpression, "", " IS NOT NULL"); } return "1=0"; } @@ -1417,7 +1455,7 @@ public String visit(IsNull isNull, List children) { // contributes no subquery rows at all. "Property has no value" is the negation of // "property has some value" (NOT EXISTS); the outer operand (A.) is never // null, so the negation is exact. - return String.format("NOT (%s)", String.format(mainExpression, "", " IS NOT NULL")); + return negate(String.format(mainExpression, "", " IS NOT NULL")); } // mainExpression is either a literal value or a SELECT expression @@ -1748,25 +1786,32 @@ public String visit(BinaryArrayOperation arrayOperation, List children) if (qualifiedColumn.second().isEmpty()) { if (isEmptyArray(arrayOperation.getArgs().get(notInverse ? 1 : 0))) { - return encodeEmptyArray( - arrayOperation.getArrayOperator(), - notInverse, - mainExpression, - aliases.get(0), - rootSchema.getSortKey().get()); + return encodeEmptyArray(arrayOperation.getArrayOperator(), notInverse, mainExpression); } if (notInverse ? arrayOperation.getArrayOperator() == A_CONTAINS : arrayOperation.getArrayOperator() == A_CONTAINEDBY) { + if (elementCount > 1) { + // "contains all values" is the conjunction of one semi-join per value; unlike a single + // subquery with GROUP BY ... HAVING count(distinct ...) = n, each of them lets the + // planner probe the joined table per candidate + Optional> elements = + renderArrayElements(arrayOperation.getArgs().get(notInverse ? 1 : 0)); + if (elements.isPresent()) { + String semiJoin = mainExpression; + return elements.get().stream() + .map(element -> String.format(semiJoin, "", " IN (" + element + ")")) + .collect(Collectors.joining(" AND ", "(", ")")); + } + } // for a single required value the `HAVING count(distinct …) = 1` is tautological (every - // group selected by `IN ()` trivially has one distinct value); dropping it - // keeps the result identical but lets the planner estimate the grouped row count, which - // it cannot do across a count(distinct) HAVING filter + // group selected by `IN ()` trivially has one distinct value), and without a + // HAVING the GROUP BY is redundant in the `IN` semi-join; dropping both keeps the result + // identical but lets the planner turn the subquery into a semi-join that probes the + // joined table per candidate instead of materializing every matching row String arrayQuery = elementCount == 1 - ? String.format( - " IN %1$s GROUP BY %2$s.%3$s", - secondExpression, aliases.get(0), rootSchema.getSortKey().get()) + ? String.format(" IN %1$s", secondExpression) : String.format( " IN %1$s GROUP BY %2$s.%3$s HAVING count(distinct %4$s) = %5$s", secondExpression, @@ -1786,10 +1831,9 @@ public String visit(BinaryArrayOperation arrayOperation, List children) elementCount); return String.format(mainExpression, "", arrayQuery); } else if (arrayOperation.getArrayOperator() == A_OVERLAPS) { - String arrayQuery = - String.format( - " IN %1$s GROUP BY %2$s.%3$s", - secondExpression, aliases.get(0), rootSchema.getSortKey().get()); + // no GROUP BY: in the `IN` semi-join it would only prevent the planner from probing the + // joined table per candidate + String arrayQuery = String.format(" IN %1$s", secondExpression); return String.format(mainExpression, "", arrayQuery); } else if (notInverse ? arrayOperation.getArrayOperator() == A_CONTAINEDBY @@ -1831,6 +1875,16 @@ public String visit(BinaryArrayOperation arrayOperation, List children) throw new IllegalStateException("unexpected array operator: " + arrayOperation); } + // the SQL literals of the elements of an array literal, if they are available individually + private Optional> renderArrayElements(Operand operand) { + if (operand instanceof ArrayLiteral arrayLiteral + && arrayLiteral.getValue() instanceof List values + && values.stream().allMatch(Scalar.class::isInstance)) { + return Optional.of(values.stream().map(value -> ((Scalar) value).accept(this)).toList()); + } + return Optional.empty(); + } + @Override public String visit(ArrayLiteral arrayLiteral, List children) { if (arrayLiteral.getValue() instanceof String) { @@ -1866,7 +1920,7 @@ public String visit(Not not, List children) { // without any related row are dropped by the INNER join and multi-valued properties get // any- instead of all-semantics. Wrap instead — the outer operand (A.) is never // null, so NOT (...) is the exact logical negation. - return String.format("%s (%s)", operator, operation); + return negate(operation); } Integer pos = null; Cql2Expression arg = not.getArgs().get(0); @@ -2758,7 +2812,7 @@ public String visit(IsNull isNull, List children) { // contributes no subquery rows at all. "Property has no value" is the negation of // "property has some value" (NOT EXISTS); the outer operand (A.) is never // null, so the negation is exact. - return String.format("NOT (%s)", String.format(mainExpression, "", " IS NOT NULL")); + return negate(String.format(mainExpression, "", " IS NOT NULL")); } // mainExpression is either a literal value or a SELECT expression @@ -3098,25 +3152,32 @@ public String visit(BinaryArrayOperation arrayOperation, List children) if (qualifiedColumn.second().isEmpty()) { if (isEmptyArray(arrayOperation.getArgs().get(notInverse ? 1 : 0))) { - return encodeEmptyArray( - arrayOperation.getArrayOperator(), - notInverse, - mainExpression, - aliases.get(0), - mapping.getMainTable().getSortKey()); + return encodeEmptyArray(arrayOperation.getArrayOperator(), notInverse, mainExpression); } if (notInverse ? arrayOperation.getArrayOperator() == A_CONTAINS : arrayOperation.getArrayOperator() == A_CONTAINEDBY) { + if (elementCount > 1) { + // "contains all values" is the conjunction of one semi-join per value; unlike a single + // subquery with GROUP BY ... HAVING count(distinct ...) = n, each of them lets the + // planner probe the joined table per candidate + Optional> elements = + renderArrayElements(arrayOperation.getArgs().get(notInverse ? 1 : 0)); + if (elements.isPresent()) { + String semiJoin = mainExpression; + return elements.get().stream() + .map(element -> String.format(semiJoin, "", " IN (" + element + ")")) + .collect(Collectors.joining(" AND ", "(", ")")); + } + } // for a single required value the `HAVING count(distinct …) = 1` is tautological (every - // group selected by `IN ()` trivially has one distinct value); dropping it - // keeps the result identical but lets the planner estimate the grouped row count, which - // it cannot do across a count(distinct) HAVING filter + // group selected by `IN ()` trivially has one distinct value), and without a + // HAVING the GROUP BY is redundant in the `IN` semi-join; dropping both keeps the result + // identical but lets the planner turn the subquery into a semi-join that probes the + // joined table per candidate instead of materializing every matching row String arrayQuery = elementCount == 1 - ? String.format( - " IN %1$s GROUP BY %2$s.%3$s", - secondExpression, aliases.get(0), mapping.getMainTable().getSortKey()) + ? String.format(" IN %1$s", secondExpression) : String.format( " IN %1$s GROUP BY %2$s.%3$s HAVING count(distinct %4$s) = %5$s", secondExpression, @@ -3136,10 +3197,9 @@ public String visit(BinaryArrayOperation arrayOperation, List children) elementCount); return String.format(mainExpression, "", arrayQuery); } else if (arrayOperation.getArrayOperator() == A_OVERLAPS) { - String arrayQuery = - String.format( - " IN %1$s GROUP BY %2$s.%3$s", - secondExpression, aliases.get(0), mapping.getMainTable().getSortKey()); + // no GROUP BY: in the `IN` semi-join it would only prevent the planner from probing the + // joined table per candidate + String arrayQuery = String.format(" IN %1$s", secondExpression); return String.format(mainExpression, "", arrayQuery); } else if (notInverse ? arrayOperation.getArrayOperator() == A_CONTAINEDBY @@ -3181,6 +3241,16 @@ public String visit(BinaryArrayOperation arrayOperation, List children) throw new IllegalStateException("unexpected array operator: " + arrayOperation); } + // the SQL literals of the elements of an array literal, if they are available individually + private Optional> renderArrayElements(Operand operand) { + if (operand instanceof ArrayLiteral arrayLiteral + && arrayLiteral.getValue() instanceof List values + && values.stream().allMatch(Scalar.class::isInstance)) { + return Optional.of(values.stream().map(value -> ((Scalar) value).accept(this)).toList()); + } + return Optional.empty(); + } + @Override public String visit(ArrayLiteral arrayLiteral, List children) { if (arrayLiteral.getValue() instanceof String) { @@ -3216,7 +3286,7 @@ public String visit(Not not, List children) { // without any related row are dropped by the INNER join and multi-valued properties get // any- instead of all-semantics. Wrap instead — the outer operand (A.) is never // null, so NOT (...) is the exact logical negation. - return String.format("%s (%s)", operator, operation); + return negate(operation); } Integer pos = null; Cql2Expression arg = not.getArgs().get(0); diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy index 6106dae79..5c0b3aa2b 100644 --- a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlInResultSetSpec.groovy @@ -116,7 +116,7 @@ class FilterEncoderSqlInResultSetSpec extends Specification { def sql = filterEncoder.encode(Not.of(filter), mappings["value_array"]) then: - sql == "NOT (A.id IN (SELECT AA.id FROM externalprovider AA JOIN externalprovider_externalprovidername AB ON (AA.id=AB.externalprovider_fk) WHERE AB.externalprovidername IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id = 'foo') SELECT rs_value FROM _rs_0_s1)))" + sql == "NOT EXISTS (SELECT 1 FROM (SELECT AA.id FROM externalprovider AA JOIN externalprovider_externalprovidername AB ON (AA.id=AB.externalprovider_fk) WHERE AB.externalprovidername IN (WITH _rs_0_s1 AS MATERIALIZED (SELECT A.id AS rs_value FROM externalprovider A WHERE A.id = 'foo') SELECT rs_value FROM _rs_0_s1)) NE_ WHERE NE_.id = A.id)" } def 'negated consumer on the id queryable'() { @@ -138,7 +138,7 @@ class FilterEncoderSqlInResultSetSpec extends Specification { def sql = filterEncoder.encode(filter, mappings["value_array"]) then: - sql == "NOT (A.id IN (SELECT AA.id FROM externalprovider AA JOIN externalprovider_externalprovidername AB ON (AA.id=AB.externalprovider_fk) WHERE AB.externalprovidername IS NOT NULL))" + sql == "NOT EXISTS (SELECT 1 FROM (SELECT AA.id FROM externalprovider AA JOIN externalprovider_externalprovidername AB ON (AA.id=AB.externalprovider_fk) WHERE AB.externalprovidername IS NOT NULL) NE_ WHERE NE_.id = A.id)" } def 'chained result sets nest recursively'() { diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpec.groovy index 36e200368..4c1478a27 100644 --- a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpec.groovy +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpec.groovy @@ -342,7 +342,7 @@ class FilterEncoderSqlSpec extends Specification { def filter = Not.of(CqlFilterExamples.EXAMPLE_16) when: - String expected = "NOT (A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE ST_Intersects(AB.location, ST_GeomFromText('POLYGON((-10.0 -10.0,10.0 -10.0,10.0 10.0,-10.0 -10.0))',4326))))" + String expected = "NOT EXISTS (SELECT 1 FROM (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE ST_Intersects(AB.location, ST_GeomFromText('POLYGON((-10.0 -10.0,10.0 -10.0,10.0 10.0,-10.0 -10.0))',4326))) NE_ WHERE NE_.id = A.id)" String actual = filterEncoder.encode(filter, instanceContainer) @@ -558,7 +558,7 @@ class FilterEncoderSqlSpec extends Specification { def filter = IsNull.of("location") when: - String expected = "NOT (A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL))" + String expected = "NOT EXISTS (SELECT 1 FROM (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL) NE_ WHERE NE_.id = A.id)" String actual = filterEncoder.encode(filter, instanceContainer) @@ -728,7 +728,7 @@ class FilterEncoderSqlSpec extends Specification { def filter = CqlFilterExamples.EXAMPLE_AContains_ValidFor_JOINED_GEOMETRY when: - String expected = "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id','location') GROUP BY AA.id HAVING count(distinct AB.location) = 2)" + String expected = "(A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id')) AND A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('location')))" String actual = filterEncoder.encode(filter, instanceContainer) @@ -738,14 +738,14 @@ class FilterEncoderSqlSpec extends Specification { } - def 'test AContains with a single value omits the tautological HAVING'() { + def 'test AContains with a single value omits the tautological HAVING and GROUP BY'() { given: def instanceContainer = QuerySchemaFixtures.JOINED_GEOMETRY def filter = CqlFilterExamples.EXAMPLE_AContains_SingleValue_ValidFor_JOINED_GEOMETRY when: - String expected = "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id') GROUP BY AA.id)" + String expected = "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id'))" String actual = filterEncoder.encode(filter, instanceContainer) @@ -762,7 +762,7 @@ class FilterEncoderSqlSpec extends Specification { def filter = Not.of(CqlFilterExamples.EXAMPLE_AContains_ValidFor_JOINED_GEOMETRY) when: - String expected = "NOT (A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id','location') GROUP BY AA.id HAVING count(distinct AB.location) = 2))" + String expected = "NOT ((A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id')) AND A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('location'))))" String actual = filterEncoder.encode(filter, instanceContainer) @@ -796,7 +796,7 @@ class FilterEncoderSqlSpec extends Specification { def filter = CqlFilterExamples.EXAMPLE_AOverlaps_ValidFor_JOINED_GEOMETRY when: - String expected = "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id','location') GROUP BY AA.id)" + String expected = "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IN ('id','location'))" String actual = filterEncoder.encode(filter, instanceContainer) @@ -822,6 +822,19 @@ class FilterEncoderSqlSpec extends Specification { } + def 'negating a single semi-join becomes an anti-join, anything else is wrapped'() { + + expect: + FilterEncoderSql.negate(predicate) == expected + + where: + predicate || expected + "A.id IN (SELECT AA.id FROM t AA WHERE AA.x = ')(')" || "NOT EXISTS (SELECT 1 FROM (SELECT AA.id FROM t AA WHERE AA.x = ')(') NE_ WHERE NE_.id = A.id)" + "A.id IN (SELECT AA.id FROM t AA WHERE AA.x = 1) AND A.id IN (SELECT AA.id FROM t AA WHERE AA.y = 2)" || "NOT (A.id IN (SELECT AA.id FROM t AA WHERE AA.x = 1) AND A.id IN (SELECT AA.id FROM t AA WHERE AA.y = 2))" + "(A.id IN (SELECT AA.id FROM t AA WHERE AA.x = 1) AND A.id IN (SELECT AA.id FROM t AA WHERE AA.y = 2))" || "NOT ((A.id IN (SELECT AA.id FROM t AA WHERE AA.x = 1) AND A.id IN (SELECT AA.id FROM t AA WHERE AA.y = 2)))" + "A.x = 1" || "NOT (A.x = 1)" + } + def 'array predicates with an empty array'() { given: @@ -843,8 +856,8 @@ class FilterEncoderSqlSpec extends Specification { where: operator | literalFirst || expected - "A_CONTAINS" | false || "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL GROUP BY AA.id)" - "A_CONTAINEDBY" | true || "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL GROUP BY AA.id)" + "A_CONTAINS" | false || "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL)" + "A_CONTAINEDBY" | true || "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE AB.location IS NOT NULL)" "A_OVERLAPS" | false || "1=0" "A_EQUALS" | false || "1=0" "A_CONTAINEDBY" | false || "1=0"