From a2e7706ae2e0eaa0e6d4bb5a6a189a47203365cb Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 1 Jul 2026 01:02:49 -0700 Subject: [PATCH 1/2] Widen narrow integer operands in PPL +/-/* to prevent overflow PPL integer arithmetic (`+`, `-`, `*` and the named add/subtract/multiply functions) inferred `SMALLINT op SMALLINT -> SMALLINT` (and likewise for TINYINT) because the operators were registered with Calcite's stock `SqlStdOperatorTable.PLUS/MINUS/MULTIPLY`, whose return-type inference (`ReturnTypes.NULLABLE_SUM` / `PRODUCT_NULLABLE`) falls through to `LEAST_RESTRICTIVE` for non-decimal integers. As a result the product/sum of two narrow-integer columns overflowed the inferred type on every backend, just differently: - DataFusion / analytics-engine: silently wraps the i16 result, so e.g. `eval area = ResolutionWidth * ResolutionHeight | where area > 2000000` returned 0 rows instead of the matching row. - Calcite Enumerable engine: throws `ArithmeticException: value out of range`. - v2 legacy engine: `ExprShortValue` narrows via `shortValue()`, wrapping. Fix in the SQL-plugin lowering so all backends are corrected at once: widen the operands (byte/short -> INTEGER, any int/long -> BIGINT) before applying the operator. Casting the operands rather than only relabelling the result type is required, otherwise DataFusion still computes the narrow multiply and wraps before any outer cast. Non-integral operands (float/double/decimal/ datetime/mixed) are left untouched and defer to Calcite's default inference. The string-concat `ADD` variant and the DATETIME-DATETIME `SUBTRACT` variant are unchanged. mvindex's internal array-index arithmetic now uses the raw Calcite PLUS/MINUS operators so array indices stay INTEGER for ITEM/ARRAY_SLICE codegen (the widened result would otherwise be rejected as a long index). Note: this changes user-visible result column types for integer arithmetic (int-operand expressions now report bigint), which is the intended trade-off for overflow-safe, backend-consistent results. Adds CalcitePPLBuiltinFunctionIT coverage for the short->int and int->bigint widening tiers and updates the affected logical-plan / Spark-SQL snapshots. Signed-off-by: Kai Huang --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 4 +- .../function/CollectionUDF/MVAppendCore.java | 45 ++++- .../CollectionUDF/MVAppendFunctionImpl.java | 17 +- .../CollectionUDF/MVIndexFunctionImp.java | 36 ++-- .../expression/function/PPLFuncImpTable.java | 167 +++++++++++++++++- .../remote/CalcitePPLBuiltinFunctionIT.java | 65 ++++++- .../sql/ppl/MathematicalFunctionIT.java | 43 ++++- .../calcite/clickbench/q36.yaml | 4 +- .../calcite/explain_agg_group_merge.yaml | 2 +- .../explain_agg_paginating_having3.yaml | 4 +- .../calcite/explain_agg_with_script.yaml | 2 +- .../explain_agg_with_sum_enhancement.yaml | 2 +- ...complex_sort_expr_no_expr_output_push.yaml | 4 +- ...n_complex_sort_expr_project_then_sort.yaml | 6 +- .../explain_complex_sort_expr_push.yaml | 6 +- ...lex_sort_expr_single_expr_output_push.yaml | 6 +- .../explain_complex_sort_nested_expr.yaml | 6 +- .../explain_complex_sort_then_field_sort.yaml | 4 +- .../calcite/explain_limit_push.yaml | 2 +- ...scalar_uncorrelated_subquery_in_where.yaml | 6 +- .../explain_simple_sort_expr_push.json | 4 +- ...ain_simple_sort_expr_pushdown_for_smj.yaml | 6 +- ...ple_sort_expr_single_expr_output_push.json | 2 +- .../explain_sort_complex_and_simple_expr.yaml | 6 +- .../explain_agg_group_merge.yaml | 2 +- .../explain_agg_with_script.yaml | 2 +- .../explain_agg_with_sum_enhancement.yaml | 2 +- ...complex_sort_expr_no_expr_output_push.yaml | 4 +- ...n_complex_sort_expr_project_then_sort.yaml | 4 +- .../explain_complex_sort_expr_push.yaml | 4 +- ...lex_sort_expr_single_expr_output_push.yaml | 4 +- .../explain_complex_sort_nested_expr.yaml | 4 +- .../explain_complex_sort_then_field_sort.yaml | 4 +- .../explain_filter_script_push.yaml | 2 +- .../explain_limit_push.yaml | 2 +- .../explain_simple_sort_expr_push.json | 6 +- ...ain_simple_sort_expr_pushdown_for_smj.yaml | 6 +- ...ple_sort_expr_single_expr_output_push.json | 4 +- .../explain_sort_complex_and_simple_expr.yaml | 4 +- .../ppl/calcite/CalcitePPLAppendPipeTest.java | 8 +- .../sql/ppl/calcite/CalcitePPLAppendTest.java | 8 +- .../calcite/CalcitePPLArrayFunctionTest.java | 4 +- .../sql/ppl/calcite/CalcitePPLDedupTest.java | 29 +-- .../sql/ppl/calcite/CalcitePPLEvalTest.java | 13 +- .../calcite/CalcitePPLExistsSubqueryTest.java | 6 +- .../calcite/CalcitePPLFieldFormatTest.java | 2 +- 46 files changed, 438 insertions(+), 135 deletions(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 064eff32d7..3320391c0d 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -272,7 +272,7 @@ public void selectExpressionWithoutFrom() { givenQuery("SELECT 1 + 1") .assertPlan( """ - LogicalProject(1 + 1=[+(1, 1)]) + LogicalProject(1 + 1=[+(1:BIGINT, 1:BIGINT)]) LogicalValues(tuples=[[{ 0 }]]) """); } @@ -404,7 +404,7 @@ public void testArithmeticOnAggregates() { givenQuery("SELECT MAX(age) + MIN(age) AS range_sum FROM catalog.employees") .assertPlan( """ - LogicalProject(range_sum=[+($0, $1)]) + LogicalProject(range_sum=[+(CAST($0):BIGINT, CAST($1):BIGINT)]) LogicalAggregate(group=[{}], MAX(age)=[MAX($0)], MIN(age)=[MIN($0)]) LogicalProject(age=[$2]) LogicalTableScan(table=[[catalog, employees]]) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java index f9a67e4d6d..c37189cb20 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java @@ -5,37 +5,72 @@ package org.opensearch.sql.expression.function.CollectionUDF; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.sql.type.SqlTypeName; /** Core logic for `mvappend` command to collect elements from list of args */ public class MVAppendCore { /** * Collect non-null elements from `args`. If an item is a list, it will collect non-null elements - * of the list. See {@ref MVAppendFunctionImplTest} for detailed behavior. + * of the list. Each collected element is coerced to {@code elementType} so a heterogeneously + * boxed input (e.g. an {@code array(int_col)} operand contributing {@code Integer} cells to a + * {@code BIGINT}-typed result) does not throw {@code ClassCastException} when the array is later + * materialized by Avatica's per-type accessor. See {@ref MVAppendFunctionImplTest} for detailed + * behavior. */ + /** Untyped overload — collects without element coercion (used by map-append and unit tests). */ public static List collectElements(Object... args) { + return collectElements((SqlTypeName) null, args); + } + + public static List collectElements(SqlTypeName elementType, Object... args) { List elements = new ArrayList<>(); for (Object arg : args) { if (arg == null) { continue; } else if (arg instanceof List) { - addListElements((List) arg, elements); + addListElements((List) arg, elements, elementType); } else { - elements.add(arg); + elements.add(coerce(arg, elementType)); } } return elements.isEmpty() ? null : elements; } - private static void addListElements(List list, List elements) { + private static void addListElements( + List list, List elements, SqlTypeName elementType) { for (Object item : list) { if (item != null) { - elements.add(item); + elements.add(coerce(item, elementType)); } } } + + /** + * Align a boxed numeric element to the array's target element type. Only numeric widenings that + * arise from operand widening (e.g. INTEGER cells into a BIGINT array) are handled; non-numeric + * or null-typed targets pass the value through unchanged so mixed / ANY-typed arrays keep their + * existing {@code Object[]} runtime semantics. + */ + private static Object coerce(Object value, SqlTypeName elementType) { + if (elementType == null || !(value instanceof Number)) { + return value; + } + Number num = (Number) value; + return switch (elementType) { + case TINYINT -> num.byteValue(); + case SMALLINT -> num.shortValue(); + case INTEGER -> num.intValue(); + case BIGINT -> num.longValue(); + case FLOAT, REAL -> num.floatValue(); + case DOUBLE -> num.doubleValue(); + case DECIMAL -> num instanceof BigDecimal ? num : BigDecimal.valueOf(num.doubleValue()); + default -> value; + }; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java index bafbeb09c4..dc3402e464 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -127,12 +127,27 @@ public Expression implement( coerced.add(EnumUtils.convert(op, elementClass)); } } + // Pass the target element SqlTypeName so the runtime can align the elements flattened out of + // ARRAY operands. Calcite does not element-wise cast inside an array operand, so + // `mvappend(array(int_col), int_col * 2)` — where operand widening makes the result element + // type BIGINT while `array(int_col)` still yields Integer cells — would otherwise throw + // `Integer cannot be cast to Long` when the array is materialized. Scalars are already + // pre-cast above; the runtime coercion is a no-op for them. + SqlTypeName targetType = elementType == null ? null : elementType.getSqlTypeName(); return Expressions.call( - Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Types.lookupMethod( + MVAppendFunctionImpl.class, "mvappendTyped", SqlTypeName.class, Object[].class), + Expressions.constant(targetType, SqlTypeName.class), Expressions.newArrayInit(Object.class, coerced)); } } + /** Codegen entry point: coerces flattened elements to {@code elementType}. */ + public static Object mvappendTyped(SqlTypeName elementType, Object... args) { + return MVAppendCore.collectElements(elementType, args); + } + + /** Untyped entry point used by unit tests; performs no element coercion. */ public static Object mvappend(Object... args) { return MVAppendCore.collectElements(args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java index 24e4b48963..a244e290ae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java @@ -5,17 +5,16 @@ package org.opensearch.sql.expression.function.CollectionUDF; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_LENGTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_SLICE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; import java.math.BigDecimal; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.sql.expression.function.PPLFuncImpTable; /** @@ -37,6 +36,10 @@ *
  • Range access uses Calcite's ARRAY_SLICE operator (0-based indexing with length parameter) *
  • Index conversion handles the difference between PPL's 0-based indexing and Calcite's * conventions + *
  • Index arithmetic uses Calcite's raw {@code PLUS}/{@code MINUS} rather than PPL's widening + * {@code +}/{@code -} operators: array indices are int-domain and {@code ITEM}/{@code + * ARRAY_SLICE} require an INTEGER index, so the deliberate integer-overflow widening applied + * to user arithmetic must not leak into these internal, bounded computations. * */ public class MVIndexFunctionImp implements PPLFuncImpTable.FunctionImp { @@ -59,6 +62,16 @@ public RexNode resolve(RexBuilder builder, RexNode... args) { } } + /** Non-widening integer addition for internal, int-domain array-index math. */ + private static RexNode add(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.PLUS, left, right); + } + + /** Non-widening integer subtraction for internal, int-domain array-index math. */ + private static RexNode subtract(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.MINUS, left, right); + } + /** * Resolves single element access: mvindex(array, index) * @@ -72,11 +85,9 @@ private RexNode resolveSingleElement( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode sumArrayLenStart = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); - RexNode negativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, sumArrayLenStart, one); - RexNode positiveCase = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, startIdx, one); + RexNode sumArrayLenStart = add(builder, arrayLen, startIdx); + RexNode negativeCase = add(builder, sumArrayLenStart, one); + RexNode positiveCase = add(builder, startIdx, one); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isNegative, negativeCase, positiveCase); @@ -97,21 +108,18 @@ private RexNode resolveRange( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isStartNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode startNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); + RexNode startNegativeCase = add(builder, arrayLen, startIdx); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isStartNegative, startNegativeCase, startIdx); RexNode isEndNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, endIdx, zero); - RexNode endNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, endIdx); + RexNode endNegativeCase = add(builder, arrayLen, endIdx); RexNode normalizedEnd = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isEndNegative, endNegativeCase, endIdx); // Calculate length: (normalizedEnd - normalizedStart) + 1 - RexNode diff = - PPLFuncImpTable.INSTANCE.resolve(builder, SUBTRACT, normalizedEnd, normalizedStart); - RexNode length = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, diff, one); + RexNode diff = subtract(builder, normalizedEnd, normalizedStart); + RexNode length = add(builder, diff, one); // Call ARRAY_SLICE(array, normalizedStart, length) return PPLFuncImpTable.INSTANCE.resolve(builder, ARRAY_SLICE, array, normalizedStart, length); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 5750bf6cae..b9350d18d8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -286,9 +286,12 @@ import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlLibraryOperators; @@ -572,6 +575,10 @@ public RexNode resolve( // return type of the lambda function. compulsoryCast(builder, functionName, args); + // Align integer operand widths when a comparison has a scalar subquery on one side; see + // coerceNumericComparisonOperands for why this is needed for correct decorrelated join keys. + args = coerceNumericComparisonOperands(builder, functionName, args); + List argTypes = Arrays.stream(args).map(RexNode::getType).toList(); try { for (Map.Entry implement : implementList) { @@ -606,6 +613,62 @@ public RexNode resolve( functionName, allowedSignatures, PlanUtils.getActualSignature(argTypes))); } + /** + * Widens the operands of a binary comparison against a scalar subquery to a common integer type + * when the two operands are integers of differing width (e.g. INT vs BIGINT, SMALLINT vs INT). + * Returns the original array unchanged for every other case, so only the ambiguous + * integer-width-against-a-subquery case is touched. + * + *

    Calcite's comparison operators accept mixed integer widths by family, so {@code makeCall(=, + * int, bigint)} type-checks and evaluates correctly as an ordinary scalar predicate (against a + * column or literal), which is why those comparisons are left alone. But when one side is a + * scalar subquery, Calcite's decorrelator turns the comparison into a join whose keys keep their + * original widths; the generated key extractors then box the two sides as different Java types + * (e.g. {@code Integer} vs {@code Long}) that never compare equal, so the join silently drops + * every row. Casting both sides to their least-restrictive common integer type keeps the boolean + * result identical while making the derived join keys share a single type. This surfaces once + * integer arithmetic widens (e.g. {@code min(salary) + 1000} is BIGINT) so a subquery result is + * compared against a narrower column. + */ + private static RexNode[] coerceNumericComparisonOperands( + RexBuilder builder, BuiltinFunctionName functionName, RexNode... args) { + if (args.length != 2 || !BuiltinFunctionName.COMPARATORS.contains(functionName)) { + return args; + } + if (!containsSubQuery(args[0]) && !containsSubQuery(args[1])) { + return args; + } + RelDataType leftType = args[0].getType(); + RelDataType rightType = args[1].getType(); + if (!SqlTypeName.INT_TYPES.contains(leftType.getSqlTypeName()) + || !SqlTypeName.INT_TYPES.contains(rightType.getSqlTypeName()) + || leftType.getSqlTypeName() == rightType.getSqlTypeName()) { + return args; + } + RelDataType commonType = + builder.getTypeFactory().leastRestrictive(List.of(leftType, rightType)); + if (commonType == null) { + return args; + } + return new RexNode[] { + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, leftType.isNullable()), args[0]), + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, rightType.isNullable()), args[1]) + }; + } + + /** Whether {@code node} is, or transitively contains, a {@link RexSubQuery}. */ + private static boolean containsSubQuery(RexNode node) { + if (node instanceof RexSubQuery) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(PPLFuncImpTable::containsSubQuery); + } + return false; + } + /** * Ad-hoc coercion for some functions that require specific casting of arguments. Now it only * applies to the REDUCE function. @@ -727,6 +790,89 @@ protected void registerDivideFunction(BuiltinFunctionName functionName) { PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); } + /** + * Register an arithmetic operator ({@code +}, {@code -}, {@code *}) that widens narrow integer + * operands before applying the operation, deriving the type checker from the operator. + * + *

    Calcite infers {@code SMALLINT op SMALLINT -> SMALLINT} (via {@code ReturnTypes.PLUS} / + * {@code PRODUCT_NULLABLE}, which fall through to {@code LEAST_RESTRICTIVE}), so the product of + * two {@code short} columns overflows: DataFusion silently wraps the {@code i16} result while + * the Calcite Enumerable engine throws {@code ArithmeticException: value out of range}. Casting + * the operands up (byte/short -> int, int -> long) makes the arithmetic compute at a width that + * cannot overflow for the widened tier, matching v2 arithmetic intent across all backends. + */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator) { + PPLTypeChecker typeChecker = + wrapSqlOperandTypeChecker(operator.getOperandTypeChecker(), operator.getName(), false); + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + /** Same as above but with an explicit {@link PPLTypeChecker}. */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator, PPLTypeChecker typeChecker) { + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + private static FunctionImp wideningIntegerArithmetic(SqlOperator operator) { + // +, -, * are strictly binary; FunctionImp2 documents and enforces the two-operand contract. + return (FunctionImp2) + (builder, left, right) -> + builder.makeCall(operator, widenIntegerOperands(builder, left, right)); + } + + private static RexNode[] widenIntegerOperands(RexBuilder builder, RexNode... args) { + SqlTypeName promoted = promotedIntegerType(args); + if (promoted == null) { + return args; + } + // Skip widening arithmetic inside higher-order-function lambda bodies (reduce/mvmap/...). + // Those operands reference lambda parameters whose types are placeholders resolved later by + // the HOF's own return-type inference (see LambdaUtils.inferReturnTypeFromLambda); wrapping a + // RexLambdaRef in a CAST breaks that index-based resolution. They also execute JVM-side via + // linq4j (operands already promote to int), so they never hit the backend wrap path. + for (RexNode arg : args) { + if (referencesLambdaParameter(arg)) { + return args; + } + } + RexNode[] widened = new RexNode[args.length]; + for (int i = 0; i < args.length; i++) { + RelDataType target = TYPE_FACTORY.createSqlType(promoted, args[i].getType().isNullable()); + widened[i] = builder.makeCast(target, args[i]); + } + return widened; + } + + private static boolean referencesLambdaParameter(RexNode node) { + if (node instanceof RexLambdaRef) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(AbstractBuilder::referencesLambdaParameter); + } + return false; + } + + /** + * Target integer type that all operands should widen to, or {@code null} to leave the call + * untouched (any non-integral operand, e.g. FLOAT/DOUBLE/DECIMAL/DATETIME, defers to Calcite's + * default inference). byte/short -> INTEGER; anything involving int/long -> BIGINT. + */ + private static SqlTypeName promotedIntegerType(RexNode... args) { + boolean needsLong = false; + for (RexNode arg : args) { + switch (arg.getType().getSqlTypeName()) { + case TINYINT, SMALLINT -> {} + case INTEGER, BIGINT -> needsLong = true; + default -> { + return null; + } + } + } + return needsLong ? SqlTypeName.BIGINT : SqlTypeName.INTEGER; + } + void populate() { // register operators for comparison registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); @@ -741,23 +887,25 @@ void populate() { registerOperator(OR, SqlStdOperatorTable.OR); registerOperator(NOT, SqlStdOperatorTable.NOT); - // Register ADDFUNCTION for numeric addition only - registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator( + // Register ADDFUNCTION for numeric addition only. Widen narrow integer operands so the sum + // cannot overflow the (mis-)inferred SMALLINT/TINYINT result type; see + // registerWideningIntegerOperator. + registerWideningIntegerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); + registerWideningIntegerOperator( SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - registerOperator( + registerWideningIntegerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - // Add DATETIME-DATETIME variant for timestamp binning support + // Add DATETIME-DATETIME variant for timestamp binning support (no integer widening) registerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); - registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); - registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); registerOperator(ASCII, SqlStdOperatorTable.ASCII); registerOperator(LENGTH, SqlStdOperatorTable.CHAR_LENGTH); @@ -1125,8 +1273,9 @@ void populate() { SqlStdOperatorTable.CONCAT, PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); // Register ADD (+ symbol) for numeric addition - // Replace type checker since PLUS also supports binary addition - registerOperator( + // Replace type checker since PLUS also supports binary addition. Widen narrow integer + // operands so the sum cannot overflow the (mis-)inferred SMALLINT/TINYINT result type. + registerWideningIntegerOperator( ADD, SqlStdOperatorTable.PLUS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java index cbd94683fd..82debdb031 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java @@ -267,7 +267,8 @@ public void testModFloatAndNegative() throws IOException { "source=%s | eval f = mod(float_number, 2), n = -1 * short_number %% 2, nd = -1 *" + " double_number %% 2 | fields f, n, nd", TEST_INDEX_DATATYPE_NUMERIC)); - verifySchema(actual, schema("f", "float"), schema("n", "int"), schema("nd", "double")); + // -1 * short_number widens the integer operands to bigint (overflow-safe widening). + verifySchema(actual, schema("f", "float"), schema("n", "bigint"), schema("nd", "double")); verifyDataRows(actual, closeTo(0.2, -1, -1.1)); } @@ -414,4 +415,66 @@ public void testDivideShouldReturnNull() throws IOException { schema("r6", "double")); verifyDataRows(actual, rows(null, null, null, null, null)); } + + /** + * Integer arithmetic must widen narrow operands so the result cannot overflow the (mis-)inferred + * SMALLINT/TINYINT result type. Historically {@code short * short} stayed SHORT, silently + * wrapping on the analytics-engine backend and throwing "value out of range" on the Calcite path. + * byte/short now widen to INT and int/long widen to BIGINT for {@code +}, {@code -}, {@code *}. + */ + @Test + public void testIntegerArithmeticWidensResultType() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval ss = short_number * short_number, bb = byte_number *" + + " byte_number, ii = integer_number * integer_number, sp = short_number +" + + " short_number, sm = short_number - byte_number | fields ss, bb, ii, sp, sm", + TEST_INDEX_DATATYPE_NUMERIC)); + // short/byte products widen to int; int product widens to bigint; +/- widen likewise. + verifySchema( + actual, + schema("ss", "int"), + schema("bb", "int"), + schema("ii", "bigint"), + schema("sp", "int"), + schema("sm", "int")); + // short_number=3, byte_number=4, integer_number=2. + verifyDataRows(actual, rows(9, 16, 4, 6, -1)); + } + + /** + * Regression for the reported overflow: {@code short * } whose product exceeds the 16-bit + * SHORT range (32767) must produce the correct widened value rather than a wrapped one. + */ + @Test + public void testShortArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval area = short_number * 20000 | where area > 32767 | fields area", + TEST_INDEX_DATATYPE_NUMERIC)); + // 3 * 20000 = 60000, which overflows SHORT (max 32767) but is exact once widened. The INTEGER + // literal 20000 promotes the product to BIGINT (any int operand -> long). + verifySchema(actual, schema("area", "bigint")); + verifyDataRows(actual, rows(60000)); + } + + /** + * The INTEGER->BIGINT tier: {@code integer_number * } whose product exceeds + * the 32-bit INT range must widen to BIGINT and stay exact rather than wrapping i32. + */ + @Test + public void testIntegerArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval big = integer_number * 2000000000 | where big > 2147483647 |" + + " fields big", + TEST_INDEX_DATATYPE_NUMERIC)); + // integer_number=2; 2 * 2,000,000,000 = 4,000,000,000 overflows INT (max 2,147,483,647) but is + // exact once widened to BIGINT. + verifySchema(actual, schema("big", "bigint")); + verifyDataRows(actual, rows(4000000000L)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index 42f6901027..0a81ed92ad 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -571,7 +571,12 @@ public void testEvalSumMultipleIntegers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + // Calcite widens integer arithmetic operands to avoid overflow, so the result is bigint. + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(6), rows(6), rows(6), rows(6), rows(6)); } @@ -590,7 +595,11 @@ public void testEvalSumWithFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, 10) | fields f | head 7", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(42), rows(46), rows(38), rows(43), rows(46), rows(49), rows(44)); } @@ -600,7 +609,11 @@ public void testEvalSumMultipleNumericArguments() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3, 4, 5) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(15), rows(15), rows(15), rows(15), rows(15)); } @@ -681,7 +694,11 @@ public void testEvalSumAndAvgComparison() throws IOException { "source=%s | eval sum_val = sum(10, 20, 30), avg_val = avg(10, 20, 30) | fields" + " sum_val, avg_val | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + if (isCalciteEnabled()) { + verifySchema(result, schema("sum_val", null, "bigint"), schema("avg_val", null, "double")); + } else { + verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + } verifyDataRows( result, rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0)); } @@ -693,7 +710,11 @@ public void testEvalSumInWhereClause() throws IOException { String.format( "source=%s | where sum(age, 10) > 40 | eval f = sum(age, 10) | fields f | head 6", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // Should return rows where age + 10 > 40, so age > 30 verifyDataRows(result, rows(42), rows(46), rows(43), rows(46), rows(49), rows(44)); } @@ -740,7 +761,11 @@ public void testEvalSumWithMultipleFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, age, 10) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // sum(age, age, 10) = age + age + 10 = 2*age + 10 verifyDataRows(result, rows(74), rows(82), rows(66), rows(76), rows(82)); } @@ -762,7 +787,11 @@ public void testEvalSumWithNegativeNumbers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(-5, 10, -3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(2), rows(2), rows(2), rows(2), rows(2)); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml index 5f1b457eaa..d964a1422a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml @@ -6,8 +6,8 @@ calcite: LogicalAggregate(group=[{0, 1, 2, 3}], c=[COUNT()]) LogicalProject(ClientIP=[$76], ClientIP - 1=[$111], ClientIP - 2=[$112], ClientIP - 3=[$113]) LogicalFilter(condition=[AND(IS NOT NULL($76), IS NOT NULL($111), IS NOT NULL($112), IS NOT NULL($113))]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-($76, 1)], ClientIP - 2=[-($76, 2)], ClientIP - 3=[-($76, 3)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-(CAST($76):BIGINT, 1)], ClientIP - 2=[-(CAST($76):BIGINT, 2)], ClientIP - 3=[-(CAST($76):BIGINT, 3)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[-($t0, $t2)], expr#4=[2], expr#5=[-($t0, $t4)], expr#6=[3], expr#7=[-($t0, $t6)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t3], ClientIP - 2=[$t5], ClientIP - 3=[$t7]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[1:BIGINT], expr#4=[-($t2, $t3)], expr#5=[2:BIGINT], expr#6=[-($t2, $t5)], expr#7=[3:BIGINT], expr#8=[-($t2, $t7)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t4], ClientIP - 2=[$t6], ClientIP - 3=[$t8]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"ClientIP":{"terms":{"field":"ClientIP","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml index acd95f0ec6..841e131df2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(age1=[*($8, 10)], age2=[+($8, 10)], age3=[10], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml index e7589d8109..4f57bbf4cd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -8,5 +8,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[>($t6, $t5)], expr#9=[OR($t7, $t8)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t9]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1:BIGINT], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[1], expr#9=[>($t6, $t8)], expr#10=[OR($t7, $t9)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index c543431c51..bc65d5c4c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18..1d664d5cd4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index 4a9a143cba..c42ecef213 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index e8e9ac1f4f..ffd55ffb1f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -2,8 +2,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index 5aa37ee329..64e868f9f8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index d80ebc5735..8f60e23e49 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index 31efd3c688..7ad040f826 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[-($t13, $t10)], proj#0..14=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+($10, $7), $10) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCsHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[-($t14, $t13)], proj#0..12=[{exprs}], age2=[$t14], age3=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEzHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml index 3fd07a9682..5aef351b78 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml @@ -5,10 +5,10 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[ABS($t7)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml index 690c3ce24e..27110780ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], ageMinus=[$t2]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30:BIGINT], expr#2=[-($t0, $t1)], ageMinus=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml index 042787a458..e98a407d37 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(CAST($2):BIGINT, +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,6 +12,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(CAST($1):BIGINT, +($2, 999))], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json index 066b678a96..2c5f37e681 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[2:BIGINT], expr#3=[+($t1, $t2)], age=[$t0], age2=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml index d0e66b1d14..e427478196 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -12,14 +12,14 @@ calcite: EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[1:BIGINT], expr#15=[+($t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", "missing" : "_last" } }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20:BIGINT], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ "balance" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json index 0a36ec4648..7494f2453c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1:BIGINT], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index 08e75fbbde..f5834404f1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[1], expr#15=[+($t7, $t14)], proj#0..13=[{exprs}], balance2=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[1:BIGINT], expr#16=[+($t7, $t15)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml index a694c63b2c..28bee6245d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) EnumerableAggregate(group=[{8}], count()=[COUNT()]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 285d0b221e..1db12fc013 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index bf861c337b..655e16839e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml index 5c479c6867..575e40c0f7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], age=[$t0]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml index a95c277b40..2eb653bfbb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml index ef4ea5fc43..723ca37b3e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml index 7df4a4d7f4..87baf3c839 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml index 711608264e..1a9c720ee8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$14], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[-($t19, $t10)], proj#0..12=[{exprs}], age2=[$t19], age3=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[-($t20, $t19)], proj#0..12=[{exprs}], age2=[$t20], age3=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml index 362f847ae6..156ec314c5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml @@ -5,11 +5,11 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..13=[{inputs}], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) EnumerableSort(sort0=[$10], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], proj#0..12=[{exprs}], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], proj#0..12=[{exprs}], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml index 90492abbaf..a1b5e3d3ed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml @@ -6,5 +6,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2:BIGINT], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml index fb3daa0676..2ee55737ac 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[-($t8, $t17)], ageMinus=[$t18]) EnumerableLimit(fetch=[5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json index adb4cb6244..b48850334a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[2], expr#20=[+($t10, $t19)], age=[$t10], age2=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[2:BIGINT], expr#21=[+($t19, $t20)], age=[$t10], age2=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml index 8897a1023c..30be0be9ca 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -13,9 +13,9 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[1:BIGINT], expr#21=[+($t19, $t20)], proj#0..12=[{exprs}], $f13=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20:BIGINT], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) EnumerableLimit(fetch=[50000]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json index 67cf82580e..2fffb64bef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:BIGINT], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml index 873a778f97..df4e941dfb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[1], expr#21=[+($t7, $t20)], proj#0..12=[{exprs}], age2=[$t19], balance2=[$t21]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[1:BIGINT], expr#22=[+($t7, $t21)], proj#0..12=[{exprs}], age2=[$t20], balance2=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java index 56ed409b4d..dc476b77cf 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java @@ -44,18 +44,18 @@ public void testAppendPipeWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java index a163af186d..027062485c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java @@ -211,18 +211,18 @@ public void testAppendWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java index 1d6792b099..fbbee80ff7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java @@ -656,8 +656,8 @@ public void testMvmapWithNestedFunction() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr` * 10)" - + " `result`\n" + "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr`" + + " * 10) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java index 5f32c1b85b..3c13297f8f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java @@ -198,7 +198,7 @@ public void testDedupExpr() { + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)])\n" + " LogicalFilter(condition=[IS NOT NULL($4)])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], DEPTNO=[$7]," - + " NEW_DEPTNO=[+($7, 1)])\n" + + " NEW_DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -216,7 +216,8 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -229,10 +230,10 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } @@ -277,10 +278,10 @@ public void testSortThenDedupWithEval() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); // After fix, the sort order (NEW_DEPTNO ASC) must be preserved through dedup. @@ -304,7 +305,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -317,7 +319,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -330,10 +333,10 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java index 9b37ab5b40..3cd9e417a5 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java @@ -104,7 +104,7 @@ public void testEvalSum() { String ppl = "source=EMP | eval total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -131,7 +131,8 @@ public void testEvalAvg() { String ppl = "source=EMP | eval average = avg(10, 20, 30) | fields EMPNO, average"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20, 30)), 3.0E0:DOUBLE)])\n" + "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20:BIGINT, 30:BIGINT))," + + " 3.0E0:DOUBLE)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -210,7 +211,7 @@ public void testEvalUsingExistingFields() { "LogicalProject(EMPNO=[$0], EMPNO_PLUS=[$8])\n" + " LogicalSort(sort0=[$8], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+($0, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+(CAST($0):BIGINT NOT NULL, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -223,7 +224,7 @@ public void testEvalUsingExistingFields() { String expectedSparkSql = "SELECT `EMPNO`, `EMPNO_PLUS`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " `EMPNO` + 1 `EMPNO_PLUS`\n" + + " CAST(`EMPNO` AS BIGINT) + 1 `EMPNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY 9 DESC\n" + "LIMIT 3) `t0`"; @@ -239,7 +240,7 @@ public void testEvalOverridingExistingFields() { "LogicalProject(EMPNO=[$0], SAL=[$7])\n" + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " COMM=[$6], DEPTNO=[$7], SAL=[+($7, 10000)])\n" + + " COMM=[$6], DEPTNO=[$7], SAL=[+(CAST($7):BIGINT, 10000)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -248,7 +249,7 @@ public void testEvalOverridingExistingFields() { String expectedSparkSql = "" - + "SELECT `EMPNO`, `DEPTNO` + 10000 `SAL`\n" + + "SELECT `EMPNO`, CAST(`DEPTNO` AS BIGINT) + 10000 `SAL`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY `EMPNO` DESC\n" + "LIMIT 3"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java index 76c280db92..84b509d16e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java @@ -490,14 +490,14 @@ public void testCorrelatedExistsSubqueryWithOverridingFields() { + " LogicalTableScan(table=[[scott, DEPT]])\n" + "})], variablesSet=[[$cor0]])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[+($7, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT *\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO` + 1" - + " `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " CAST(`DEPTNO` AS BIGINT) + 1 `DEPTNO`\n" + "FROM `scott`.`EMP`) `t`\n" + "WHERE EXISTS (SELECT *\n" + "FROM `scott`.`DEPT`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java index 5bef9c397e..85dfb3eb65 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -177,7 +177,7 @@ public void testFieldFormatSum() { "source=EMP |sort EMPNO | head 3| fieldformat total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; From 447b9583b80a548cf46d706741d15ebb71533be0 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 1 Jul 2026 15:20:56 -0700 Subject: [PATCH 2/2] Detect long (BIGINT) arithmetic overflow instead of silently wrapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PPL/SQL long arithmetic (+, -, *) in eval/SELECT expressions silently wrapped on overflow in the Calcite engine (e.g. long_field * 999...9 wrapped to a negative value with HTTP 200). SqlStdOperatorTable.PLUS/MINUS/MULTIPLY generate plain Java +/-/* in the Enumerable code path, which wrap. Rewrite long +/-/* to their overflow-checked variants (CHECKED_PLUS / CHECKED_MINUS / CHECKED_MULTIPLY), which generate Math.addExact etc. and throw ArithmeticException on overflow. The exception is caught in QueryService and surfaced as a 4xx client error instead of wrapping or falling back to V2. Scoped to BIGINT operands only: narrower integer arithmetic (byte/short/int) is widened to a type that cannot overflow before this rewrite runs (PPLFuncImpTable promotes byte/short to INT and any int/long to BIGINT for +/-/*), so long — which has no wider integer type — is the sole remaining overflow case on the Calcite engine. Float/double/decimal follow IEEE 754 / decimal semantics and have no CHECKED_* runtime, so they are left untouched. The rewrite runs after the analytics-engine fork, so only the Calcite path is affected (the DataFusion backend handles its own overflow). The rewrite runs before pushdown so both coordinator-executed and pushed-down (script) arithmetic are checked; PPLAggregateConvertRule, OpenSearchRelOptUtil, and ExtendedRelJson recognize the CHECKED_* kinds so aggregate/sort pushdown and script serialization keep working. Adds a REST yaml test (issues/5164.yml) and updates the affected explain fixtures. Resolves #5164 Signed-off-by: Kai Huang --- .../plan/rule/PPLAggregateConvertRule.java | 11 +- .../opensearch/sql/executor/QueryService.java | 108 ++++++++++- docs/user/ppl/functions/expressions.md | 4 + .../calcite/explain_agg_counts_by6.yaml | 2 +- ...complex_sort_expr_no_expr_output_push.yaml | 2 +- ...n_complex_sort_expr_project_then_sort.yaml | 2 +- .../explain_complex_sort_expr_push.yaml | 2 +- ...lex_sort_expr_single_expr_output_push.yaml | 2 +- .../explain_complex_sort_nested_expr.yaml | 2 +- .../calcite/explain_count_agg_push7.yaml | 2 +- .../calcite/explain_filter_script_push.yaml | 2 +- .../calcite/explain_skip_script_encoding.yaml | 2 +- .../explain_sort_complex_and_simple_expr.yaml | 2 +- .../rest-api-spec/test/issues/5164.yml | 170 ++++++++++++++++++ .../storage/serde/ExtendedRelJson.java | 1 + .../opensearch/util/OpenSearchRelOptUtil.java | 8 +- 16 files changed, 304 insertions(+), 18 deletions(-) create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java index 81e2c295de..7c6338fe87 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLAggregateConvertRule.java @@ -135,7 +135,9 @@ public void apply(RelOptRuleCall call, LogicalAggregate aggregate, LogicalProjec final Function> literalConverterProvider; RexCall rexCall = (RexCall) project.getProjects().get(aggCall.getArgList().getFirst()); if (rexCall.getOperator().kind == SqlKind.PLUS - || rexCall.getOperator().kind == SqlKind.MINUS) { + || rexCall.getOperator().kind == SqlKind.MINUS + || rexCall.getOperator().kind == SqlKind.CHECKED_PLUS + || rexCall.getOperator().kind == SqlKind.CHECKED_MINUS) { AggregateCall countCall = AggregateCall.create( aggCall.getParserPosition(), @@ -280,7 +282,12 @@ private static boolean isCallWithLiteral(RexNode node) { List CONVERTABLE_FUNCTIONS = List.of( - SqlKind.PLUS, SqlKind.MINUS, SqlKind.TIMES + SqlKind.PLUS, + SqlKind.MINUS, + SqlKind.TIMES, + SqlKind.CHECKED_PLUS, + SqlKind.CHECKED_MINUS, + SqlKind.CHECKED_TIMES // Don't support division because of the issue of integer division // e.g. (2000 / 3) * 3 = 1998 while 2000 * 3 / 3 = 2000 // SqlKind.DIVIDE diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index ddb4338bc8..7112304d5a 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -16,10 +16,16 @@ import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; @@ -174,7 +180,9 @@ public void executeWithCalcite( RelNode calcitePlan = StageErrorHandler.executeStage( QueryProcessingStage.PLAN_CONVERSION, - () -> convertToCalcitePlan(relNode, context), + () -> + withCheckedArithmetic( + convertToCalcitePlan(relNode, context), context), "while converting the query to an executable plan"); analyzeMetric.set(System.nanoTime() - analyzeStart); @@ -187,7 +195,15 @@ public void executeWithCalcite( }, QueryService.class); } catch (Throwable t) { - if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { + ArithmeticException overflow = findArithmeticOverflow(t); + if (overflow != null) { + // Checked arithmetic detected integer/long overflow. Surface as a client error + // instead of wrapping (silently) or falling back to the V2 engine. + propagateCalciteError( + new NonFallbackCalciteException( + "Arithmetic overflow: " + overflow.getMessage(), overflow), + listener); + } else if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { log.warn("Fallback to V2 query engine since got exception", t); executeWithLegacy(plan, queryType, listener, Optional.of(t)); } else { @@ -227,7 +243,8 @@ public void explainWithCalcite( context.run( () -> { RelNode relNode = analyze(plan, context); - RelNode calcitePlan = convertToCalcitePlan(relNode, context); + RelNode calcitePlan = + withCheckedArithmetic(convertToCalcitePlan(relNode, context), context); if (format != null) { executionEngine.explain(calcitePlan, mode, format, context, listener); } else { @@ -383,6 +400,91 @@ private boolean isCalciteEnabled(Settings settings) { } } + /** + * Rewrite {@code +}/{@code -}/{@code *} to their overflow-checked variants ({@code CHECKED_PLUS} + * / {@code CHECKED_MINUS} / {@code CHECKED_MULTIPLY}) so integer and long arithmetic overflow + * throws {@link ArithmeticException} (via {@code Math.addExact} etc.) instead of silently + * wrapping. Applied before pushdown so both coordinator-executed and pushed-down (script) + * arithmetic are checked. Floating-point arithmetic is unchanged (IEEE 754). + * + *

    This does the same rewrite as Calcite's {@code ConvertToChecked} but preserves each call's + * originally inferred type (via {@code makeCall(type, op, operands)}) and touches only the three + * arithmetic operators, so it does not re-derive the types of unrelated calls (e.g. {@code + * CEIL}/{@code DIVIDE}) the way {@code ConvertToChecked} does. + */ + private static RelNode withCheckedArithmetic(RelNode calcitePlan, CalcitePlanContext context) { + RexShuttle checkedShuttle = + new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + RexNode visited = super.visitCall(call); + if (!(visited instanceof RexCall rexCall)) { + return visited; + } + SqlOperator checked = + switch (rexCall.getOperator().getKind()) { + case PLUS -> SqlStdOperatorTable.CHECKED_PLUS; + case MINUS -> SqlStdOperatorTable.CHECKED_MINUS; + case TIMES -> SqlStdOperatorTable.CHECKED_MULTIPLY; + default -> null; + }; + // Only integer/long arithmetic can overflow silently and has a checked + // implementation (Math.addExact etc.). Float/double/decimal have no checked variant + // (SqlFunctions.checkedMultiply(double,double) does not exist) and follow IEEE 754, so + // leave them untouched. + if (checked == null || !isCheckableIntegerArithmetic(rexCall)) { + return visited; + } + return context.rexBuilder.makeCall(rexCall.getType(), checked, rexCall.getOperands()); + } + }; + return calcitePlan.accept( + new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode other) { + RelNode visited = super.visitChildren(other); + return visited.accept(checkedShuttle); + } + }); + } + + /** + * Checked arithmetic is applied to BIGINT ({@code long}) operands only. Narrower integer + * arithmetic (byte/short/int) is already widened to a type that cannot overflow before this + * rewrite runs — {@code PPLFuncImpTable} promotes byte/short to INT and any int/long operand to + * BIGINT for {@code +}/{@code -}/{@code *} — so the sole remaining overflow case that reaches the + * Calcite engine is {@code long} arithmetic, which has no wider integer type to widen into. + * Float/double/decimal follow IEEE 754 (or decimal semantics) and have no {@code CHECKED_*} + * runtime (e.g. {@code SqlFunctions.checkedMultiply(double, double)} does not exist), so they are + * left untouched. Require both the result and every operand to be BIGINT. + */ + private static boolean isCheckableIntegerArithmetic(RexCall call) { + if (!isCheckableLongType(call.getType())) { + return false; + } + return call.getOperands().stream().allMatch(op -> isCheckableLongType(op.getType())); + } + + private static boolean isCheckableLongType(org.apache.calcite.rel.type.RelDataType type) { + return type.getSqlTypeName() == org.apache.calcite.sql.type.SqlTypeName.BIGINT; + } + + /** + * Walk the cause chain to find an {@link ArithmeticException} raised by checked arithmetic. Row- + * level overflow surfaces wrapped (SQLException -> RuntimeException -> ErrorReport), so a + * top-level {@code catch (ArithmeticException)} is insufficient. + */ + private static ArithmeticException findArithmeticOverflow(@Nullable Throwable t) { + for (Throwable cause = t; + cause != null && cause != cause.getCause(); + cause = cause.getCause()) { + if (cause instanceof ArithmeticException arithmeticException) { + return arithmeticException; + } + } + return null; + } + // TODO https://github.com/opensearch-project/sql/issues/3457 // Calcite is not available for SQL query now. Maybe release in 3.1.0? private boolean shouldUseCalcite(QueryType queryType) { diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index e42d867705..427a0334b5 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -11,6 +11,10 @@ Arithmetic expressions are formed by combining numeric literals and binary arith 4. `/`: Division. When [`plugins.ppl.syntax.legacy.preferred`](../admin/settings.md) is `true` (default), integer operands follow the legacy truncating result. When the setting is `false`, the operands are promoted to floating-point, preserving the fractional part. Division by zero returns `NULL`. 5. `%`: Modulo. This operator can only be used with integers and returns the remainder of the division. +### Overflow behavior + +Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + ### Precedence You can use parentheses to control the precedence of arithmetic operators. Otherwise, operators with higher precedence are performed first. diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index f349523ec5..b6adbc5833 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index c42ecef213..efd325acf5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index ffd55ffb1f..101238a2a7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -6,4 +6,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index 64e868f9f8..90ec92f0b2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index 8f60e23e49..5186752243 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index 7ad040f826..9ac70f8b5c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[-($t14, $t13)], proj#0..12=[{exprs}], age2=[$t14], age3=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEzHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQE3HsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICIrIiwKICAgICAgICAia2luZCI6ICJDSEVDS0VEX1BMVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index e1328084f7..1009e374b1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject($f1=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml index fbec63d8c6..88bae9d4d5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCwnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCynsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml index de78240cbe..9be9d4ea11 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"CHECKED_MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index f5834404f1..f1b1e2b452 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[1:BIGINT], expr#16=[+($t7, $t15)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t16]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml new file mode 100644 index 0000000000..930e61fdf7 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml @@ -0,0 +1,170 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# Integer/long arithmetic (+, -, *) must not silently wrap on overflow. +# +# byte/short/int overflow is prevented by operand widening (short/byte -> INT, any int/long -> +# BIGINT) in PPLFuncImpTable, so those produce the correct wider value (HTTP 200). long (BIGINT) +# arithmetic has no wider integer type to widen into, so overflow is detected via checked +# arithmetic (Math.addExact / multiplyExact) and surfaced as a 4xx client error instead of wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_overflow_5164 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + int_field: + type: integer + long_field: + type: long + small_int: + type: integer + short_field: + type: short + - do: + bulk: + index: test_overflow_5164 + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"int_field": 2147483647, "long_field": 9223372036854775807, "small_int": 10, "short_field": 30000}' + - '{"index": {"_id": "2"}}' + - '{"int_field": 100, "long_field": 200, "small_int": 5, "short_field": 3}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_overflow_5164 + ignore_unavailable: true + +--- +"Normal integer addition does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where small_int = 5 | eval sum = int_field + small_int | fields int_field, small_int, sum + - match: { total: 1 } + - match: { datarows: [[100, 5, 105]] } + +--- +"Integer addition overflow widens instead of wrapping": + - skip: + features: + - headers + # int + int is widened to BIGINT before the operation; 2147483647 + 1 = 2147483648 fits exactly. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field + 1 | fields widened + - match: { total: 1 } + - match: { datarows: [[2147483648]] } + +--- +"Integer multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # 2147483647 * 2 = 4294967294 exceeds INT range but is exact once widened to BIGINT. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field * 2 | fields widened + - match: { total: 1 } + - match: { datarows: [[4294967294]] } + +--- +"Short multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # short * short historically wrapped into the 16-bit SMALLINT range (bug bash #8: 30000 * 30000 + # wrapped). Widening to INT keeps the exact value 900000000. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where short_field = 30000 | eval widened = short_field * short_field | fields widened + - match: { total: 1 } + - match: { datarows: [[900000000]] } + +--- +"Long addition overflow throws error": + - skip: + features: + - headers + # long + long has no wider integer type; overflow is detected and surfaced as a 4xx. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field + 1 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long multiplication overflow throws error": + - skip: + features: + - headers + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field * 2 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long subtraction overflow throws error": + - skip: + features: + - headers + # long_field is i64::MAX; subtracting -1 overflows past the top of the BIGINT range. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field - (-1) | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Normal long arithmetic does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 200 | eval product = long_field * 2 | fields product + - match: { total: 1 } + - match: { datarows: [[400]] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java index d77dee3e29..8a0b4daa57 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -232,6 +232,7 @@ public Object toJson(RexNode node) { map.put("operands", list); switch (node.getKind()) { case MINUS: + case CHECKED_MINUS: case CAST: case SAFE_CAST: map.put("type", toJson(node.getType())); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index dab778923b..aad2623a6f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -55,7 +55,7 @@ public static Optional> getOrderEquivalentInputInfo(RexNo case MINUS_PREFIX: return getOrderEquivalentInputInfo(((RexCall) expr).getOperands().get(0)) .map(inputInfo -> Pair.of(inputInfo.getLeft(), !inputInfo.getRight())); - case PLUS, MINUS: + case PLUS, MINUS, CHECKED_PLUS, CHECKED_MINUS: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1); @@ -68,12 +68,14 @@ public static Optional> getOrderEquivalentInputInfo(RexNo } RexNode variable = operand0Lit ? operand1 : operand0; - boolean flipped = (expr.getKind() == SqlKind.MINUS) && operand0Lit; + boolean isMinus = + expr.getKind() == SqlKind.MINUS || expr.getKind() == SqlKind.CHECKED_MINUS; + boolean flipped = isMinus && operand0Lit; return getOrderEquivalentInputInfo(variable) .map(inputInfo -> Pair.of(inputInfo.getLeft(), flipped != inputInfo.getRight())); } - case TIMES: + case TIMES, CHECKED_TIMES: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1);