From 4c2abfee80c2621a64b67862a57fb298dc1241f5 Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Tue, 4 Aug 2026 11:58:38 +0200 Subject: [PATCH 1/3] [SPARK-49828][SQL] Make Column(expression) usable outside of the org.apache.spark package `ClassicConversions.ColumnConstructorExt` is a public `@DeveloperApi` that adds a `Column(e: Expression)` factory to the `Column` companion, but it cannot be used outside of `org.apache.spark`, because `object Column` itself is `private[spark]`. `ExpressionUtils`, `ExpressionColumnNode` and `ColumnNode` are package private too, so since SPARK-49022 removed the public `new Column(expr: Expression)` constructor there has been no public Expression -> Column path at all. The reverse direction is public and works: `ColumnConversions.expression(col)`. This makes `object Column` public. Every member of it is already `private[spark]` or narrower, so the already shipped extension resolves while no new member becomes visible; the two `apply` overloads that relied on the object's own visibility are now marked `private[spark]` explicitly. It also adds a named `ClassicConversions.column(e)` for callers that prefer not to rely on an implicit. The named factory is on the object rather than on the `ClassicConversions` trait on purpose: on the trait it shadows `functions.column(colName: String)` for everyone who mixes the trait in, which breaks existing call sites. Prior art: https://github.com/apache/spark/pull/48306 by holdenk, which took the broader approach of exposing the ColumnNode AST types and was closed by the stale bot. --- .../scala/org/apache/spark/sql/Column.scala | 12 +++-- .../spark/sql/classic/conversions.scala | 16 +++++- .../spark/sql/ExpressionToColumnSuite.scala | 49 +++++++++++++++++++ 3 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala index 56a9787db092e..a9ee2b214dd01 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala @@ -30,11 +30,17 @@ import org.apache.spark.sql.internal.{ColumnNode, TableValuedFunctionArgument} import org.apache.spark.sql.types._ import org.apache.spark.util.ArrayImplicits._ -private[spark] object Column { +/** + * The companion object is public so that extension developers can reference the `Column` type as a + * value, which is what makes the `Column(expression)` factory provided by + * `org.apache.spark.sql.classic.ClassicConversions.ColumnConstructorExt` usable outside of the + * `org.apache.spark` package. All of its members intentionally stay internal. + */ +object Column { - def apply(colName: String): Column = new Column(colName) + private[spark] def apply(colName: String): Column = new Column(colName) - def apply(node: => ColumnNode): Column = withOrigin(new Column(node)) + private[spark] def apply(node: => ColumnNode): Column = withOrigin(new Column(node)) /** * Invoke a function with an options map as its last argument. If there are no options, its diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala index 3cfdace73c458..fd5771778b277 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala @@ -59,7 +59,21 @@ trait ClassicConversions { } @DeveloperApi -object ClassicConversions extends ClassicConversions +object ClassicConversions extends ClassicConversions { + /** + * Convert an [[Expression]] into a [[Column]]. This is the counterpart of + * [[ColumnConversions.expression]], for callers that would rather name the conversion than rely + * on the implicit [[ClassicConversions.ColumnConstructorExt]]. + * + * This is intentionally defined on the object rather than on the [[ClassicConversions]] trait: + * on the trait it would shadow `functions.column(colName: String)` for everyone who mixes the + * trait in, which is the trait's documented use case. + * + * @since 4.4.0 + */ + @DeveloperApi + def column(e: Expression): Column = ExpressionUtils.column(e) +} /** * Conversions from a [[Column]] to an [[Expression]]. diff --git a/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala b/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala new file mode 100644 index 0000000000000..c03453b4856a4 --- /dev/null +++ b/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test.org.apache.spark.sql + +import org.apache.spark.sql.{Column, QueryTest, Row} +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.classic.{ClassicConversions, ColumnConversions} +import org.apache.spark.sql.classic.ClassicConversions._ +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Tests the public Expression <-> Column conversions from a package outside of + * `org.apache.spark`, which is the only way to catch that a step of the path is package-private. + * Compiling this suite is as much a part of the test as running it. + */ +class ExpressionToColumnSuite extends QueryTest with SharedSparkSession { + + test("SPARK-49828: build a Column from an Expression via the Column companion") { + val e: Expression = Literal(1) + val c: Column = Column(e) + assert(ColumnConversions.expression(c) == e) + } + + test("SPARK-49828: build a Column from an Expression via ClassicConversions.column") { + val e: Expression = Literal(1) + val c: Column = ClassicConversions.column(e) + assert(ColumnConversions.expression(c) == e) + } + + test("SPARK-49828: a Column built from an Expression is usable in a query") { + val df = spark.range(2).select(Column(Literal(1)).as("one")) + checkAnswer(df, Seq(Row(1), Row(1))) + } +} From 171c8503b263110b57983fbb214b95370af9262b Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Tue, 4 Aug 2026 15:12:23 +0200 Subject: [PATCH 2/3] Reformat the new scaladoc to satisfy scalafmt sql/api is a scalafmt governed module with maxColumn = 98, one line was 99. --- sql/api/src/main/scala/org/apache/spark/sql/Column.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala index a9ee2b214dd01..d708c864d3ea3 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala @@ -31,8 +31,8 @@ import org.apache.spark.sql.types._ import org.apache.spark.util.ArrayImplicits._ /** - * The companion object is public so that extension developers can reference the `Column` type as a - * value, which is what makes the `Column(expression)` factory provided by + * The companion object is public so that extension developers can reference the `Column` type as + * a value, which is what makes the `Column(expression)` factory provided by * `org.apache.spark.sql.classic.ClassicConversions.ColumnConstructorExt` usable outside of the * `org.apache.spark` package. All of its members intentionally stay internal. */ From f4b2d232a86e3ba6f8ce599a75f020ec7b7489fc Mon Sep 17 00:00:00 2001 From: Peter Toth Date: Tue, 4 Aug 2026 19:01:26 +0200 Subject: [PATCH 3/3] Address review: annotate the Column companion and neutralize its doc - Mark `object Column` `@DeveloperApi` with `@since 4.4.0`, matching the audience of the conversions that make `Column(expression)` work, which are `@DeveloperApi` themselves. - Drop the classic-specific `ClassicConversions.ColumnConstructorExt` reference from the scaladoc. `Column.scala` lives in `sql/api`, which Spark Connect shares, so a Connect user browsing the API docs would have been pointed at a classic-only API. --- .../src/main/scala/org/apache/spark/sql/Column.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala index d708c864d3ea3..4befebd8418b5 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql import scala.jdk.CollectionConverters._ -import org.apache.spark.annotation.Stable +import org.apache.spark.annotation.{DeveloperApi, Stable} import org.apache.spark.internal.Logging import org.apache.spark.internal.LogKeys.{LEFT_EXPR, RIGHT_EXPR} import org.apache.spark.sql.catalyst.parser.DataTypeParser @@ -31,11 +31,13 @@ import org.apache.spark.sql.types._ import org.apache.spark.util.ArrayImplicits._ /** - * The companion object is public so that extension developers can reference the `Column` type as - * a value, which is what makes the `Column(expression)` factory provided by - * `org.apache.spark.sql.classic.ClassicConversions.ColumnConstructorExt` usable outside of the - * `org.apache.spark` package. All of its members intentionally stay internal. + * The companion object is public so that the `Column` type can be referenced as a value. This + * allows an implementation to add a `Column(expression)` factory through an extension method. All + * of its members are internal to Spark. + * + * @since 4.4.0 */ +@DeveloperApi object Column { private[spark] def apply(colName: String): Column = new Column(colName)