Add lazy jq input with wader/gojq (jqresult package)

.github/workflows/go.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.23.2'
+        go-version: '1.24.0'
 
     - name: Build
       run: go build -v ./...

.github/workflows/golangci-lint.yml

@@ -19,9 +19,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.23.2'
+          go-version: '1.24.0'
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.60
+          version: v1.64.8
           args: --timeout=5m

.github/workflows/ko.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.23.x'
+          go-version: '1.24.x'
       - uses: actions/checkout@v3
 
       - uses: ko-build/setup-ko@v0.6

README.md

@@ -54,7 +54,7 @@ Arguments:
   database:                                    (required) ID of the database.
 ```
 
-Local build requires Go 1.23.
+Local build requires Go 1.24.
 
 ```
 $ go install github.com/apstndb/execspansql@latest
@@ -114,7 +114,18 @@ $ execspansql ${DATABASE_ID} --query-mode=PROFILE \
 
 ### Embedded jq
 
-execspansql can process output using embedded [gojq](https://github.com/itchyny/gojq) using `--jq-filter` flag.
+execspansql can process output using embedded [wader/gojq](https://github.com/wader/gojq) (jq-compatible; includes `JQValue` for lazy inputs) using `--filter` flag.
+
+`--jq-input-mode` controls how results are passed to jq (json/yaml only):
+
+| Mode | Input | Typical filter |
+|------|--------|----------------|
+| `eager` (default) | Full ResultSet object | `.`, `.stats.queryPlan` |
+| `lazy` | `JQValue` root (`metadata` / `rows` Iter / `stats`) | `.rows[]`, `.stats.queryPlan` |
+
+In `lazy` mode, `metadata` is populated after the first row is read from Spanner (or after a zero-row result). Prefer `.rows[]` to stream rows. Bare `.rows` is a lazy iterator: reuse it in one object literal (for example `{a: .rows, b: .rows}`) may not duplicate rows because jq can evaluate the subexpression once; use `{a: [.rows[]], b: [.rows[]]}` when you need two row arrays. After `.stats` drains the iterator, captured `.rows` values replay from materialized rows.
+
+Output expands top-level `gojq.Iter` to one JSON/YAML document per row (JSONL-style). Nested `Iter` values inside objects are expanded to arrays on encode.
 
 #### Example: Extract QueryPlan
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/apstndb/execspansql
 
-go 1.23.2
+go 1.24.0
 
 require (
 	cloud.google.com/go/spanner v1.84.1
@@ -14,9 +14,9 @@ require (
 	github.com/cloudspannerecosystem/memefish v0.6.2
 	github.com/google/go-cmp v0.7.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
-	github.com/itchyny/gojq v0.12.16
 	github.com/jessevdk/go-flags v1.6.1
 	github.com/samber/lo v1.53.0
+	github.com/wader/gojq v0.12.1-0.20260315123642-6d8c75fc0e74
 	go.opentelemetry.io/otel v1.36.0
 	go.opentelemetry.io/otel/bridge/opencensus v1.27.0
 	go.opentelemetry.io/otel/sdk v1.36.0
@@ -71,7 +71,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
-	github.com/itchyny/timefmt-go v0.1.6 // indirect
+	github.com/itchyny/timefmt-go v0.1.7 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -112,7 +112,7 @@ require (
 	golang.org/x/net v0.42.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.27.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect

go.sum

@@ -918,10 +918,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
-github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
-github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
-github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
+github.com/itchyny/timefmt-go v0.1.7 h1:xyftit9Tbw+Dc/huSSPJaEmX1TVL8lw5vxjJLK4GMMA=
+github.com/itchyny/timefmt-go v0.1.7/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI=
 github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
 github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -1058,6 +1056,8 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/wader/gojq v0.12.1-0.20260315123642-6d8c75fc0e74 h1:eLFDUQ8b/cmQTT50/+Jm/L8TpMwu6yra+BKz6X9eE/g=
+github.com/wader/gojq v0.12.1-0.20260315123642-6d8c75fc0e74/go.mod h1:USEjy5fdczNWD8kQrhsj1EjA0xeCxLtvF1ev8pN2qpE=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1385,8 +1385,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=

integration_test.go

@@ -14,6 +14,7 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 	"spheric.cloud/xiter"
 
+	"github.com/apstndb/execspansql/jqresult"
 	"github.com/apstndb/execspansql/params"
 )
 
@@ -286,4 +287,120 @@ func TestWithCloudSpannerEmulator(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("lazy jq with RowIterator", func(t *testing.T) {
+		runLazy := func(t *testing.T, filter string, redact bool, mode *sppb.ExecuteSqlRequest_QueryMode) []any {
+			t.Helper()
+			var opts spanner.QueryOptions
+			if mode != nil {
+				opts.Mode = mode
+			}
+			rowIter := client.Single().QueryWithOptions(ctx,
+				spanner.Statement{SQL: "SELECT SingerId FROM Singers ORDER BY SingerId LIMIT 3"},
+				opts,
+			)
+			code, err := jqresult.Compile(filter, jqresult.InputLazy)
+			if err != nil {
+				t.Fatal(err)
+			}
+			iter, cleanup, err := jqresult.Execute(code, jqresult.InputLazy, rowIter, nil, redact)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			var out []any
+			for {
+				v, ok := iter.Next()
+				if !ok {
+					return out
+				}
+				if err, isErr := v.(error); isErr {
+					t.Fatal(err)
+				}
+				out = append(out, v)
+			}
+		}
+
+		rows := runLazy(t, ".rows[]", false, sppb.ExecuteSqlRequest_NORMAL.Enum())
+		if len(rows) != 3 {
+			t.Fatalf("got %d row values, want 3", len(rows))
+		}
+
+		meta := runLazy(t, ".metadata.rowType.fields[0].name", false, sppb.ExecuteSqlRequest_NORMAL.Enum())
+		if len(meta) != 1 || meta[0] != "SingerId" {
+			t.Fatalf("metadata field name: got %v", meta)
+		}
+
+		stats := runLazy(t, ".stats.rowCount", false, sppb.ExecuteSqlRequest_PROFILE.Enum())
+		if len(stats) != 1 {
+			t.Fatalf("stats rowCount: got %v", stats)
+		}
+
+		lens := runLazy(t, "{alen: (.rows|length), blen: (.rows|length)}", false, sppb.ExecuteSqlRequest_NORMAL.Enum())
+		if len(lens) != 1 {
+			t.Fatalf("rows length output: got %v", lens)
+		}
+		lensObj, ok := lens[0].(map[string]any)
+		if !ok {
+			t.Fatalf("rows length type: %T", lens[0])
+		}
+		if lensObj["alen"] != 3 || lensObj["blen"] != 3 {
+			t.Fatalf("rows lengths: got %#v", lensObj)
+		}
+
+		dupRows := runLazy(t, "{a: [.rows[]], b: [.rows[]]}", false, sppb.ExecuteSqlRequest_NORMAL.Enum())
+		if len(dupRows) != 1 {
+			t.Fatalf("duplicate rows output: got %v", dupRows)
+		}
+		dupObj, ok := dupRows[0].(map[string]any)
+		if !ok {
+			t.Fatalf("duplicate rows type: %T", dupRows[0])
+		}
+		for _, key := range []string{"a", "b"} {
+			rowSlice, ok := dupObj[key].([]any)
+			if !ok || len(rowSlice) != 3 {
+				t.Fatalf("%s: got %v (%T)", key, dupObj[key], dupObj[key])
+			}
+		}
+
+		objectRows := runLazy(t, "{rows: .rows, rc: .stats.rowCount}", false, sppb.ExecuteSqlRequest_PROFILE.Enum())
+		if len(objectRows) != 1 {
+			t.Fatalf("object rows output: got %v", objectRows)
+		}
+		objRows, ok := objectRows[0].(map[string]any)
+		if !ok {
+			t.Fatalf("object rows type: %T", objectRows[0])
+		}
+		normalized, err := jqresult.NormalizeForEncode(objRows["rows"])
+		if err != nil {
+			t.Fatal(err)
+		}
+		rowSlice, ok := normalized.([]any)
+		if !ok || len(rowSlice) != 3 {
+			t.Fatalf("normalized rows after stats: got %v (%T)", normalized, normalized)
+		}
+
+		combined := runLazy(t, "{rc: .stats.rowCount, ids: [.rows[] | .[0]]}", false, sppb.ExecuteSqlRequest_PROFILE.Enum())
+		if len(combined) != 1 {
+			t.Fatalf("combined output: got %v", combined)
+		}
+		obj, ok := combined[0].(map[string]any)
+		if !ok {
+			t.Fatalf("combined type: %T", combined[0])
+		}
+		ids, ok := obj["ids"].([]any)
+		if !ok || len(ids) != 3 {
+			t.Fatalf("combined ids: got %v", obj["ids"])
+		}
+
+		redacted := runLazy(t, ".stats.rowCount", true, sppb.ExecuteSqlRequest_PROFILE.Enum())
+		if len(redacted) != 1 {
+			t.Fatalf("redacted stats: got %v", redacted)
+		}
+
+		redactedRows := runLazy(t, ".rows[]", true, sppb.ExecuteSqlRequest_NORMAL.Enum())
+		if len(redactedRows) != 0 {
+			t.Fatalf("redacted .rows[]: got %d values, want 0", len(redactedRows))
+		}
+	})
 }

jqresult/compile.go

@@ -0,0 +1,14 @@
+package jqresult
+
+import (
+	"github.com/wader/gojq"
+)
+
+// Compile parses filter and returns executable jq code for the given input mode.
+func Compile(filter string, mode InputMode) (*gojq.Code, error) {
+	q, err := gojq.Parse(filter)
+	if err != nil {
+		return nil, err
+	}
+	return gojq.Compile(q)
+}