diff --git a/README.md b/README.md index bb969cb2e33dd..4563969cb1473 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -![]()

Cube Core — Open-Source Semantic Layer

@@ -15,7 +14,8 @@ __Cube Core is the open-source semantic layer.__ Define metrics, dimensions, joi Cube Core works with all SQL data sources, including cloud data warehouses like Snowflake, Databricks, and BigQuery; query engines like Presto and Amazon Athena; and application databases like Postgres. It has a built-in relational caching engine to provide sub-second latency and high concurrency for API requests. Cube Core — semantic layer connecting data sources to embedded analytics, BI tools, and AI agents diff --git a/docs-mintlify/favicon.svg b/docs-mintlify/favicon.svg index f221d010d5b34..4380e08f5bb49 100644 --- a/docs-mintlify/favicon.svg +++ b/docs-mintlify/favicon.svg @@ -1,5 +1,4 @@ - - - - + + + diff --git a/docs-mintlify/logo/dark.svg b/docs-mintlify/logo/dark.svg index b7c87b15ed44c..f1fe59ae44cc0 100644 --- a/docs-mintlify/logo/dark.svg +++ b/docs-mintlify/logo/dark.svg @@ -1,14 +1,7 @@ - - - - - - - - - - - - - + + + + + + diff --git a/docs-mintlify/logo/light.svg b/docs-mintlify/logo/light.svg index 8ab6b62ae02c6..a1cedfe1b90d6 100644 --- a/docs-mintlify/logo/light.svg +++ b/docs-mintlify/logo/light.svg @@ -1,9 +1,7 @@ - - - - - - - - + + + + + + diff --git a/docs/content/cube-core-logo.png b/docs/content/cube-core-logo.png index 2132bf291a841..90522bf9f8b3d 100644 Binary files a/docs/content/cube-core-logo.png and b/docs/content/cube-core-logo.png differ diff --git a/docs/content/cube-core-schema.png b/docs/content/cube-core-schema.png new file mode 100644 index 0000000000000..8cfffdc1babf1 Binary files /dev/null and b/docs/content/cube-core-schema.png differ diff --git a/docs/public/apple-touch-icon-precomposed.png b/docs/public/apple-touch-icon-precomposed.png index 32d0216088c0d..9d9c654300869 100644 Binary files a/docs/public/apple-touch-icon-precomposed.png and b/docs/public/apple-touch-icon-precomposed.png differ diff --git a/docs/public/apple-touch-icon.png b/docs/public/apple-touch-icon.png index 32d0216088c0d..9d9c654300869 100644 Binary files a/docs/public/apple-touch-icon.png and b/docs/public/apple-touch-icon.png differ diff --git a/docs/public/favicon-16x16.png b/docs/public/favicon-16x16.png index 2fb9eeab96eff..78659e3e75052 100644 Binary files a/docs/public/favicon-16x16.png and b/docs/public/favicon-16x16.png differ diff --git a/docs/public/favicon-32x32.png b/docs/public/favicon-32x32.png index 102bec69d9d9f..4234a249e9b3e 100644 Binary files a/docs/public/favicon-32x32.png and b/docs/public/favicon-32x32.png differ diff --git a/docs/public/favicon.ico b/docs/public/favicon.ico index 820635b9bedd7..e23553ab7ed3b 100644 Binary files a/docs/public/favicon.ico and b/docs/public/favicon.ico differ diff --git a/docs/public/favicon.svg b/docs/public/favicon.svg index f0236c5742cc5..4380e08f5bb49 100644 --- a/docs/public/favicon.svg +++ b/docs/public/favicon.svg @@ -1,8 +1,4 @@ - - - - - - - - \ No newline at end of file + + + + diff --git a/packages/cubejs-backend-shared/package.json b/packages/cubejs-backend-shared/package.json index 0e83eccdd3e89..38f8770f6f1dc 100644 --- a/packages/cubejs-backend-shared/package.json +++ b/packages/cubejs-backend-shared/package.json @@ -30,7 +30,6 @@ "@cubejs-backend/linter": "1.7.19", "@types/bytes": "^3.1.5", "@types/cli-progress": "^3.9.1", - "@types/decompress": "^4.2.7", "@types/jest": "^29", "@types/node": "^22", "@types/node-fetch": "^2.5.8", @@ -45,8 +44,8 @@ "bytes": "^3.1.2", "cli-progress": "^3.9.0", "cross-spawn": "^7.0.3", - "decompress": "^4.2.1", "env-var": "^6.3.0", + "extract-zip": "^2.0.1", "fs-extra": "^9.1.0", "generic-pool": "^3.9.0", "lru-cache": "^11.1.0", @@ -55,6 +54,7 @@ "node-fetch": "^2.6.1", "proxy-agent": "^6.5.0", "shelljs": "^0.8.5", + "tar": "^7.5.22", "throttle-debounce": "^3.0.1", "uuid": "^8.3.2" }, diff --git a/packages/cubejs-backend-shared/src/http-utils.ts b/packages/cubejs-backend-shared/src/http-utils.ts index 246c95d569e3e..e46c37e08c9b2 100644 --- a/packages/cubejs-backend-shared/src/http-utils.ts +++ b/packages/cubejs-backend-shared/src/http-utils.ts @@ -1,4 +1,5 @@ -import decompress from 'decompress'; +import * as tar from 'tar'; +import extractZip from 'extract-zip'; import fetch, { Headers, Request, Response } from 'node-fetch'; import bytes from 'bytes'; import { throttle } from 'throttle-debounce'; @@ -63,6 +64,114 @@ export async function streamWithProgress( ); } +/** + * Options shared by every `tar.x` call here. + * + * `preserveOwner` defaults to true when running as root, which is the normal case + * inside the Cube image; extracted files would then take whatever uid/gid the + * tarball recorded. Writing as the current user matches how this path has always + * behaved. + * + * `onwarn` is load-bearing: tar *drops* unsafe entries with a warning rather than + * failing, so an archive consisting only of `../evil` would extract to nothing and + * resolve successfully, leaving the caller to fail later on a confusing + * missing-file error. + * + * Only `TAR_ENTRY_ERROR` goes through `internalExceptions`, deliberately: that + * helper calls `process.exit(1)` under `CUBEJS_INTERNAL_EXCEPTIONS=exit`, and tar + * also warns about benign conditions (unsupported entry types such as fifos and + * devices, `TAR_ENTRY_INVALID`, failed utime/chown). Routing those through it + * would let one odd entry in a third-party tarball take the process down + * mid-download, where previously it extracted and carried on. + * + * `TAR_ENTRY_ERROR` is not only the path-rejection code — measured, tar reports + * per-entry write failures through it too (a read-only target raises it once per + * entry, same as a `..` name does). Both belong on this side of the split: a + * half-extracted install is a real failure, and escalating it is what the opt-in + * `exit` setting asks for. Everything else is logged and ignored, as tar treats + * it. + */ +const tarOptions = { + preserveOwner: false, + onwarn: (code: string, message: string) => { + const warning = `tar skipped an entry while extracting (${code}): ${message}`; + + if (code === 'TAR_ENTRY_ERROR') { + internalExceptions(new Error(warning)); + + return; + } + + console.warn(warning); + }, +}; + +/** + * Extract a downloaded archive into `cwd`, which is created if missing. + * + * Dispatches on magic bytes, not the filename, because there is no filename to + * dispatch on: `streamWithProgress` saves downloads as + * `crypto.randomBytes(16).toString('hex')`, with no extension. + * + * Handles gzip (`.tar.gz` / `.tgz`), uncompressed tar and zip. Two gaps are + * deliberate and both throw a named error rather than failing obscurely: bzip2, + * and pre-POSIX v7 tars, which carry no `ustar` magic at offset 257 to detect them + * by. + * + * Neither backend writes outside `cwd`: `tar` strips a leading `/` on extraction and + * drops entries containing `..`, and `extract-zip` rejects entries that resolve outside + * the target. + */ +export async function extractArchive(archivePath: string, cwd: string): Promise { + // `extract-zip` creates its target but `tar.x` throws `CwdError` when it is + // missing, so without this the contract would depend on the archive's format — + // which callers cannot know in advance, that being the point of magic-byte dispatch. + mkdirpSync(cwd); + + // 262 bytes: enough for the `ustar` magic a plain tar carries at offset 257. + const header = Buffer.alloc(262); + const fd = await fs.promises.open(archivePath, 'r'); + + let bytesRead: number; + + try { + ({ bytesRead } = await fd.read(header, 0, header.length, 0)); + } finally { + await fd.close(); + } + + const startsWith = (...magic: number[]) => bytesRead >= magic.length && magic.every((byte, i) => header[i] === byte); + + // gzip (1f 8b) covers .tar.gz/.tgz; `tar.x` gunzips transparently. + if (startsWith(0x1f, 0x8b)) { + await tar.x({ file: archivePath, cwd, ...tarOptions }); + return; + } + + // zip: the two-byte "PK" prefix, shared by a local file header and by the + // end-of-central-directory record that an empty archive consists of. + if (startsWith(0x50, 0x4b)) { + await extractZip(archivePath, { dir: path.resolve(cwd) }); + return; + } + + // Uncompressed tar: "ustar" at offset 257. + if (bytesRead >= 262 && header.subarray(257, 262).toString('latin1') === 'ustar') { + await tar.x({ file: archivePath, cwd, ...tarOptions }); + return; + } + + if (startsWith(0x42, 0x5a, 0x68)) { + throw new Error( + 'Unsupported archive format: bzip2. Supported formats are gzip (.tar.gz/.tgz), tar and zip.' + ); + } + + throw new Error( + 'Unable to detect archive format from its contents. Supported formats are gzip (.tar.gz/.tgz), tar and zip.' + ); +} + type DownloadAndExtractFile = { showProgress: boolean; cwd: string; @@ -111,7 +220,7 @@ export async function downloadAndExtractFile(url: string, { cwd, skipExtract, ds fs.copyFileSync(savedFilePath, destPath); } } else { - await decompress(savedFilePath, cwd); + await extractArchive(savedFilePath, cwd); } try { diff --git a/packages/cubejs-backend-shared/test/http-utils.test.ts b/packages/cubejs-backend-shared/test/http-utils.test.ts new file mode 100644 index 0000000000000..50e08d1de80be --- /dev/null +++ b/packages/cubejs-backend-shared/test/http-utils.test.ts @@ -0,0 +1,288 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import * as tar from 'tar'; +import { crc32 } from 'zlib'; + +import { extractArchive } from '../src/http-utils'; + +/** + * `extractArchive` replaced the unmaintained `decompress`, which carries two + * unfixed advisories — GHSA-mp2f-45pm-3cg9 ("archive extraction can create files + * and links outside of the target directory") and GHSA-h39j-r5qq-r9mm (Zip Slip). + * + * These tests exist to prove the replacement is not vulnerable to the same class, + * so they build genuinely hostile archives rather than asserting on library + * version numbers. They also cover the happy paths, because dispatch is by magic + * bytes: `streamWithProgress` saves downloads under a random hex name with no + * extension, so there is nothing to dispatch on by filename. + */ +describe('extractArchive', () => { + let work: string; + + beforeEach(() => { + work = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'extract-archive-')); + }); + + afterEach(() => { + fs.rmSync(work, { recursive: true, force: true }); + }); + + const targetDir = () => { + const dir = path.join(work, 'target'); + fs.mkdirSync(dir, { recursive: true }); + return dir; + }; + + /** + * Assert the fixture really is hostile before extracting it. + * + * The zip fixtures are safe by construction — the rejection itself proves the + * hostile name survived into the archive. The tar fixtures have no such witness: + * absolute-path stripping already happens in tar's `WriteEntry` constructor, and + * only ordering keeps the `..` name assigned in `onWriteEntry` intact. If a future + * tar normalises it, the fixture silently becomes benign and these tests keep + * passing while proving nothing — the exact trap the zip fixture is hand-rolled to + * avoid. So read the names back. + */ + const storedNames = async (archive: string) => { + const names: string[] = []; + await tar.t({ file: archive, onReadEntry: (e) => names.push(e.path) }); + return names; + }; + + /** + * Build a .zip with entry names stored verbatim. + * + * Hand-rolled (stored/uncompressed, so no deflate needed) rather than using a + * zip library, because every maintained writer *sanitises* what it stores: + * `archiver` silently rewrites `../ZIP_PWNED.txt` to `ZIP_PWNED.txt`, which + * would make the Zip Slip test below extract a perfectly benign archive and + * pass for the wrong reason. Byte control is the point. + */ + const writeZip = async (file: string, entries: { name: string; content: string; mode?: number }[]) => { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + + for (const entry of entries) { + const name = Buffer.from(entry.name, 'utf8'); + const data = Buffer.from(entry.content, 'utf8'); + const sum = crc32(data); + + const lfh = Buffer.alloc(30); + lfh.writeUInt32LE(0x04034b50, 0); // local file header signature + lfh.writeUInt16LE(10, 4); // version needed + lfh.writeUInt16LE(0, 8); // method: stored + lfh.writeUInt32LE(sum, 14); + lfh.writeUInt32LE(data.length, 18); // compressed size + lfh.writeUInt32LE(data.length, 22); // uncompressed size + lfh.writeUInt16LE(name.length, 26); + local.push(lfh, name, data); + + const cdh = Buffer.alloc(46); + cdh.writeUInt32LE(0x02014b50, 0); // central directory signature + // version made by: high byte is the host system. 3 = unix, which is what a + // producer capable of recording a symlink emits — with the default 0 (MS-DOS) + // the external-attributes field is formally DOS attribute bits and the unix + // mode below is not meant to be read at all. + cdh.writeUInt16LE((3 << 8) | 20, 4); + cdh.writeUInt16LE(10, 6); // version needed + cdh.writeUInt16LE(0, 10); // method: stored + cdh.writeUInt32LE(sum, 16); + cdh.writeUInt32LE(data.length, 20); + cdh.writeUInt32LE(data.length, 24); + cdh.writeUInt16LE(name.length, 28); + // External attributes carry the unix mode in the high 16 bits, which is how a + // zip records a symlink (`0o120000`). `>>> 0` because the shift overflows into a + // negative signed int32 otherwise. + cdh.writeUInt32LE((((entry.mode ?? 0o100644) << 16) >>> 0), 38); + cdh.writeUInt32LE(offset, 42); // relative offset of local header + central.push(cdh, name); + + offset += lfh.length + name.length + data.length; + } + + const centralBuf = Buffer.concat(central); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); // end of central directory signature + eocd.writeUInt16LE(entries.length, 8); + eocd.writeUInt16LE(entries.length, 10); + eocd.writeUInt32LE(centralBuf.length, 12); + eocd.writeUInt32LE(offset, 16); + + await fs.promises.writeFile(file, Buffer.concat([...local, centralBuf, eocd])); + }; + + /** Build a .tar.gz whose entries we control byte-for-byte, including hostile names. */ + const writeTarGz = async (file: string, entries: { name: string; content?: string; symlinkTo?: string }[]) => { + const stage = fs.mkdtempSync(path.join(work, 'stage-')); + const names: string[] = []; + + for (const entry of entries) { + // Stage under a safe name, then rewrite the stored name via tar's own API. + const safe = `entry-${names.length}`; + if (entry.symlinkTo !== undefined) { + fs.symlinkSync(entry.symlinkTo, path.join(stage, safe)); + } else { + fs.writeFileSync(path.join(stage, safe), entry.content ?? ''); + } + names.push(safe); + } + + await tar.c( + { + file, + gzip: true, + cwd: stage, + portable: true, + onWriteEntry(e) { + const idx = names.indexOf(e.path); + if (idx >= 0) { + // eslint-disable-next-line no-param-reassign + e.path = entries[idx].name; + } + }, + }, + names + ); + }; + + describe('refuses to write outside the target directory', () => { + it('drops a tar entry that traverses up with ..', async () => { + const archive = path.join(work, 'evil.tar.gz'); + await writeTarGz(archive, [ + { name: '../PWNED.txt', content: 'pwned' }, + // A benign sibling, so a pass distinguishes "dropped the bad entry" from + // "extracted nothing at all". + { name: 'safe.txt', content: 'safe' }, + ]); + + expect(await storedNames(archive)).toContain('../PWNED.txt'); + + const target = targetDir(); + await extractArchive(archive, target); + + expect(fs.existsSync(path.join(work, 'PWNED.txt'))).toBe(false); + expect(fs.readFileSync(path.join(target, 'safe.txt'), 'utf8')).toBe('safe'); + }); + + it('contains a tar entry with an absolute path instead of honouring it', async () => { + const archive = path.join(work, 'abs.tar.gz'); + const escapeTo = path.join(work, 'ABS_PWNED.txt'); + await writeTarGz(archive, [{ name: escapeTo, content: 'pwned' }]); + + // The absolute name survives verbatim into the archive — tar strips the leading + // `/` when *extracting*, not when writing — so the fixture really is hostile. + expect(await storedNames(archive)).toContain(escapeTo); + + const target = targetDir(); + await extractArchive(archive, target); + + // tar strips the leading `/` rather than writing to the absolute location, so + // the entry lands *inside* the target, re-rooted at its otherwise-unchanged + // path. Assert that positively: "nothing escaped" alone cannot distinguish + // contained from dropped. + expect(fs.existsSync(escapeTo)).toBe(false); + expect(fs.existsSync(path.join(target, escapeTo))).toBe(true); + }); + + it('rejects a zip entry that traverses up with .. (Zip Slip)', async () => { + const archive = path.join(work, 'evil.zip'); + await writeZip(archive, [{ name: '../ZIP_PWNED.txt', content: 'pwned' }]); + + await expect(extractArchive(archive, targetDir())).rejects.toThrow(/invalid relative path/i); + expect(fs.existsSync(path.join(work, 'ZIP_PWNED.txt'))).toBe(false); + }); + + it('does not follow a tar symlink that points outside the target', async () => { + const archive = path.join(work, 'sym.tar.gz'); + const outside = path.join(work, 'outside'); + fs.mkdirSync(outside); + + await writeTarGz(archive, [ + { name: 'esc', symlinkTo: outside }, + { name: 'esc/SYM_PWNED.txt', content: 'pwned' }, + ]); + + expect(await storedNames(archive)).toEqual( + expect.arrayContaining(['esc', 'esc/SYM_PWNED.txt']) + ); + + // Either it refuses the entry or it writes inside the target; it must not + // materialise a file in `outside`. + await extractArchive(archive, targetDir()).catch(() => undefined); + + expect(fs.existsSync(path.join(outside, 'SYM_PWNED.txt'))).toBe(false); + }); + + it('does not write through a zip symlink that points outside the target', async () => { + // The zip backend's containment is the half worth proving separately: a symlink + // entry has a clean relative *name*, so only a check on the resolved destination + // catches the entry written through it afterwards. + const archive = path.join(work, 'zipsym.zip'); + const outside = path.join(work, 'outside'); + fs.mkdirSync(outside); + + await writeZip(archive, [ + { name: 'esc', content: outside, mode: 0o120777 }, + { name: 'esc/PWNED.txt', content: 'pwned-through-symlink' }, + ]); + + await expect(extractArchive(archive, targetDir())).rejects.toThrow(/out of bound path/i); + expect(fs.existsSync(path.join(outside, 'PWNED.txt'))).toBe(false); + }); + }); + + describe('extracts the formats the previous implementation supported', () => { + it('detects gzip from magic bytes and extracts a .tar.gz', async () => { + const archive = path.join(work, 'good.tar.gz'); + await writeTarGz(archive, [{ name: 'dir/file.txt', content: 'legit-content' }]); + + const target = targetDir(); + await extractArchive(archive, target); + + expect(fs.readFileSync(path.join(target, 'dir', 'file.txt'), 'utf8')).toBe('legit-content'); + }); + + it('detects a zip from the PK magic and extracts it', async () => { + const archive = path.join(work, 'good.zip'); + await writeZip(archive, [{ name: 'dir/file.txt', content: 'legit-content' }]); + + const target = targetDir(); + await extractArchive(archive, target); + + expect(fs.readFileSync(path.join(target, 'dir', 'file.txt'), 'utf8')).toBe('legit-content'); + }); + + it('detects an uncompressed tar from the ustar magic at offset 257', async () => { + const stage = fs.mkdtempSync(path.join(work, 'plain-')); + fs.mkdirSync(path.join(stage, 'dir')); + fs.writeFileSync(path.join(stage, 'dir', 'file.txt'), 'legit-content'); + const archive = path.join(work, 'good.tar'); + await tar.c({ file: archive, cwd: stage, portable: true }, ['dir']); + + const target = targetDir(); + await extractArchive(archive, target); + + expect(fs.readFileSync(path.join(target, 'dir', 'file.txt'), 'utf8')).toBe('legit-content'); + }); + }); + + describe('fails loudly on formats it cannot handle', () => { + it('names bzip2 rather than failing obscurely', async () => { + // BZh magic; the body does not need to be a valid stream to be classified. + const archive = path.join(work, 'x.tar.bz2'); + fs.writeFileSync(archive, Buffer.concat([Buffer.from('BZh9'), Buffer.alloc(300)])); + + await expect(extractArchive(archive, targetDir())).rejects.toThrow(/bzip2/); + }); + + it('rejects a file that is not an archive at all', async () => { + const archive = path.join(work, 'junk.bin'); + fs.writeFileSync(archive, Buffer.concat([Buffer.from([0, 1, 2]), Buffer.alloc(300)])); + + await expect(extractArchive(archive, targetDir())).rejects.toThrow(/Unable to detect archive format/); + }); + }); +}); diff --git a/packages/cubejs-playground/public/apple-touch-icon.png b/packages/cubejs-playground/public/apple-touch-icon.png index 32d0216088c0d..9d9c654300869 100644 Binary files a/packages/cubejs-playground/public/apple-touch-icon.png and b/packages/cubejs-playground/public/apple-touch-icon.png differ diff --git a/packages/cubejs-playground/public/cube-core-logo-adapted_for_dark_bg.svg b/packages/cubejs-playground/public/cube-core-logo-adapted_for_dark_bg.svg index c1eb24423d03a..c5952c30654c2 100644 --- a/packages/cubejs-playground/public/cube-core-logo-adapted_for_dark_bg.svg +++ b/packages/cubejs-playground/public/cube-core-logo-adapted_for_dark_bg.svg @@ -1,18 +1,11 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + diff --git a/packages/cubejs-playground/public/favicon-16x16.png b/packages/cubejs-playground/public/favicon-16x16.png index 2fb9eeab96eff..78659e3e75052 100644 Binary files a/packages/cubejs-playground/public/favicon-16x16.png and b/packages/cubejs-playground/public/favicon-16x16.png differ diff --git a/packages/cubejs-playground/public/favicon-32x32.png b/packages/cubejs-playground/public/favicon-32x32.png index 102bec69d9d9f..4234a249e9b3e 100644 Binary files a/packages/cubejs-playground/public/favicon-32x32.png and b/packages/cubejs-playground/public/favicon-32x32.png differ diff --git a/packages/cubejs-playground/public/favicon.ico b/packages/cubejs-playground/public/favicon.ico index 820635b9bedd7..e23553ab7ed3b 100644 Binary files a/packages/cubejs-playground/public/favicon.ico and b/packages/cubejs-playground/public/favicon.ico differ diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/primary-key-multi-fact.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/primary-key-multi-fact.test.ts new file mode 100644 index 0000000000000..3e14b599e7198 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/integration/postgres/primary-key-multi-fact.test.ts @@ -0,0 +1,94 @@ +import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; +import { prepareYamlCompiler } from '../../unit/PrepareCompiler'; +import { dbRunner } from './PostgresDBRunner'; + +// A cube's own primary key as a query dimension, next to measures from two +// cubes. The measures split into per-cube subqueries, and the one on the +// `one` side of the join is multiplied by the fan-out, so it is read through +// the keys subquery and re-joined to its own cube by that same primary key. +// The key plays two roles at once - query dimension and re-join key - and has +// to be projected once: two columns under one alias make the re-join's +// reference to it ambiguous. +describe('Primary key dimension on the multi-fact path', () => { + jest.setTimeout(200000); + + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(` +cubes: + - name: cube_a + sql_alias: a + sql: > + SELECT 1 AS id, 100 AS value_a UNION ALL + SELECT 2 AS id, 200 AS value_a + dimensions: + - name: id + sql: id + type: number + primary_key: true + public: true + measures: + - name: measure_a + sql: value_a + type: sum + + - name: cube_b + sql_alias: b + sql: > + SELECT 10 AS id, 1 AS a_id, '2026-07-05'::timestamp AS date, 5 AS value_b UNION ALL + SELECT 11 AS id, 1 AS a_id, '2026-07-10'::timestamp AS date, 7 AS value_b UNION ALL + SELECT 12 AS id, 2 AS a_id, '2026-07-15'::timestamp AS date, 9 AS value_b + joins: + - name: cube_a + relationship: many_to_one + sql: "{CUBE.a_id} = {cube_a.id}" + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: a_id + sql: a_id + type: number + - name: date + sql: date + type: time + measures: + - name: measure_b + sql: value_b + type: sum + `); + + async function runQuery(q) { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, q); + return dbRunner.testQuery(query.buildSqlAndParams()); + } + + it('primary key dimension next to measures from two cubes', async () => { + // measure_a must be counted once per cube_a row despite the two cube_b + // rows that share a_id = 1. + expect(await runQuery({ + measures: ['cube_b.measure_b', 'cube_a.measure_a'], + dimensions: ['cube_a.id'], + timeDimensions: [{ + dimension: 'cube_b.date', + granularity: 'month', + dateRange: ['2026-07-01', '2026-07-31'], + }], + order: [{ id: 'cube_a.id' }], + timezone: 'UTC', + })).toEqual([ + { + a__id: 1, + b__date_month: '2026-07-01T00:00:00.000Z', + b__measure_b: '12', + a__measure_a: '100', + }, + { + a__id: 2, + b__date_month: '2026-07-01T00:00:00.000Z', + b__measure_b: '9', + a__measure_a: '200', + }, + ]); + }); +}); diff --git a/packages/cubejs-templates/package.json b/packages/cubejs-templates/package.json index 5ac1e8c6de074..d943f961be222 100644 --- a/packages/cubejs-templates/package.json +++ b/packages/cubejs-templates/package.json @@ -33,12 +33,11 @@ "dependencies": { "@cubejs-backend/shared": "1.7.19", "cross-spawn": "^7.0.3", - "decompress": "^4.2.1", - "decompress-targz": "^4.1.1", "fs-extra": "^9.1.0", "node-fetch": "^2.6.1", "ramda": "^0.27.2", - "source-map-support": "^0.5.19" + "source-map-support": "^0.5.19", + "tar": "^7.5.22" }, "devDependencies": { "@cubejs-backend/linter": "1.7.19", diff --git a/packages/cubejs-templates/src/PackageFetcher.ts b/packages/cubejs-templates/src/PackageFetcher.ts index c5e0d36169ce4..f8d00a6b4e38d 100644 --- a/packages/cubejs-templates/src/PackageFetcher.ts +++ b/packages/cubejs-templates/src/PackageFetcher.ts @@ -1,6 +1,5 @@ import fs from 'fs-extra'; -import decompress from 'decompress'; -import decompressTargz from 'decompress-targz'; +import * as tar from 'tar'; import path from 'path'; import { executeCommand } from '@cubejs-backend/shared'; @@ -61,8 +60,14 @@ export class PackageFetcher { public async downloadPackages() { await this.downloadRepo(); - await decompress(this.repoArchivePath, this.tmpFolderPath, { - plugins: [decompressTargz()], + // Only ever a gzipped tar (GitHub's /archive/.tar.gz). `tar.x` refuses to + // write outside `cwd`: a leading `/` is stripped on extraction and entries containing `..` are + // dropped — but dropped with a warning rather than an error, so surface it. + await tar.x({ + file: this.repoArchivePath, + cwd: this.tmpFolderPath, + preserveOwner: false, + onwarn: (code, message) => console.warn(`tar skipped an entry (${code}): ${message}`), }); const dir = fs.readdirSync(this.tmpFolderPath).find((name) => !name.endsWith('tar.gz')); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs index 86a28dbea19f8..28d2365924f86 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/processors/keys_sub_query.rs @@ -86,6 +86,20 @@ impl<'a> LogicalNodeProcessor<'a, KeysSubQuery> for KeysSubQueryProcessor<'a> { if !context.dimensions_query { for member in keys_subquery.primary_keys_dimensions().iter() { + // A primary key that is also a query dimension is already + // projected above. Projecting it again would put two columns + // under one alias, making every reference to it from the + // enclosing re-join ambiguous. Symbols are matched the way + // `Schema::find_column_for_member` matches them, so that the + // re-join resolves to the surviving column. + let resolved = member.clone().resolve_reference_chain(); + if keys_subquery + .schema() + .all_dimensions() + .any(|dim| dim.clone().resolve_reference_chain() == resolved) + { + continue; + } let alias = member.alias(); references_builder.resolve_references_for_member( member.clone(), diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_fact.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_fact.rs index f7381b4e99a6e..794148bbba6fd 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_fact.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/multi_fact.rs @@ -598,6 +598,35 @@ async fn test_non_multiplied_multi_join() { } } +#[tokio::test(flavor = "multi_thread")] +async fn test_multiplied_aggregate_grouped_by_own_primary_key() { + let ctx = create_context(); + + // customers.total_lifetime_value is multiplied by the customers→orders + // join, so it is read through the keys subquery and re-joined to customers + // by customers' primary key. That key is also a query dimension here, so it + // plays both roles at once and the keys subquery has to project it exactly + // once - two columns under one alias make every reference to it from the + // re-join ambiguous. + let query = indoc! {" + measures: + - customers.total_lifetime_value + - orders.count + dimensions: + - customers.id + - orders.status + order: + - id: customers.id + - id: orders.status + "}; + + ctx.build_sql(query).unwrap(); + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + #[tokio::test(flavor = "multi_thread")] async fn test_multi_fact_view_two_facts_with_measure_filter() { let schema = MockSchema::from_yaml_file("common/integration_multi_fact_view.yaml"); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__multi_fact__multiplied_aggregate_grouped_by_own_primary_key.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__multi_fact__multiplied_aggregate_grouped_by_own_primary_key.snap new file mode 100644 index 0000000000000..91ae41cf6c1ac --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__multi_fact__multiplied_aggregate_grouped_by_own_primary_key.snap @@ -0,0 +1,12 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/multi_fact.rs +expression: result +--- +customers__id | orders__status | customers__total_lifetime_value | orders__count +--------------+----------------+---------------------------------+-------------- +1 | completed | 1000.00 | 2 +1 | pending | 1000.00 | 2 +2 | completed | 2000.00 | 2 +2 | pending | 2000.00 | 1 +3 | NULL | 500.00 | 0 +4 | completed | 1500.00 | 1 diff --git a/yarn.lock b/yarn.lock index b79cf7d85d62c..142b68dd722b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8124,13 +8124,6 @@ "@types/node" "*" aws-sdk "^2.819.0" -"@types/decompress@^4.2.7": - version "4.2.7" - resolved "https://registry.yarnpkg.com/@types/decompress/-/decompress-4.2.7.tgz#604f69b69d519ecb74dea1ea0829f159b85e1332" - integrity sha512-9z+8yjKr5Wn73Pt17/ldnmQToaFHZxK0N1GHysuk/JIPT8RIdQeoInM01wWPgypRcvb6VH1drjuFpQ4zmY437g== - dependencies: - "@types/node" "*" - "@types/dedent@^0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050" @@ -12169,7 +12162,7 @@ decompress-tarbz2@^4.0.0: seek-bzip "^1.0.5" unbzip2-stream "^1.0.9" -decompress-targz@^4.0.0, decompress-targz@^4.1.1: +decompress-targz@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/decompress-targz/-/decompress-targz-4.1.1.tgz#c09bc35c4d11f3de09f2d2da53e9de23e7ce1eee" integrity sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w== @@ -13685,7 +13678,7 @@ extract-stack@^2.0.0: resolved "https://registry.yarnpkg.com/extract-stack/-/extract-stack-2.0.0.tgz#11367bc865bfcd9bc0db3123e5edb57786f11f9b" integrity sha512-AEo4zm+TenK7zQorGK1f9mJ8L14hnTDi2ZQPR+Mub1NX8zimka1mXpV5LpH8x9HoUmFSHZCfLHqWvp0Y4FxxzQ== -extract-zip@2.0.1: +extract-zip@2.0.1, extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== @@ -22720,16 +22713,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -22812,7 +22796,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -22833,13 +22817,6 @@ strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -23127,6 +23104,17 @@ tar@^7.4.3, tar@^7.5.2: minizlib "^3.1.0" yallist "^5.0.0" +tar@^7.5.22: + version "7.5.22" + resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.22.tgz#a696f998136e71487dc3f869a85bba2c67971ba9" + integrity sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA== + dependencies: + "@isaacs/fs-minipass" "^4.0.0" + chownr "^3.0.0" + minipass "^7.1.2" + minizlib "^3.1.0" + yallist "^5.0.0" + tarn@^3.0.1, tarn@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693" @@ -24708,7 +24696,7 @@ workerpool@^9.2.0: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-9.2.0.tgz#f74427cbb61234708332ed8ab9cbf56dcb1c4371" integrity sha512-PKZqBOCo6CYkVOwAxWxQaSF2Fvb5Iv2fCeTP7buyWI2GiynWr46NcXSgK/idoV6e60dgCBfgYc+Un3HMvmqP8w== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -24726,15 +24714,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"