From f7d4653e5f8cbd34e033ae9238572d53cd9e08e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Fri, 31 Jul 2026 02:10:48 +0200 Subject: [PATCH] Read Reprint integration metadata through its command --- apps/cli/commands/pull-reprint.ts | 92 +++++--- apps/cli/commands/tests/pull-reprint.test.ts | 206 +++++++++--------- apps/cli/lib/pull/reprint-metadata.test.ts | 46 ++++ apps/cli/lib/pull/reprint-metadata.ts | 38 ++++ apps/cli/lib/pull/reprint-selector.test.ts | 2 +- apps/cli/lib/pull/reprint-selector.ts | 23 +- apps/cli/lib/pull/reprint-state.test.ts | 138 +++++------- apps/cli/lib/pull/reprint-state.ts | 154 +------------ .../lib/pull/runtime-start-options.test.ts | 64 ++---- apps/cli/lib/pull/runtime-start-options.ts | 100 +++------ 10 files changed, 374 insertions(+), 489 deletions(-) create mode 100644 apps/cli/lib/pull/reprint-metadata.test.ts create mode 100644 apps/cli/lib/pull/reprint-metadata.ts diff --git a/apps/cli/commands/pull-reprint.ts b/apps/cli/commands/pull-reprint.ts index a83c17abb8..2cf724f357 100644 --- a/apps/cli/commands/pull-reprint.ts +++ b/apps/cli/commands/pull-reprint.ts @@ -40,6 +40,10 @@ import { runReprintCommandUntilComplete, } from 'cli/lib/pull/migration-client'; import { preserveUnselectedLocalContent } from 'cli/lib/pull/preserve-local-content'; +import { + readReprintImportMetadata, + type ReprintImportMetadata, +} from 'cli/lib/pull/reprint-metadata'; import { fetchReprintPullTree, mapCliOnlyToReprint, @@ -49,12 +53,6 @@ import { type SelectedSymlink, } from 'cli/lib/pull/reprint-selector'; import { - getAbspathFromState, - getContentDirFromState, - getCoreRootsFromState, - hasLocalFilesIndex, - getTablePrefixFromState, - hasSkippedFiles, markSkippedFilesPending, resetEssentialFilesState, setSqliteRuntimeTarget, @@ -145,7 +143,7 @@ export const registerCommand = ( yargs: StudioArgv ) => { /** * Where Studio stores the raw filesystem scratch space for each pulled - * site — reprint's `.import-state.json`, the preflight cache, and the + * site — Reprint's `pull/state.json`, the preflight cache, and the * raw/runtime working dirs. Each site's pull lives in a subdirectory * keyed by its `siteId` (see {@link getPullTechnicalDirectory}); there is * no Studio-owned progress file. `studio delete` removes this scratch. @@ -156,7 +154,7 @@ const PULLS_ROOT = path.join( os.homedir(), '.studio', 'pulls' ); * The on-disk scratch layout for a site's pull, all derived from the * site's identity (`siteId`) and layout (`SiteData`). There is no * Studio-owned progress file: "where do I continue from?" is computed - * from observable state (reprint's own `.import-state.json` cursor, + * from observable state (Reprint's own `pull/state.json` cursor, * whether the server is running, whether skipped files remain) and the * durable {@link SiteData} flags (`status`, `importComplete`), not from * a written stage cursor. @@ -268,8 +266,8 @@ class PullError extends LoggerError { * creates or deletes a site. There is no Studio-owned progress file: * the site is marked `status: 'pulling'` up front and every phase is * idempotent, so a crash (or `Ctrl-C`) just leaves the site `pulling` - * and re-running resumes by derivation — reprint resumes its own - * pipeline from `.import-state.json`, the server-start phase keys off + * and re-running resumes by derivation — Reprint resumes its own + * pipeline from `pull/state.json`, the server-start phase keys off * whether the process is already running, and the skipped-files phase * keys off `hasSkippedFiles`. A pull that errors or is killed lands the * site in `status: 'pull-failed'`; success returns it to `status: @@ -378,6 +376,10 @@ export async function runCommand( secret, verbose ); + const importMetadata = await readReprintImportMetadata( + studioMetadata.stateDirectory, + studioMetadata.rawDirectory + ); // Selective sync: apply `--only`/`--skip-*` flags, or prompt // interactively with the wp-content folder tree + database toggle. // A partial first-pull selection gets the core roots added and the @@ -386,7 +388,8 @@ export async function runCommand( // marked `pulling` so a cancel is a clean no-op. const selection = await applySelection( { session: studioMetadata, - isFirstPull: ! site.importComplete || ! hasLocalFilesIndex( studioMetadata.stateDirectory ), + sourceSite: importMetadata.sourceSite, + isFirstPull: ! site.importComplete || ! importMetadata.hasLocalIndex, cli: cliSelection, apiUrl, secret, @@ -421,8 +424,8 @@ export async function runCommand( // The pull pipeline runs as separate reprint commands (pull-files → // pull-db → flat-docroot → apply-runtime) so the selection can skip // the database step entirely; see runFullPull. Always re-invoked: - // every command is idempotent and reprint resumes its own pipeline - // from `.import-state.json`, so there is no Studio-side guard. + // every command is idempotent and Reprint resumes its own pipeline + // from `pull/state.json`, so there is no Studio-side guard. await runFullPull( SITE_RUNTIME_NATIVE_PHP, studioMetadata, @@ -430,6 +433,7 @@ export async function runCommand( secret, verbose, ! isRepull, + importMetadata, selection ); @@ -468,9 +472,13 @@ export async function runCommand( } runtimeStartOptions = nativeStartOptions; } else { - await ensureImportedSiteSqliteReady( studioMetadata.runtimeBlueprintPath ); + await ensureImportedSiteSqliteReady( + studioMetadata.runtimeBlueprintPath, + importMetadata.sourceSite.contentDirectory + ); runtimeStartOptions = await loadImportedRuntimeStartOptions( - studioMetadata.runtimeBlueprintPath + studioMetadata.runtimeBlueprintPath, + importMetadata.sourceSite.extraDirectories ); } @@ -544,15 +552,21 @@ export async function runCommand( // any remain — unless the user excluded the media library. Keyed off // observable state (`hasSkippedFiles`), not a stage cursor, so it // runs exactly when there's a tail outstanding. - if ( ! selection.skipUploads && hasSkippedFiles( studioMetadata.stateDirectory ) ) { - await downloadSkippedFiles( - getSiteRuntime( site ), - studioMetadata, - apiUrl, - secret, - verbose, - selection + if ( ! selection.skipUploads ) { + const completedPullMetadata = await readReprintImportMetadata( + studioMetadata.stateDirectory, + studioMetadata.rawDirectory ); + if ( completedPullMetadata.hasSkippedFiles ) { + await downloadSkippedFiles( + getSiteRuntime( site ), + studioMetadata, + apiUrl, + secret, + verbose, + selection + ); + } } // The pull is done: drop the selection sidecar so the next pull asks @@ -619,13 +633,14 @@ export async function runCommand( */ async function applySelection( params: { session: PullSession; + sourceSite: ReprintImportMetadata[ 'sourceSite' ]; isFirstPull: boolean; cli: CliSelectionOptions; apiUrl: string; secret: string; verbose: boolean; } ): Promise< PullSelection | null > { - const { session, isFirstPull, cli, apiUrl, secret, verbose } = params; + const { session, sourceSite, isFirstPull, cli, apiUrl, secret, verbose } = params; // A partial first-pull `--only` set must include the core roots. When // preflight did not expose them, fall back to a full file pull rather @@ -635,7 +650,12 @@ async function applySelection( params: { if ( ! isFirstPull || ( fileOnlyPaths.length === 0 && ! databaseOnly ) ) { return fileOnlyPaths; } - const coreRoots = getCoreRootsFromState( session.stateDirectory ); + const coreRoots = sourceSite.wordpressRoots.filter( + ( rootPath ) => + ! sourceSite.wordpressRoots.some( + ( other ) => other !== rootPath && other.startsWith( `${ rootPath }/` ) + ) + ); if ( coreRoots.length === 0 ) { console.log( __( @@ -672,7 +692,7 @@ async function applySelection( params: { if ( ! selection.skipDatabase ) { return; } - const contentDir = getContentDirFromState( session.stateDirectory ) ?? ''; + const contentDir = sourceSite.contentDirectory ?? ''; const candidates = [ path.join( session.sitePath, 'wp-content', 'database', '.ht.sqlite' ), path.join( @@ -700,7 +720,7 @@ async function applySelection( params: { skipUploads: !! cli.skipUploads, }; if ( cliOnly.length > 0 ) { - const contentDir = getContentDirFromState( session.stateDirectory ); + const contentDir = sourceSite.contentDirectory; if ( ! contentDir ) { throw new LoggerError( __( @@ -715,6 +735,7 @@ async function applySelection( params: { const { linkTargets } = await fetchReprintPullTree( { stateDirectory: session.stateDirectory, rawDirectory: session.rawDirectory, + contentDirectory: contentDir, apiUrl, secret, runtime: SITE_RUNTIME_NATIVE_PHP, @@ -755,6 +776,7 @@ async function applySelection( params: { const { tree, contentDir, linkTargets } = await fetchReprintPullTree( { stateDirectory: session.stateDirectory, rawDirectory: session.rawDirectory, + contentDirectory: sourceSite.contentDirectory, apiUrl, secret, runtime: SITE_RUNTIME_NATIVE_PHP, @@ -908,8 +930,11 @@ function getPullTechnicalDirectory( siteId: string ): string { * generated runtime prepend defines the database constants, so only the * table prefix and the wp-settings handoff are needed. */ -export function ensureScopedPullWpConfig( metadata: PullSession ): void { - const abspath = getAbspathFromState( metadata.stateDirectory ); +export function ensureScopedPullWpConfig( + metadata: PullSession, + sourceSite: ReprintImportMetadata[ 'sourceSite' ] +): void { + const abspath = sourceSite.wordpressAbsolutePath; if ( ! abspath ) { return; } @@ -933,7 +958,7 @@ export function ensureScopedPullWpConfig( metadata: PullSession ): void { } } - const tablePrefix = getTablePrefixFromState( metadata.stateDirectory ) ?? 'wp_'; + const tablePrefix = sourceSite.tablePrefix ?? 'wp_'; // Escape for a PHP single-quoted string: backslashes first, then single // quotes (both are the only special characters there). const escapedTablePrefix = tablePrefix.replace( /\\/g, '\\\\' ).replace( /'/g, "\\'" ); @@ -1021,9 +1046,10 @@ export async function runFullPull( secret: string, verbose: boolean, force: boolean, + importMetadata: ReprintImportMetadata, selection: PullSelection = {} ): Promise< void > { - const contentDir = getContentDirFromState( metadata.stateDirectory ); + const contentDir = importMetadata.sourceSite.contentDirectory; const sqlitePath = contentDir ? path.join( metadata.rawDirectory, @@ -1063,7 +1089,7 @@ export async function runFullPull( const rawEntries = fs.existsSync( metadata.rawDirectory ) ? fs.readdirSync( metadata.rawDirectory ) : []; - if ( rawEntries.length > 0 && ! hasLocalFilesIndex( metadata.stateDirectory ) ) { + if ( rawEntries.length > 0 && ! importMetadata.hasLocalIndex ) { fs.rmSync( metadata.rawDirectory, { recursive: true, force: true } ); fs.mkdirSync( metadata.rawDirectory, { recursive: true } ); resetEssentialFilesState( metadata.stateDirectory ); @@ -1124,7 +1150,7 @@ export async function runFullPull( // wp-content selection — reachable only through a symlink under the // core root. Synthesize a minimal one so WordPress can boot. if ( ( selection.fileOnlyPaths ?? [] ).length > 0 ) { - ensureScopedPullWpConfig( metadata ); + ensureScopedPullWpConfig( metadata, importMetadata.sourceSite ); } // Recreate the selected entries that are symlinks on the remote (wp.com diff --git a/apps/cli/commands/tests/pull-reprint.test.ts b/apps/cli/commands/tests/pull-reprint.test.ts index 906f0f1dfe..8002852547 100644 --- a/apps/cli/commands/tests/pull-reprint.test.ts +++ b/apps/cli/commands/tests/pull-reprint.test.ts @@ -18,6 +18,7 @@ import { resolveSourceSite, } from '../pull-reprint'; import type { SyncSite } from '@studio/common/types/sync'; +import type { ReprintImportMetadata } from 'cli/lib/pull/reprint-metadata'; // This file contains integration-style tests that reload the CLI module graph // and perform multiple atomic config writes. Those can exceed the default @@ -105,6 +106,33 @@ function readSeededCliConfig( homeDir: string ): { return JSON.parse( fs.readFileSync( path.join( homeDir, '.studio', 'cli.json' ), 'utf-8' ) ); } +function makeImportMetadata( + overrides: Partial< Omit< ReprintImportMetadata, 'sourceSite' > > = {}, + sourceSiteOverrides: Partial< ReprintImportMetadata[ 'sourceSite' ] > = {} +): ReprintImportMetadata { + const sourceSite = { + homeUrl: null, + siteUrl: null, + tablePrefix: null, + wordpressDatabaseCharset: null, + serverDatabaseCharset: null, + contentDirectory: null, + wordpressAbsolutePath: null, + wordpressRoots: [], + extraDirectories: [], + ...sourceSiteOverrides, + }; + + return { + hasCompletedOnce: false, + hasLocalIndex: true, + hasSkippedFiles: false, + pullStage: null, + ...overrides, + sourceSite, + }; +} + describe( 'CLI: studio pull-reprint helpers', () => { it( 'normalizes URLs by stripping hashes and trailing slashes', () => { expect( normalizeSiteUrl( 'https://example.com/foo//#section' ) ).toBe( @@ -141,13 +169,13 @@ describe( 'CLI: studio pull-reprint helpers', () => { ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); - fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.mkdirSync( path.join( stateDirectory, 'pull' ), { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); // State as pull-db's prepare_repull leaves it: the skipped_pending // flag pull-files set has been reset, though deferred files remain. fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), + path.join( stateDirectory, 'pull', 'state.json' ), JSON.stringify( { filter: 'essential-files', pull_pipeline: { started_by_command: 'pull-db', skipped_pending: false }, @@ -181,7 +209,7 @@ describe( 'CLI: studio pull-reprint helpers', () => { // The tail restored the flag its recovery keys on, preserving the // rest of the state file. const state = JSON.parse( - fs.readFileSync( path.join( stateDirectory, '.import-state.json' ), 'utf-8' ) + fs.readFileSync( path.join( stateDirectory, 'pull', 'state.json' ), 'utf-8' ) ); expect( state.pull_pipeline.skipped_pending ).toBe( true ); expect( state.pull_pipeline.started_by_command ).toBe( 'pull-db' ); @@ -199,22 +227,6 @@ describe( 'CLI: studio pull-reprint helpers', () => { fs.mkdirSync( rawAbspath, { recursive: true } ); fs.mkdirSync( path.join( rawDirectory, 'srv', 'htdocs' ), { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - database: { - wp: { - table_prefix: 'wp_abc123_', - paths_urls: { abspath: '/wordpress/core/7.0' }, - }, - }, - }, - }, - } ) - ); - // The WP Cloud layout a scoped pull recreates: parent-of-ABSPATH // wp-config.php is a symlink to a document-root file that was never // fetched (empty placeholder). @@ -225,7 +237,14 @@ describe( 'CLI: studio pull-reprint helpers', () => { ); const metadata = { stateDirectory, rawDirectory } as never; - ensureScopedPullWpConfig( metadata ); + const sourceSite = makeImportMetadata( + {}, + { + tablePrefix: 'wp_abc123_', + wordpressAbsolutePath: '/wordpress/core/7.0', + } + ).sourceSite; + ensureScopedPullWpConfig( metadata, sourceSite ); // Written through the symlink into its target, with the remote prefix. const written = fs.readFileSync( @@ -240,7 +259,7 @@ describe( 'CLI: studio pull-reprint helpers', () => { path.join( rawDirectory, 'srv', 'htdocs', 'wp-config.php' ), ' { fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( path.join( rawDirectory, 'wordpress', 'core', '7.0' ), { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - database: { - wp: { - table_prefix: "wp\\x'_", - paths_urls: { abspath: '/wordpress/core/7.0' }, - }, - }, - }, - }, - } ) + ensureScopedPullWpConfig( + { stateDirectory, rawDirectory } as never, + makeImportMetadata( + {}, + { + tablePrefix: "wp\\x'_", + wordpressAbsolutePath: '/wordpress/core/7.0', + } + ).sourceSite ); - ensureScopedPullWpConfig( { stateDirectory, rawDirectory } as never ); - // With no config at either candidate, it writes to the parent-of-ABSPATH // location wp-load falls back to. const written = fs.readFileSync( @@ -304,27 +316,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - // Preflight reported the remote site's wp-content path at - // database.wp.paths_urls.content_dir; the pull's db-apply stage targets - // an sqlite file under rawDirectory + that path so flat-docroot can - // symlink it into the flattened site. - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - database: { - wp: { - paths_urls: { - content_dir: '/srv/htdocs/wp-content', - }, - }, - }, - }, - }, - } ) - ); - const reprint = vi .spyOn( migrationClient, 'runReprintCommandUntilComplete' ) .mockResolvedValue( { stdout: '{"ok":true}', stderr: '', exitCode: 0 } ); @@ -351,7 +342,8 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - true + true, + makeImportMetadata( {}, { contentDirectory: '/srv/htdocs/wp-content' } ) ); // The pipeline runs as separate commands so the selection can skip steps. @@ -418,7 +410,7 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { } // No Studio-owned progress file is written: resume is by derivation - // (reprint's own `.import-state.json` + the site's `status`), so the + // (Reprint's own pull state + the site's `status`), so the // pull is always re-invoked rather than skipped via a stage cursor. expect( fs.existsSync( path.join( technicalSiteDirectory, 'pull.json' ) ) ).toBe( false ); @@ -433,10 +425,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { preflight: { data: {} } } ) - ); const reprint = vi .spyOn( migrationClient, 'runReprintCommandUntilComplete' ) @@ -457,6 +445,7 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'hmac-secret', false, false, + makeImportMetadata(), { skipDatabase: true } ); @@ -476,10 +465,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { preflight: { data: {} } } ) - ); const reprint = vi .spyOn( migrationClient, 'runReprintCommandUntilComplete' ) @@ -500,6 +485,7 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'hmac-secret', false, false, + makeImportMetadata(), { fileOnlyPaths: [ ':wp-plugins:', '/srv/htdocs/wp-content/plugins/akismet' ] } ); @@ -520,15 +506,15 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); - fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.mkdirSync( path.join( stateDirectory, 'pull' ), { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), + path.join( stateDirectory, 'pull', 'state.json' ), JSON.stringify( { command: 'files-pull', status: 'complete', preflight: { data: {} } } ) ); // Damage: raw holds leftovers but the local index is gone. fs.writeFileSync( path.join( rawDirectory, 'stale-blocker' ), 'junk' ); - fs.writeFileSync( path.join( stateDirectory, '.import-remote-index.jsonl' ), '{"p":1}\n' ); + fs.writeFileSync( path.join( stateDirectory, 'pull', 'remote-index.jsonl' ), '{"p":1}\n' ); const reprint = vi .spyOn( migrationClient, 'runReprintCommandUntilComplete' ) @@ -548,17 +534,18 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - true + true, + makeImportMetadata( { hasLocalIndex: false } ) ); // The scratch was wiped for a clean initial sync: raw is empty, the // stale derived indexes are gone, and only preflight survives in state. expect( fs.readdirSync( rawDirectory ) ).toEqual( [] ); - expect( fs.existsSync( path.join( stateDirectory, '.import-remote-index.jsonl' ) ) ).toBe( + expect( fs.existsSync( path.join( stateDirectory, 'pull', 'remote-index.jsonl' ) ) ).toBe( false ); expect( - JSON.parse( fs.readFileSync( path.join( stateDirectory, '.import-state.json' ), 'utf-8' ) ) + JSON.parse( fs.readFileSync( path.join( stateDirectory, 'pull', 'state.json' ), 'utf-8' ) ) ).toEqual( { preflight: { data: {} } } ); // The pull still ran, in default mode (no preserve-local escape hatch). const filesArgs = reprint.mock.calls[ 0 ][ 2 ] as string[]; @@ -574,13 +561,13 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { ); const stateDirectory = path.join( technicalSiteDirectory, 'state' ); const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); - fs.mkdirSync( stateDirectory, { recursive: true } ); + fs.mkdirSync( path.join( stateDirectory, 'pull' ), { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), + path.join( stateDirectory, 'pull', 'state.json' ), JSON.stringify( { preflight: { data: {} } } ) ); - fs.writeFileSync( path.join( stateDirectory, '.import-index.jsonl' ), '{"path":"a"}\n' ); + fs.writeFileSync( path.join( stateDirectory, 'pull', 'local-index.jsonl' ), '{"path":"a"}\n' ); fs.writeFileSync( path.join( rawDirectory, 'existing-file' ), 'keep me' ); vi.spyOn( migrationClient, 'runReprintCommandUntilComplete' ).mockResolvedValue( { @@ -603,13 +590,16 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - false + false, + makeImportMetadata() ); expect( fs.readFileSync( path.join( rawDirectory, 'existing-file' ), 'utf-8' ) ).toBe( 'keep me' ); - expect( fs.existsSync( path.join( stateDirectory, '.import-index.jsonl' ) ) ).toBe( true ); + expect( fs.existsSync( path.join( stateDirectory, 'pull', 'local-index.jsonl' ) ) ).toBe( + true + ); fs.rmSync( technicalSiteDirectory, { recursive: true, force: true } ); } ); @@ -622,10 +612,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { const rawDirectory = path.join( technicalSiteDirectory, 'raw' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { preflight: { data: {} } } ) - ); // Selection captured in the sidecar, but the pull must ignore it for now. fs.writeFileSync( path.join( stateDirectory, 'selection.json' ), @@ -654,7 +640,8 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - true + true, + makeImportMetadata() ); const passedArgs = reprint.mock.calls[ 0 ][ 2 ] as string[]; @@ -675,11 +662,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { preflight: { data: {} } } ) - ); - const reprint = vi .spyOn( migrationClient, 'runReprintCommandUntilComplete' ) .mockResolvedValue( { stdout: '{"ok":true}', stderr: '', exitCode: 0 } ); @@ -708,7 +690,8 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - false + false, + makeImportMetadata() ); // With no content dir from preflight, the sqlite target (on the @@ -741,10 +724,6 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { const runtimeDirectory = path.join( technicalSiteDirectory, 'runtime' ); fs.mkdirSync( stateDirectory, { recursive: true } ); fs.mkdirSync( rawDirectory, { recursive: true } ); - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { preflight: { data: {} } } ) - ); vi.spyOn( migrationClient, 'runReprintCommandUntilComplete' ).mockRejectedValue( new Error( 'reprint exited with code 1' ) @@ -773,7 +752,8 @@ describe( 'CLI: studio pull-reprint single pull phase', () => { 'https://example.com/?reprint-api', 'hmac-secret', false, - true + true, + makeImportMetadata() ) ).rejects.toThrow( 'reprint exited with code 1' ); @@ -1254,6 +1234,13 @@ describe( 'CLI: studio pull-reprint delta re-pull of a completed pull', () => { exitCode: 0, }; } + if ( args[ 0 ] === 'import-metadata' ) { + return { + stdout: JSON.stringify( makeImportMetadata() ), + stderr: '', + exitCode: 0, + }; + } if ( args[ 0 ] === 'pull-files' ) { expect( args ).toEqual( expect.arrayContaining( [ '--filter=essential-files' ] ) ); @@ -1271,6 +1258,7 @@ describe( 'CLI: studio pull-reprint delta re-pull of a completed pull', () => { expect( reprintSpy.mock.calls.map( ( call ) => call[ 2 ][ 0 ] ) ).toEqual( [ 'preflight', + 'import-metadata', 'pull-files', ] ); } ); @@ -1335,11 +1323,11 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { fs.mkdirSync( path.join( sitePath, 'wp-content', 'database' ), { recursive: true } ); fs.writeFileSync( path.join( sitePath, 'wp-content', 'database', '.ht.sqlite' ), 'local-db' ); - // Preflight data (content dir + core roots) as the real preflight - // stage would have persisted it into reprint's state file. - fs.mkdirSync( stateDirectory, { recursive: true } ); + // files-index still needs a throwaway copy of Reprint's preflight + // state until Reprint exposes an inspection command for that index. + fs.mkdirSync( path.join( stateDirectory, 'pull' ), { recursive: true } ); fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), + path.join( stateDirectory, 'pull', 'state.json' ), JSON.stringify( { preflight: { data: { @@ -1371,13 +1359,28 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { exitCode: 0, }; } + if ( args[ 0 ] === 'import-metadata' ) { + return { + stdout: JSON.stringify( + makeImportMetadata( + { hasLocalIndex: false }, + { + contentDirectory: '/srv/htdocs/wp-content', + wordpressRoots: [ '/wordpress/core/7.0', '/wordpress/core' ], + } + ) + ), + stderr: '', + exitCode: 0, + }; + } if ( args[ 0 ] === 'files-index' ) { const stateDirArg = args .find( ( arg ) => arg.startsWith( '--state-dir=' ) )! .slice( '--state-dir='.length ); const encode = ( value: string ) => Buffer.from( value, 'utf-8' ).toString( 'base64' ); fs.writeFileSync( - path.join( stateDirArg, '.import-remote-index.jsonl' ), + path.join( stateDirArg, 'pull', 'remote-index.jsonl' ), [ JSON.stringify( { path: encode( '/srv/htdocs/wp-content/themes/some-theme/style.css' ), @@ -1430,6 +1433,7 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { // remote symlinks for the selection. expect( reprintSpy.mock.calls.map( ( call ) => call[ 2 ][ 0 ] ) ).toEqual( [ 'preflight', + 'import-metadata', 'files-index', 'pull-files', 'flat-docroot', @@ -1437,7 +1441,7 @@ describe( 'CLI: studio pull-reprint first-pull selective sync', () => { // The include-list carries the live core root (the ancestor root // holding other core versions is dropped) plus the selection. - const filesArgs = reprintSpy.mock.calls[ 2 ][ 2 ] as string[]; + const filesArgs = reprintSpy.mock.calls[ 3 ][ 2 ] as string[]; expect( filesArgs ).toContain( '--only=/wordpress/core/7.0' ); expect( filesArgs ).not.toContain( '--only=/wordpress/core' ); expect( filesArgs ).toContain( '--only=/srv/htdocs/wp-content/themes' ); diff --git a/apps/cli/lib/pull/reprint-metadata.test.ts b/apps/cli/lib/pull/reprint-metadata.test.ts new file mode 100644 index 0000000000..8934f0eb62 --- /dev/null +++ b/apps/cli/lib/pull/reprint-metadata.test.ts @@ -0,0 +1,46 @@ +import { SITE_RUNTIME_NATIVE_PHP } from '@studio/common/lib/site-runtime'; +import { vi } from 'vitest'; +import { runReprintCommandUntilComplete } from 'cli/lib/pull/migration-client'; +import { readReprintImportMetadata } from 'cli/lib/pull/reprint-metadata'; + +vi.mock( 'cli/lib/pull/migration-client', () => ( { + runReprintCommandUntilComplete: vi.fn(), +} ) ); + +describe( 'reprint import metadata', () => { + it( 'reads metadata through the Reprint command', async () => { + const metadata = { + hasCompletedOnce: false, + hasLocalIndex: true, + hasSkippedFiles: false, + pullStage: 'files', + sourceSite: { + homeUrl: 'https://example.com', + siteUrl: 'https://example.com/wordpress', + tablePrefix: 'wp_', + wordpressDatabaseCharset: 'utf8mb4', + serverDatabaseCharset: 'utf8mb4', + contentDirectory: '/srv/htdocs/wp-content', + wordpressAbsolutePath: '/wordpress/core/7.0', + wordpressRoots: [ '/wordpress/core/7.0' ], + extraDirectories: [ '/scripts' ], + }, + }; + vi.mocked( runReprintCommandUntilComplete ).mockResolvedValue( { + stdout: JSON.stringify( metadata ), + stderr: '', + exitCode: 0, + } ); + + await expect( readReprintImportMetadata( '/pull/state', '/pull/raw' ) ).resolves.toEqual( + metadata + ); + expect( runReprintCommandUntilComplete ).toHaveBeenCalledWith( + '/pull/state', + '/pull/raw', + [ 'import-metadata', '--state-dir=/pull/state' ], + undefined, + { runtime: SITE_RUNTIME_NATIVE_PHP } + ); + } ); +} ); diff --git a/apps/cli/lib/pull/reprint-metadata.ts b/apps/cli/lib/pull/reprint-metadata.ts new file mode 100644 index 0000000000..2b0e732a2b --- /dev/null +++ b/apps/cli/lib/pull/reprint-metadata.ts @@ -0,0 +1,38 @@ +import { SITE_RUNTIME_NATIVE_PHP } from '@studio/common/lib/site-runtime'; +import { runReprintCommandUntilComplete } from 'cli/lib/pull/migration-client'; + +export interface ReprintImportMetadata { + hasCompletedOnce: boolean; + hasLocalIndex: boolean; + hasSkippedFiles: boolean; + pullStage: unknown; + sourceSite: { + homeUrl: string | null; + siteUrl: string | null; + tablePrefix: string | null; + wordpressDatabaseCharset: string | null; + serverDatabaseCharset: string | null; + contentDirectory: string | null; + wordpressAbsolutePath: string | null; + wordpressRoots: string[]; + extraDirectories: string[]; + }; +} + +/** + * Reads Reprint-owned import state through its public metadata command. + */ +export async function readReprintImportMetadata( + stateDirectory: string, + rawDirectory: string +): Promise< ReprintImportMetadata > { + const result = await runReprintCommandUntilComplete( + stateDirectory, + rawDirectory, + [ 'import-metadata', `--state-dir=${ stateDirectory }` ], + undefined, + { runtime: SITE_RUNTIME_NATIVE_PHP } + ); + + return JSON.parse( result.stdout ) as ReprintImportMetadata; +} diff --git a/apps/cli/lib/pull/reprint-selector.test.ts b/apps/cli/lib/pull/reprint-selector.test.ts index 1bf23daae3..736afe0efb 100644 --- a/apps/cli/lib/pull/reprint-selector.test.ts +++ b/apps/cli/lib/pull/reprint-selector.test.ts @@ -37,7 +37,7 @@ describe( 'buildReprintTreeFromIndex', () => { beforeEach( () => { dir = fs.mkdtempSync( path.join( os.tmpdir(), 'reprint-selector-' ) ); - indexPath = path.join( dir, '.import-remote-index.jsonl' ); + indexPath = path.join( dir, 'remote-index.jsonl' ); } ); afterEach( () => { diff --git a/apps/cli/lib/pull/reprint-selector.ts b/apps/cli/lib/pull/reprint-selector.ts index 0430f6e2ab..7596357620 100644 --- a/apps/cli/lib/pull/reprint-selector.ts +++ b/apps/cli/lib/pull/reprint-selector.ts @@ -18,7 +18,7 @@ import { SiteRuntime } from '@studio/common/lib/site-runtime'; import { shouldLimitDepth } from '@studio/common/lib/sync/tree-utils'; import { __ } from '@wordpress/i18n'; import { runReprintCommandUntilComplete } from 'cli/lib/pull/migration-client'; -import { getContentDirFromState, getRemoteIndexPath } from 'cli/lib/pull/reprint-state'; +import { getRemoteIndexPath, getReprintStatePath } from 'cli/lib/pull/reprint-state'; import { buildRootTree } from 'cli/lib/sync-selector'; import treeCheckbox from 'cli/lib/tree-checkbox'; import type { TreeNode } from 'cli/lib/tree-checkbox'; @@ -344,6 +344,7 @@ export function mapCheckedNodesToSelection( interface FetchReprintPullTreeParams { stateDirectory: string; rawDirectory: string; + contentDirectory: string | null; apiUrl: string; secret: string; runtime: SiteRuntime; @@ -352,9 +353,9 @@ interface FetchReprintPullTreeParams { /** * Run `reprint files-index` (requires a prior preflight) and build the - * selector tree from the resulting `.import-remote-index.jsonl`. + * selector tree from the resulting remote index. * - * The index runs against a throwaway copy of the state directory: the + * The index runs against a throwaway copy of the pull state: the * shared one must stay pristine, or the leftover remote index and * `files-index` checkpoint would derail the pull that follows — an * *initial* `pull-files` appends its scoped index to any existing file @@ -365,20 +366,20 @@ export async function fetchReprintPullTree( params: FetchReprintPullTreeParams ) contentDir: string | null; linkTargets: Record< string, string >; } > { - const { stateDirectory, rawDirectory, apiUrl, secret, runtime, verbose } = params; + const { stateDirectory, rawDirectory, contentDirectory, apiUrl, secret, runtime, verbose } = + params; - const contentDir = getContentDirFromState( stateDirectory ); - if ( ! contentDir ) { + if ( ! contentDirectory ) { return { tree: [], contentDir: null, linkTargets: {} }; } const indexStateDirectory = `${ stateDirectory.replace( /\/+$/, '' ) }-tree`; fs.rmSync( indexStateDirectory, { recursive: true, force: true } ); - fs.mkdirSync( indexStateDirectory, { recursive: true } ); + fs.mkdirSync( path.dirname( getReprintStatePath( indexStateDirectory ) ), { recursive: true } ); // files-index requires preflight data; hand it the session's copy. fs.copyFileSync( - path.join( stateDirectory, '.import-state.json' ), - path.join( indexStateDirectory, '.import-state.json' ) + getReprintStatePath( stateDirectory ), + getReprintStatePath( indexStateDirectory ) ); try { @@ -403,9 +404,9 @@ export async function fetchReprintPullTree( params: FetchReprintPullTreeParams ) const { tree, linkTargets } = await buildReprintTreeFromIndex( getRemoteIndexPath( indexStateDirectory ), - contentDir + contentDirectory ); - return { tree, contentDir, linkTargets }; + return { tree, contentDir: contentDirectory, linkTargets }; } finally { fs.rmSync( indexStateDirectory, { recursive: true, force: true } ); } diff --git a/apps/cli/lib/pull/reprint-state.test.ts b/apps/cli/lib/pull/reprint-state.test.ts index 0204ad4d1d..99da1f3c16 100644 --- a/apps/cli/lib/pull/reprint-state.test.ts +++ b/apps/cli/lib/pull/reprint-state.test.ts @@ -2,90 +2,28 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import { - getContentDirFromState, - getCoreRootsFromState, - hasLocalFilesIndex, - hasSkippedFiles, + getRemoteIndexPath, + getReprintStatePath, + markSkippedFilesPending, + resetEssentialFilesState, setSqliteRuntimeTarget, } from 'cli/lib/pull/reprint-state'; -describe( 'reprint state accessors', () => { - it( 'reads the remote wp-content path from preflight state', () => { - const stateDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-reprint-state-' ) ); - - try { - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - database: { - wp: { - paths_urls: { - content_dir: '/srv/htdocs/wp-content', - }, - }, - }, - }, - }, - } ) - ); - - expect( getContentDirFromState( stateDirectory ) ).toBe( '/srv/htdocs/wp-content' ); - } finally { - fs.rmSync( stateDirectory, { recursive: true, force: true } ); - } - } ); - - it( 'reads the WordPress core roots from preflight state, decoding base64-marked paths', () => { - const stateDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-reprint-state-' ) ); - - try { - expect( getCoreRootsFromState( stateDirectory ) ).toEqual( [] ); - - // reprint persists wp_detect root paths base64-encoded with a - // `base64:` marker; plain strings are the legacy fallback. - const encode = ( value: string ) => - `base64:${ Buffer.from( value, 'utf-8' ).toString( 'base64' ) }`; - fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - wp_detect: { - roots: [ - { path: encode( '/wordpress/core/7.0' ) }, - { path: '/wordpress/core' }, - { path: null }, - ], - }, - }, - }, - } ) - ); - - // The parent root is an ancestor of the live install and would - // pull every other core version alongside it — dropped. - expect( getCoreRootsFromState( stateDirectory ) ).toEqual( [ '/wordpress/core/7.0' ] ); - } finally { - fs.rmSync( stateDirectory, { recursive: true, force: true } ); - } - } ); - +describe( 'reprint state mutations', () => { it( 'records the sqlite runtime target when the database pull is skipped', () => { const stateDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-reprint-state-' ) ); + const statePath = getReprintStatePath( stateDirectory ); try { + fs.mkdirSync( path.dirname( statePath ), { recursive: true } ); fs.writeFileSync( - path.join( stateDirectory, '.import-state.json' ), + statePath, JSON.stringify( { filter: 'essential-files', apply: { target_db: null } } ) ); setSqliteRuntimeTarget( stateDirectory, '/pulls/raw/wp-content/database/.ht.sqlite' ); - const state = JSON.parse( - fs.readFileSync( path.join( stateDirectory, '.import-state.json' ), 'utf-8' ) - ); + const state = JSON.parse( fs.readFileSync( statePath, 'utf-8' ) ); expect( state.apply.target_engine ).toBe( 'sqlite' ); expect( state.apply.target_sqlite_path ).toBe( '/pulls/raw/wp-content/database/.ht.sqlite' ); expect( state.apply.target_db ).toBeNull(); @@ -95,35 +33,61 @@ describe( 'reprint state accessors', () => { } } ); - it( 'detects whether reprint left skipped files to download', () => { + it( 'marks skipped files pending without replacing the pull pipeline', () => { const stateDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-reprint-state-' ) ); - const skippedListPath = path.join( stateDirectory, '.import-download-list-skipped.jsonl' ); + const statePath = getReprintStatePath( stateDirectory ); try { - expect( hasSkippedFiles( stateDirectory ) ).toBe( false ); + fs.mkdirSync( path.dirname( statePath ), { recursive: true } ); + fs.writeFileSync( + statePath, + JSON.stringify( { pull_pipeline: { last_completed_stage: 'database' } } ) + ); - fs.writeFileSync( skippedListPath, '' ); - expect( hasSkippedFiles( stateDirectory ) ).toBe( false ); + markSkippedFilesPending( stateDirectory ); - fs.writeFileSync( skippedListPath, '{"path":"wp-content/cache/file"}\n' ); - expect( hasSkippedFiles( stateDirectory ) ).toBe( true ); + const state = JSON.parse( fs.readFileSync( statePath, 'utf-8' ) ); + expect( state.pull_pipeline ).toEqual( { + last_completed_stage: 'database', + skipped_pending: true, + } ); } finally { fs.rmSync( stateDirectory, { recursive: true, force: true } ); } } ); - it( 'reports a local files index only when present and non-empty', () => { + it( 'resets pull artifacts while preserving preflight', () => { const stateDirectory = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-reprint-state-' ) ); + const statePath = getReprintStatePath( stateDirectory ); + const artifactPaths = [ + path.join( stateDirectory, 'pull', 'local-index.jsonl' ), + getRemoteIndexPath( stateDirectory ), + path.join( stateDirectory, 'pull', 'fetch-list.jsonl' ), + path.join( stateDirectory, 'pull', 'skipped-fetch-list.jsonl' ), + path.join( stateDirectory, 'progress.json' ), + ]; try { - const localIndexPath = path.join( stateDirectory, '.import-index.jsonl' ); - expect( hasLocalFilesIndex( stateDirectory ) ).toBe( false ); - - fs.writeFileSync( localIndexPath, '' ); - expect( hasLocalFilesIndex( stateDirectory ) ).toBe( false ); - - fs.writeFileSync( localIndexPath, '{"path":"abc"}\n' ); - expect( hasLocalFilesIndex( stateDirectory ) ).toBe( true ); + fs.mkdirSync( path.dirname( statePath ), { recursive: true } ); + fs.writeFileSync( + statePath, + JSON.stringify( { + preflight: { data: { ok: true } }, + pull_pipeline: { last_completed_stage: 'files' }, + } ) + ); + for ( const artifactPath of artifactPaths ) { + fs.writeFileSync( artifactPath, 'scratch' ); + } + + resetEssentialFilesState( stateDirectory ); + + expect( JSON.parse( fs.readFileSync( statePath, 'utf-8' ) ) ).toEqual( { + preflight: { data: { ok: true } }, + } ); + for ( const artifactPath of artifactPaths ) { + expect( fs.existsSync( artifactPath ) ).toBe( false ); + } } finally { fs.rmSync( stateDirectory, { recursive: true, force: true } ); } diff --git a/apps/cli/lib/pull/reprint-state.ts b/apps/cli/lib/pull/reprint-state.ts index ad0076e751..e58f514a2f 100644 --- a/apps/cli/lib/pull/reprint-state.ts +++ b/apps/cli/lib/pull/reprint-state.ts @@ -1,58 +1,15 @@ /** - * Accessors for reprint.phar's on-disk state files. - * - * reprint.phar writes a JSON progress file (`.import-state.json`) and a - * skipped-download list under each pull's state directory. Studio reads - * only the fields it needs to wire the imported runtime and decide - * whether to invoke the follow-up skipped-files sync. - * - * This module is the single place where Studio couples to reprint's - * on-disk output. Reprint owns its own state machine. + * Paths needed for streaming indexes and the remaining mutations of + * Reprint-owned pull state. */ import fs from 'fs'; import path from 'path'; import { isErrnoException } from '@studio/common/lib/is-errno-exception'; -import { z } from 'zod'; -const STATE_FILE = '.import-state.json'; -const REMOTE_INDEX_FILE = '.import-remote-index.jsonl'; -const LOCAL_INDEX_FILE = '.import-index.jsonl'; -const SKIPPED_DOWNLOAD_LIST = '.import-download-list-skipped.jsonl'; +const STATE_FILE = path.join( 'pull', 'state.json' ); +const REMOTE_INDEX_FILE = path.join( 'pull', 'remote-index.jsonl' ); -const reprintStateSnapshotSchema = z.looseObject( { - preflight: z - .looseObject( { - data: z - .looseObject( { - database: z - .looseObject( { - wp: z - .looseObject( { - table_prefix: z.string().nullish(), - paths_urls: z - .looseObject( { - content_dir: z.string().nullish(), - abspath: z.string().nullish(), - } ) - .optional(), - } ) - .optional(), - } ) - .optional(), - wp_detect: z - .looseObject( { - roots: z.array( z.looseObject( { path: z.string().nullish() } ) ).optional(), - } ) - .optional(), - } ) - .optional(), - } ) - .optional(), -} ); - -type ReprintStateSnapshot = z.infer< typeof reprintStateSnapshotSchema >; - -function getReprintStatePath( stateDirectory: string ): string { +export function getReprintStatePath( stateDirectory: string ): string { return path.join( stateDirectory, STATE_FILE ); } @@ -60,7 +17,7 @@ export function getRemoteIndexPath( stateDirectory: string ): string { return path.join( stateDirectory, REMOTE_INDEX_FILE ); } -function readReprintState( stateDirectory: string ): ReprintStateSnapshot | null { +function readReprintState( stateDirectory: string ): Record< string, unknown > | null { let raw: string; try { raw = fs.readFileSync( getReprintStatePath( stateDirectory ), 'utf-8' ); @@ -71,83 +28,7 @@ function readReprintState( stateDirectory: string ): ReprintStateSnapshot | null throw error; } - const parsedJson = JSON.parse( raw ); - const parsed = reprintStateSnapshotSchema.safeParse( parsedJson ); - return parsed.success ? parsed.data : null; -} - -/** - * Read the remote WP_CONTENT_DIR path from the reprint state's - * preflight data. This is the absolute path on the source server - * (e.g. "/srv/htdocs/wp-content"), which mirrors the directory layout - * inside the raw fs-root. - */ -export function getContentDirFromState( stateDirectory: string ): string | null { - const state = readReprintState( stateDirectory ); - const contentDir = state?.preflight?.data?.database?.wp?.paths_urls?.content_dir; - return typeof contentDir === 'string' ? contentDir : null; -} - -/** - * Read the remote WordPress ABSPATH from the reprint state's preflight - * data (e.g. "/wordpress/core/7.0" on WP Cloud). - */ -export function getAbspathFromState( stateDirectory: string ): string | null { - const state = readReprintState( stateDirectory ); - const abspath = state?.preflight?.data?.database?.wp?.paths_urls?.abspath; - return typeof abspath === 'string' && abspath !== '' ? decodeStatePath( abspath ) : null; -} - -/** - * Read the remote site's database table prefix from the reprint state's - * preflight data. - */ -export function getTablePrefixFromState( stateDirectory: string ): string | null { - const state = readReprintState( stateDirectory ); - const tablePrefix = state?.preflight?.data?.database?.wp?.table_prefix; - return typeof tablePrefix === 'string' && tablePrefix !== '' ? tablePrefix : null; -} - -/** - * reprint base64-encodes some path fields when persisting its state - * (`wp_detect` roots among them), marked with a `base64:` prefix; plain - * values pass through for backward compatibility. - */ -function decodeStatePath( value: string ): string { - const prefix = 'base64:'; - if ( ! value.startsWith( prefix ) ) { - return value; - } - return Buffer.from( value.slice( prefix.length ), 'base64' ).toString( 'utf-8' ); -} - -/** - * Read the WordPress core roots the remote preflight detected (e.g. - * `/wordpress/core/7.0` on WP Cloud). A `--only`-scoped pull must pass - * these explicitly: `--only` *replaces* reprint's default export roots, - * so without them a partial selection would drop WordPress core and the - * site could not be assembled. - * - * Roots that are ancestors of another root are dropped: on WP Cloud the - * detected roots are `/wordpress/core/7.0` (the live install) and - * `/wordpress/core` (its parent, which also holds every *other* core - * version) — pulling the parent would download them all. - */ -export function getCoreRootsFromState( stateDirectory: string ): string[] { - const state = readReprintState( stateDirectory ); - const roots = ( state?.preflight?.data?.wp_detect?.roots ?? [] ) - .map( ( root ) => root.path ) - .filter( ( rootPath ): rootPath is string => typeof rootPath === 'string' && rootPath !== '' ) - .map( decodeStatePath ); - return roots.filter( - ( rootPath ) => - ! roots.some( ( other ) => other !== rootPath && other.startsWith( `${ rootPath }/` ) ) - ); -} - -export function hasSkippedFiles( stateDirectory: string ): boolean { - const skippedListPath = path.join( stateDirectory, SKIPPED_DOWNLOAD_LIST ); - return fs.existsSync( skippedListPath ) && fs.statSync( skippedListPath ).size > 0; + return JSON.parse( raw ) as Record< string, unknown >; } /** @@ -203,17 +84,6 @@ function mergeReprintStateFields( stateDirectory: string, mutate: ( state: any ) fs.writeFileSync( statePath, JSON.stringify( state, null, 2 ) + '\n' ); } -/** - * True when reprint's local file index says a file sync completed, so the - * raw fs-root holds the site (WordPress core included) and a - * `--only`-restricted delta pull is safe. Unlike the durable - * `site.importComplete` flag, this reflects the actual scratch contents. - */ -export function hasLocalFilesIndex( stateDirectory: string ): boolean { - const localIndexPath = path.join( stateDirectory, LOCAL_INDEX_FILE ); - return fs.existsSync( localIndexPath ) && fs.statSync( localIndexPath ).size > 0; -} - /** * Wipe the reprint state + derived indexes so the next run starts an * essential-files sync from scratch — but preserve preflight data so @@ -224,11 +94,11 @@ export function resetEssentialFilesState( stateDirectory: string ): void { const preflight = existingState?.preflight; for ( const fileName of [ - '.import-index.jsonl', - '.import-remote-index.jsonl', - '.import-download-list.jsonl', - '.import-download-list-skipped.jsonl', - '.import-status.json', + path.join( 'pull', 'local-index.jsonl' ), + path.join( 'pull', 'remote-index.jsonl' ), + path.join( 'pull', 'fetch-list.jsonl' ), + path.join( 'pull', 'skipped-fetch-list.jsonl' ), + 'progress.json', ] ) { fs.rmSync( path.join( stateDirectory, fileName ), { force: true } ); } diff --git a/apps/cli/lib/pull/runtime-start-options.test.ts b/apps/cli/lib/pull/runtime-start-options.test.ts index c578f5a110..0efd29cd13 100644 --- a/apps/cli/lib/pull/runtime-start-options.test.ts +++ b/apps/cli/lib/pull/runtime-start-options.test.ts @@ -4,7 +4,7 @@ import path from 'path'; import { vi } from 'vitest'; import { ensureImportedSiteSqliteReady, - getExtraDirectoryMountsFromImporterState, + getExtraDirectoryMounts, loadImportedRuntimeStartOptions, loadImportedRuntimeStartOptionsNative, loadRuntimeBlueprint, @@ -93,7 +93,7 @@ if (!defined('STREAMING_SITE_MIGRATION_REMOTE_UPLOAD_PROXY_STATE_FILE')) { } ) as typeof fs.readFileSync ); await expect( - loadImportedRuntimeStartOptions( '/test/runtime/blueprint.json' ) + loadImportedRuntimeStartOptions( '/test/runtime/blueprint.json', [] ) ).resolves.toEqual( { blueprint: { landingPage: '/', @@ -130,72 +130,34 @@ if (!defined('STREAMING_SITE_MIGRATION_REMOTE_UPLOAD_PROXY_STATE_FILE')) { } ); } ); - describe( 'getExtraDirectoryMountsFromImporterState', () => { - it( 'rejects path traversal in auto_prepend_file', () => { + describe( 'getExtraDirectoryMounts', () => { + it( 'rejects paths that escape the raw directory', () => { const importRoot = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-import-traversal-' ) ); const runtimeDir = path.join( importRoot, 'runtime' ); - const stateDir = path.join( importRoot, 'state' ); const rawDir = path.join( importRoot, 'raw' ); try { fs.mkdirSync( runtimeDir, { recursive: true } ); - fs.mkdirSync( stateDir, { recursive: true } ); fs.mkdirSync( rawDir, { recursive: true } ); - // A malicious server could set auto_prepend_file to a path - // with ../ segments to escape the raw directory. - fs.writeFileSync( - path.join( stateDir, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - runtime: { - ini_get_all: { - auto_prepend_file: '/../../../etc/passwd', - }, - }, - }, - }, - } ) - ); - - // Even if the traversed path exists on the host, it must be - // rejected because it escapes the raw/ directory. - const result = getExtraDirectoryMountsFromImporterState( runtimeDir ); + const result = getExtraDirectoryMounts( runtimeDir, [ '/../../../etc' ] ); expect( result ).toEqual( [] ); } finally { fs.rmSync( importRoot, { recursive: true, force: true } ); } } ); - it( 'allows a valid auto_prepend_file within the raw directory', () => { + it( 'mounts a reported extra directory from the raw tree', () => { const importRoot = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-import-valid-' ) ); const runtimeDir = path.join( importRoot, 'runtime' ); - const stateDir = path.join( importRoot, 'state' ); const rawDir = path.join( importRoot, 'raw' ); const scriptsDir = path.join( rawDir, 'scripts' ); try { fs.mkdirSync( runtimeDir, { recursive: true } ); - fs.mkdirSync( stateDir, { recursive: true } ); fs.mkdirSync( scriptsDir, { recursive: true } ); - fs.writeFileSync( - path.join( stateDir, '.import-state.json' ), - JSON.stringify( { - preflight: { - data: { - runtime: { - ini_get_all: { - auto_prepend_file: '/scripts/env.php', - }, - }, - }, - }, - } ) - ); - - const result = getExtraDirectoryMountsFromImporterState( runtimeDir ); + const result = getExtraDirectoryMounts( runtimeDir, [ '/scripts' ] ); expect( result ).toEqual( [ { hostPath: scriptsDir, vfsPath: '/scripts' } ] ); } finally { fs.rmSync( importRoot, { recursive: true, force: true } ); @@ -205,7 +167,7 @@ if (!defined('STREAMING_SITE_MIGRATION_REMOTE_UPLOAD_PROXY_STATE_FILE')) { it( 'ensures sqlite integration is installed for imported sites', async () => { const importRoot = fs.mkdtempSync( path.join( os.tmpdir(), 'studio-import-site-' ) ); - const rawWpContent = path.join( importRoot, 'raw', 'wp-content' ); + const rawWpContent = path.join( importRoot, 'raw', 'srv', 'htdocs', 'wp-content' ); const databaseDirectory = path.join( rawWpContent, 'database' ); const runtimeDir = path.join( importRoot, 'runtime' ); const runtimeBlueprintPath = path.join( runtimeDir, 'blueprint.json' ); @@ -221,11 +183,13 @@ if (!defined('STREAMING_SITE_MIGRATION_REMOTE_UPLOAD_PROXY_STATE_FILE')) { .spyOn( sqliteIntegrationModule, 'installSqliteIntegration' ) .mockResolvedValue( undefined ); - await expect( ensureImportedSiteSqliteReady( runtimeBlueprintPath ) ).resolves.toBe( - sqlitePath - ); + await expect( + ensureImportedSiteSqliteReady( runtimeBlueprintPath, '/srv/htdocs/wp-content' ) + ).resolves.toBe( sqlitePath ); // Receives the parent of the resolved wp-content in the raw tree. - expect( installSqliteIntegrationMock ).toHaveBeenCalledWith( path.join( importRoot, 'raw' ) ); + expect( installSqliteIntegrationMock ).toHaveBeenCalledWith( + path.join( importRoot, 'raw', 'srv', 'htdocs' ) + ); } finally { fs.rmSync( importRoot, { recursive: true, force: true } ); } diff --git a/apps/cli/lib/pull/runtime-start-options.ts b/apps/cli/lib/pull/runtime-start-options.ts index a9c32d561f..211c334fb9 100644 --- a/apps/cli/lib/pull/runtime-start-options.ts +++ b/apps/cli/lib/pull/runtime-start-options.ts @@ -8,7 +8,6 @@ import path from 'path'; import { loadNodeRuntime } from '@php-wasm/node'; import { PHP, ProcessIdAllocator } from '@php-wasm/universal'; import { LatestSupportedPHPVersion } from '@studio/common/types/php-versions'; -import { getContentDirFromState } from 'cli/lib/pull/reprint-state'; import { installSqliteIntegration } from 'cli/lib/sqlite-integration'; import { LoggerError } from 'cli/logger'; import type { Blueprint } from '@wp-playground/blueprints'; @@ -136,17 +135,18 @@ function mergeBlueprintConstants( * The flattened site directory may contain a wp-content symlink that uses * VFS-relative paths (e.g. ../fs-root/srv/htdocs/wp-content) which don't * resolve on the host filesystem. Instead of traversing that symlink, we - * derive the real path from the import's raw directory and the content_dir - * recorded in the reprint preflight state. + * derive the real path from the import's raw directory and the content + * directory reported by Reprint. */ -function resolveImportedWpContentPath( runtimeBlueprintPath: string ): string { +function resolveImportedWpContentPath( + runtimeBlueprintPath: string, + contentDirectory: string | null +): string { const importRoot = path.dirname( path.dirname( runtimeBlueprintPath ) ); const rawDirectory = path.join( importRoot, 'raw' ); - const stateDirectory = path.join( importRoot, 'state' ); - const contentDir = getContentDirFromState( stateDirectory ); - if ( contentDir ) { - const resolved = path.join( rawDirectory, contentDir.replace( /^\//, '' ) ); + if ( contentDirectory ) { + const resolved = path.join( rawDirectory, contentDirectory.replace( /^\//, '' ) ); if ( fs.existsSync( resolved ) ) { return resolved; } @@ -156,9 +156,10 @@ function resolveImportedWpContentPath( runtimeBlueprintPath: string ): string { } export async function ensureImportedSiteSqliteReady( - runtimeBlueprintPath: string + runtimeBlueprintPath: string, + contentDirectory: string | null ): Promise< string > { - const wpContentPath = resolveImportedWpContentPath( runtimeBlueprintPath ); + const wpContentPath = resolveImportedWpContentPath( runtimeBlueprintPath, contentDirectory ); const databaseDirectory = path.join( wpContentPath, 'database' ); const sqlitePath = path.join( databaseDirectory, '.ht.sqlite' ); const sqlitePhpPath = path.join( databaseDirectory, '.ht.sqlite.php' ); @@ -216,7 +217,8 @@ export function loadRuntimeBlueprint( runtimeBlueprintPath: string ): Blueprint } export async function loadImportedRuntimeStartOptions( - runtimeBlueprintPath: string + runtimeBlueprintPath: string, + extraDirectories: string[] ): Promise< StartServerOptions > { const runtimeDirectory = path.dirname( runtimeBlueprintPath ); const startJsonPath = path.join( runtimeDirectory, 'start.json' ); @@ -304,10 +306,10 @@ export async function loadImportedRuntimeStartOptions( // On wp.com Atomic, auto_prepend_file points to /scripts/env.php — // a directory outside the WordPress roots that the importer's - // apply-runtime doesn't mount. Detect it from the importer state - // and add the mount so absolute paths like + // apply-runtime doesn't mount. Add Reprint's extra directories so + // absolute paths like // require_once('/scripts/object-cache.memcache.php') resolve. - const extraDirMounts = getExtraDirectoryMountsFromImporterState( runtimeDirectory ); + const extraDirMounts = getExtraDirectoryMounts( runtimeDirectory, extraDirectories ); if ( extraDirMounts.length > 0 ) { startOptions.mountsBeforeInstall = [ ...( startOptions.mountsBeforeInstall ?? [] ), @@ -346,61 +348,31 @@ export async function loadImportedRuntimeStartOptions( } /** - * Reads the importer state to find directories that need mounting at their - * original absolute paths (e.g. /scripts from auto_prepend_file). These - * directories are downloaded into the raw/ tree but aren't in the flattened - * site path or the generated start.json. + * Mount directories Reprint reports at their original absolute paths. + * These directories are downloaded into the raw/ tree but aren't in the + * flattened site path or the generated start.json. */ -export function getExtraDirectoryMountsFromImporterState( - runtimeDirectory: string +export function getExtraDirectoryMounts( + runtimeDirectory: string, + extraDirectories: string[] ): Array< { hostPath: string; vfsPath: string } > { const importRoot = path.dirname( runtimeDirectory ); - const statePath = path.join( importRoot, 'state', '.import-state.json' ); const rawDirectory = path.join( importRoot, 'raw' ); - let raw: string; - try { - raw = fs.readFileSync( statePath, 'utf-8' ); - } catch { - // State file may not exist yet. - return []; - } - - let state: Record< string, unknown >; - try { - state = JSON.parse( raw ) as Record< string, unknown >; - } catch { - // Malformed state file — skip extra mounts rather than crashing. - return []; - } - - const preflight = ( state.preflight as Record< string, unknown > | undefined )?.data as - | Record< string, unknown > - | undefined; - const runtime = preflight?.runtime as Record< string, unknown > | undefined; - const iniGetAll = runtime?.ini_get_all as Record< string, unknown > | undefined; - const autoPrepend = iniGetAll?.auto_prepend_file; - - if ( typeof autoPrepend !== 'string' || ! autoPrepend.startsWith( '/' ) ) { - return []; - } - - const dir = path.posix.dirname( autoPrepend ); - if ( ! dir || dir === '/' ) { - return []; - } - - // The raw download preserves full remote paths, so /scripts - // becomes raw/scripts on the host filesystem. - const hostPath = path.join( rawDirectory, dir.slice( 1 ) ); - const resolvedHostPath = path.resolve( hostPath ); const resolvedRawDirectory = path.resolve( rawDirectory ); - if ( ! resolvedHostPath.startsWith( resolvedRawDirectory + path.sep ) ) { - return []; - } - if ( ! fs.existsSync( hostPath ) ) { - return []; - } + return extraDirectories.flatMap( ( directory ) => { + // The raw download preserves full remote paths, so /scripts + // becomes raw/scripts on the host filesystem. + const hostPath = path.join( rawDirectory, directory.replace( /^\/+/, '' ) ); + const resolvedHostPath = path.resolve( hostPath ); + if ( + directory === '/' || + ! resolvedHostPath.startsWith( resolvedRawDirectory + path.sep ) || + ! fs.existsSync( hostPath ) + ) { + return []; + } - return [ { hostPath, vfsPath: dir } ]; + return [ { hostPath, vfsPath: directory } ]; + } ); }