Skip to content

fix: avoid partial-clone hydration during worktree admission #1003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 1 commit into from
Aug 5, 2026
+92 −9
Merged

fix: avoid partial-clone hydration during worktree admission #1003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 55 additions & 8 deletions inc/Workspace/WorktreeBootstrapper.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ final class WorktreeBootstrapper {
'composer_root_inodes' => 250000,
);

/** Blobless trees omit blob sizes; reserve 64 KiB for every tracked tree entry. */
private const BLOBLESS_TRACKED_ENTRY_BYTES = 65536;

private const DEFAULT_COMMAND_TIMEOUT_SECONDS = 600;
private const DEFAULT_TOTAL_TIMEOUT_SECONDS = 1800;
private static ?float $bootstrap_deadline = null;
Expand Down Expand Up @@ -285,10 +288,12 @@ public static function demand_plan_for_target( string $repo_path, string $target
return new \WP_Error('worktree_target_ref_invalid', sprintf('Could not resolve target ref "%s" before capacity admission.', $target_ref), array( 'status' => 400 ));
}

$blobless_partial_clone = self::is_blobless_partial_clone($repo_path);
$tree_command = 'ls-tree -r -t ' . ( $blobless_partial_clone ? '' : '-l ' ) . '-z --full-tree ' . escapeshellarg($commit);
if ( null !== $runner ) {
$tree_output = $runner($repo_path, $commit);
} else {
$tree = GitRunner::run($repo_path, 'ls-tree -r -t -l -z --full-tree ' . escapeshellarg($commit), 30);
$tree = GitRunner::run($repo_path, $tree_command, 30);
if ( $tree instanceof \WP_Error ) {
return $tree;
}
Expand All @@ -298,7 +303,11 @@ public static function demand_plan_for_target( string $repo_path, string $target
return new \WP_Error('worktree_target_tree_unavailable', 'Target tree inspection did not return parseable output.', array( 'status' => 500 ));
}

$tree_plan = self::parse_target_tree($tree_output);
$tree_plan = self::parse_target_tree($tree_output);
$blobless_entry_bytes = self::blobless_tracked_entry_bytes($repo_path);
$tracked_bytes = $blobless_partial_clone
? $tree_plan['tracked_entries'] * $blobless_entry_bytes
: $tree_plan['tracked_bytes'];
$defaults = self::filtered_demand_defaults($tree_plan['detected'], $repo_path, $bootstrap);
$counts = array(
'tracked_entries' => $tree_plan['tracked_entries'],
Expand All @@ -308,12 +317,14 @@ public static function demand_plan_for_target( string $repo_path, string $target
);

return array(
'bytes' => $tree_plan['tracked_bytes'] + $defaults['git_bytes'] + ( $counts['submodules'] * $defaults['submodule_bytes'] ) + ( $counts['package_roots'] * $defaults['package_root_bytes'] ) + ( $counts['composer_roots'] * $defaults['composer_root_bytes'] ),
'bytes' => $tracked_bytes + $defaults['git_bytes'] + ( $counts['submodules'] * $defaults['submodule_bytes'] ) + ( $counts['package_roots'] * $defaults['package_root_bytes'] ) + ( $counts['composer_roots'] * $defaults['composer_root_bytes'] ),
'inodes' => $counts['tracked_entries'] + $defaults['git_inodes'] + ( $counts['submodules'] * $defaults['submodule_inodes'] ) + ( $counts['package_roots'] * $defaults['package_root_inodes'] ) + ( $counts['composer_roots'] * $defaults['composer_root_inodes'] ),
'source' => 'target_git_tree_conservative',
'target_ref' => $target_ref,
'target_commit' => $commit,
'tracked_bytes' => $tree_plan['tracked_bytes'],
'tracked_bytes' => $tracked_bytes,
'tracked_bytes_source' => $blobless_partial_clone ? 'conservative_blobless_entry_estimate' : 'exact_git_blob_sizes',
'tracked_bytes_per_entry' => $blobless_partial_clone ? $blobless_entry_bytes : null,
'git_safety_margin' => array(
'bytes' => $defaults['git_bytes'],
'inodes' => $defaults['git_inodes'],
Expand All @@ -322,10 +333,46 @@ public static function demand_plan_for_target( string $repo_path, string $target
'detected' => $tree_plan['detected'],
'counts' => $counts,
'allowances' => $defaults,
'fallback_semantics' => 'tracked target entries and bytes are measured from Git; dependency installs use conservative allowances',
'fallback_semantics' => $blobless_partial_clone
? 'tracked target entries are measured from Git metadata; blobless partial clones reserve a conservative 64 KiB per tracked entry because exact blob sizes are unavailable; dependency installs use conservative allowances'
: 'tracked target entries and bytes are measured from Git; dependency installs use conservative allowances',
);
}

/** Whether Git config declares a promisor remote with a blob:none filter. */
private static function is_blobless_partial_clone( string $repo_path ): bool {
$config = GitRunner::probe_output($repo_path, 'config --get-regexp ' . escapeshellarg('^remote\..*\.(promisor|partialclonefilter)$'));
if ( null === $config ) {
return false;
}

$remotes = array();
foreach ( preg_split('/\r?\n/', $config) as $line ) {
if ( 1 !== preg_match('/^remote\.([A-Za-z0-9._-]+)\.(promisor|partialclonefilter)\s+(.+)$/D', $line, $matches) ) {
continue;
}
$remotes[ $matches[1] ][ $matches[2] ] = strtolower(trim($matches[3]));
}

foreach ( $remotes as $remote ) {
if ( in_array($remote['promisor'] ?? '', array( 'true', 'yes', 'on', '1' ), true) && 'blob:none' === ( $remote['partialclonefilter'] ?? '' ) ) {
return true;
}
}

return false;
}

/** Resolve the conservative per-entry estimate used when blob sizes are absent. */
private static function blobless_tracked_entry_bytes( string $repo_path ): int {
$bytes = self::BLOBLESS_TRACKED_ENTRY_BYTES;
if ( function_exists('apply_filters') ) {
$bytes = (int) apply_filters('datamachine_code_worktree_blobless_tracked_entry_bytes', $bytes, $repo_path);
}

return max(1, $bytes);
}

/** Remove demand already materialized by `git worktree add` or rebase. */
public static function remaining_demand_after_materialization( array $plan ): array {
$plan['bytes'] = max(0, (int) ( $plan['bytes'] ?? 0 ) - (int) ( $plan['tracked_bytes'] ?? 0 ));
Expand All @@ -334,7 +381,7 @@ public static function remaining_demand_after_materialization( array $plan ): ar
return $plan;
}

/** Parse bounded NUL-delimited `git ls-tree -r -t -l` output. */
/** Parse bounded NUL-delimited `git ls-tree -r -t` output, with optional blob sizes. */
public static function parse_target_tree( string $output ): array {
$tracked_entries = 0;
$tracked_bytes = 0;
Expand All @@ -344,12 +391,12 @@ public static function parse_target_tree( string $output ): array {
$lockfiles = array( 'pnpm-lock.yaml', 'bun.lockb', 'bun.lock', 'yarn.lock', 'package-lock.json' );

foreach ( explode("\0", $output) as $record ) {
if ( '' === $record || 1 !== preg_match('/^(\d{6})\s+(blob|tree|commit)\s+[0-9a-f]+\s+(-|\d+)\t(.*)$/sD', $record, $matches) ) {
if ( '' === $record || 1 !== preg_match('/^(\d{6})\s+(blob|tree|commit)\s+[0-9a-f]+(?:\s+(-|\d+))?\t(.*)$/sD', $record, $matches) ) {
continue;
}
++$tracked_entries;
$path = $matches[4];
if ( 'blob' === $matches[2] && is_numeric($matches[3]) ) {
if ( 'blob' === $matches[2] && isset($matches[3]) && is_numeric($matches[3]) ) {
$tracked_bytes += max(0, (int) $matches[3]);
}
if ( '160000' === $matches[1] ) {
Expand Down
38 changes: 37 additions & 1 deletion tests/worktree-bootstrap-demand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ function bootstrap_demand_assert( bool $condition, string $message ): void {

$fixture = sys_get_temp_dir() . '/dmc-bootstrap-demand-' . bin2hex(random_bytes(6));
$bin = $fixture . '/bin';
$git_log = $fixture . '/git.log';
$repo = $fixture . '/repo';
$rebase_worktree = $fixture . '/rebased';
mkdir($fixture . '/frontend', 0777, true);
Expand Down Expand Up @@ -67,19 +68,53 @@ function bootstrap_demand_assert( bool $condition, string $message ): void {
exec('git -C ' . escapeshellarg($repo) . ' checkout -qb target-tree');
mkdir($repo . '/frontend', 0777, true);
file_put_contents($repo . '/frontend/package-lock.json', '{}');
file_put_contents($repo . '/composer.lock', '{}');
for ( $index = 0; $index < 300; ++$index ) {
file_put_contents($repo . '/frontend/tracked-' . $index . '.txt', 'x');
}
exec('git -C ' . escapeshellarg($repo) . ' add frontend && git -C ' . escapeshellarg($repo) . ' commit -qm target');
$submodule_commit = trim((string) shell_exec('git -C ' . escapeshellarg($repo) . ' rev-parse HEAD'));
exec('git -C ' . escapeshellarg($repo) . ' add frontend composer.lock && git -C ' . escapeshellarg($repo) . ' update-index --add --cacheinfo 160000,' . $submodule_commit . ',dependency && git -C ' . escapeshellarg($repo) . ' commit -qm target');
exec('git -C ' . escapeshellarg($repo) . ' checkout -q ' . escapeshellarg($primary_branch));

$primary_tree = WorktreeBootstrapper::demand_plan_for_target($repo, $primary_branch, true);
$target_tree = WorktreeBootstrapper::demand_plan_for_target($repo, 'target-tree', true);
bootstrap_demand_assert(! is_wp_error($primary_tree) && ! is_wp_error($target_tree), 'Both primary and differing target refs must resolve before admission.');
bootstrap_demand_assert(0 === $primary_tree['counts']['package_roots'] && 1 === $target_tree['counts']['package_roots'], 'Dependency demand must come from the target tree rather than the current primary checkout.');
bootstrap_demand_assert(1 === $target_tree['counts']['composer_roots'] && 1 === $target_tree['counts']['submodules'], 'Target-tree planning must detect Composer and submodule capacity from Git metadata.');
bootstrap_demand_assert($target_tree['counts']['tracked_entries'] > 256, 'Target-tree tracked entry demand fixture must exceed the old fixed Git reserve.');
bootstrap_demand_assert($target_tree['inodes'] >= $target_tree['counts']['tracked_entries'] + $target_tree['git_safety_margin']['inodes'], 'Admission must reserve tracked materialization plus an explicit Git lock margin.');
bootstrap_demand_assert($target_tree['target_commit'] !== $primary_tree['target_commit'], 'Differing refs must retain their exact resolved commits in demand evidence.');
bootstrap_demand_assert('exact_git_blob_sizes' === $target_tree['tracked_bytes_source'], 'Full clones must retain exact tracked-byte accounting.');

$system_git = trim((string) shell_exec('command -v git'));
file_put_contents($bin . '/git', "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " . escapeshellarg($git_log) . "\nexec " . escapeshellarg($system_git) . " \"$@\"\n");
chmod($bin . '/git', 0700);
$original_path = (string) getenv('PATH');
putenv('PATH=' . $bin . PATH_SEPARATOR . $original_path);
$logged_full_tree = WorktreeBootstrapper::demand_plan_for_target($repo, 'target-tree', true);
putenv('PATH=' . $original_path);
bootstrap_demand_assert(! is_wp_error($logged_full_tree), 'Full-clone command fixture must remain planable.');
$full_commands = (string) file_get_contents($git_log);
bootstrap_demand_assert(str_contains($full_commands, 'ls-tree -r -t -l -z --full-tree ' . $logged_full_tree['target_commit']), 'Full clones must inspect target trees with exact blob sizes.');

exec('git -C ' . escapeshellarg($repo) . ' config remote.origin.promisor true');
exec('git -C ' . escapeshellarg($repo) . ' config remote.origin.partialclonefilter blob:none');
file_put_contents($git_log, '');
putenv('PATH=' . $bin . PATH_SEPARATOR . $original_path);
$blobless_tree = WorktreeBootstrapper::demand_plan_for_target($repo, 'target-tree', true);
putenv('PATH=' . $original_path);
bootstrap_demand_assert(! is_wp_error($blobless_tree), 'Blobless target-tree command fixture must remain planable.');
$blobless_commands = (string) file_get_contents($git_log);
bootstrap_demand_assert(str_contains($blobless_commands, 'config --get-regexp ^remote\\..*\\.(promisor|partialclonefilter)$'), 'Blobless detection must inspect only promisor filter metadata.');
bootstrap_demand_assert(str_contains($blobless_commands, 'ls-tree -r -t -z --full-tree ' . $blobless_tree['target_commit']) && ! str_contains($blobless_commands, 'ls-tree -r -t -l -z --full-tree '), 'Blobless clones must inspect target metadata without requesting blob sizes.');
bootstrap_demand_assert('conservative_blobless_entry_estimate' === $blobless_tree['tracked_bytes_source'], 'Blobless plans must expose their conservative tracked-byte contract.');
bootstrap_demand_assert($blobless_tree['tracked_bytes'] === $blobless_tree['counts']['tracked_entries'] * 65536, 'Blobless plans must reserve the documented conservative byte estimate for every tracked entry.');
bootstrap_demand_assert(65536 === $blobless_tree['tracked_bytes_per_entry'], 'Blobless plans must expose the per-entry estimate used for capacity review.');
bootstrap_demand_assert($blobless_tree['counts']['package_roots'] === $target_tree['counts']['package_roots'] && $blobless_tree['counts']['composer_roots'] === $target_tree['counts']['composer_roots'] && $blobless_tree['counts']['submodules'] === $target_tree['counts']['submodules'], 'Metadata-only plans must preserve package, Composer, and submodule capacity detection.');
$GLOBALS['bootstrap_demand_filters']['datamachine_code_worktree_blobless_tracked_entry_bytes'] = static fn() => 32768;
$filtered_blobless_tree = WorktreeBootstrapper::demand_plan_for_target($repo, 'target-tree', true);
unset($GLOBALS['bootstrap_demand_filters']['datamachine_code_worktree_blobless_tracked_entry_bytes']);
bootstrap_demand_assert($filtered_blobless_tree['tracked_bytes'] === $filtered_blobless_tree['counts']['tracked_entries'] * 32768, 'Installations must be able to tune the conservative blobless estimate without changing core.');

exec('git -C ' . escapeshellarg($repo) . ' checkout -qb stale-branch');
file_put_contents($repo . '/stale.txt', 'stale');
Expand Down Expand Up @@ -143,6 +178,7 @@ function bootstrap_demand_assert( bool $condition, string $message ): void {
unlink($fixture . '/' . $file);
}
if ( is_file($bin . '/git') ) { unlink($bin . '/git'); }
if ( is_file($git_log) ) { unlink($git_log); }
rmdir($bin);
rmdir($fixture . '/frontend');
rmdir($fixture . '/php');
Expand Down