Skip to content

Implement AST caching #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lm-sousa wants to merge 7 commits into
base: staging-ast-dump-cache-parent
Choose a base branch
from
Open

Implement AST caching #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c0633bc
perf(clang): skip unchanged AST dumps
lm-sousa Aug 25, 2026
0139863
fix(clang): harden AST dump cache validation
lm-sousa Aug 25, 2026
084942c
fix(clang): bypass unsafe AST cache inputs
lm-sousa Aug 25, 2026
920f7d7
perf(clang): buffer volatile macro dependency scans
lm-sousa Aug 25, 2026
c46ad9c
fix(clang): scan include probes as raw bytes
lm-sousa Aug 25, 2026
141813d
perf(clang): derive dumper ids from source paths
lm-sousa Aug 25, 2026
8ed0a98
refactor(clang): simplify cache lifecycle
lm-sousa Aug 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
559 changes: 559 additions & 0 deletions ClangAstParser/src/pt/up/fe/specs/clang/AstDumpCache.java
View file
Open in desktop

Large diffs are not rendered by default.

95 changes: 72 additions & 23 deletions ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.HexFormat;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;

final class CacheFiles {
Expand All @@ -49,7 +51,7 @@ final class CacheFiles {
private CacheFiles() {
}

static <T> T withMaintenanceLock(Path cacheRoot, Supplier<T> action) {
private static <T> T withMaintenanceLock(Path cacheRoot, Supplier<T> action) {
var lockPath = cacheRoot.resolve(MAINTENANCE_LOCK_FILENAME);
synchronized (MAINTENANCE_MONITOR) {
try {
Expand All @@ -64,13 +66,53 @@ static <T> T withMaintenanceLock(Path cacheRoot, Supplier<T> action) {
}
}

static void withMaintenanceLock(Path cacheRoot, Runnable action) {
private static void withMaintenanceLock(Path cacheRoot, Runnable action) {
withMaintenanceLock(cacheRoot, () -> {
action.run();
return null;
});
}

/**
* Claims a published directory, refreshes its use time, and uses it outside the maintenance lock.
*
* <p>An empty result means the directory was invalid and removes it before returning. Exceptions leave the
* directory untouched so callers can report malformed published resources instead of silently repairing them.</p>
*/
static <T> Optional<T> useDirectory(Path cacheRoot, Path directory,
Function<Path, Optional<T>> use) {
boolean claimed = withMaintenanceLock(cacheRoot, () -> {
if (!Files.isDirectory(directory)) {
return false;
}

touchLocked(directory);
return true;
});

if (!claimed) {
return Optional.empty();
}

var result = use.apply(directory);
if (result.isEmpty()) {
withMaintenanceLock(cacheRoot, () -> deleteQuietly(directory));
}

return result;
}

/** Refreshes existing cache paths as one maintenance operation. */
static void touch(Path cacheRoot, Path... paths) {
withMaintenanceLock(cacheRoot, () -> {
for (var path : paths) {
if (Files.exists(path)) {
touchLocked(path);
}
}
});
}

static StagingDirectory createStagingDirectory(Path cacheRoot, Path parent, String prefix) {
return withMaintenanceLock(cacheRoot, () -> createStagingDirectoryLocked(parent, prefix));
}
Expand Down Expand Up @@ -138,16 +180,13 @@ record StagingDirectory(Path path, Path lockPath, FileChannel channel) implement

@Override
public void close() {
try {
channel.close();
} catch (IOException e) {
throw new UncheckedIOException("Could not close cache staging lock '" + lockPath + "'", e);
}
deleteQuietly(path);

try {
channel.close();
Files.deleteIfExists(lockPath);
} catch (IOException e) {
throw new UncheckedIOException("Could not close cache staging lock '" + lockPath + "'", e);
} catch (IOException ignored) {
// Staging cleanup is best-effort. A remaining lock lets a later cache cleanup safely retry.
}
}
}
Expand All @@ -163,9 +202,8 @@ static File installFile(Path cacheRoot, File destination, FileResourceProvider r
return destination;
}

var stagingDirectory = createStagingDirectory(cacheRoot, destination.getParentFile().toPath(),
"." + destination.getName() + ".tmp-");
try {
try (var stagingDirectory = createStagingDirectory(cacheRoot, destination.getParentFile().toPath(),
"." + destination.getName() + ".tmp-")) {
File stagedFile = resource.write(stagingDirectory.path().toFile());
if (stagedFile == null || !stagedFile.isFile()) {
throw new RuntimeException("Could not download " + description);
Expand All @@ -182,12 +220,6 @@ static File installFile(Path cacheRoot, File destination, FileResourceProvider r
}

return publish(stagedFile.toPath(), destination.toPath()).toFile();
} finally {
try {
deleteQuietly(stagingDirectory.path());
} finally {
stagingDirectory.close();
}
}
}

Expand Down Expand Up @@ -232,15 +264,28 @@ static boolean hasExpectedSha256(File file, String expectedSha256) {
return expectedSha256.equalsIgnoreCase(calculateSha256(file));
}

static void touch(Path path) {
private static void touchLocked(Path path) {
try {
Files.setLastModifiedTime(path, FileTime.from(Instant.now()));
} catch (IOException e) {
throw new UncheckedIOException("Could not update cache use time for '" + path + "'", e);
}
}

static void deleteStaleDirectories(Path cacheRoot, Path parent, Instant cutoff, Path excluded) {
/** Removes stale published directories and abandoned staging directories in one locked pass. */
static void cleanupDirectories(Path cacheRoot, Path parent, Instant cutoff, Path excluded) {
withMaintenanceLock(cacheRoot, () -> {
deleteStaleDirectories(parent, cutoff, excluded);
deleteUnlockedStagingDirectories(parent);
});
}

/** Removes abandoned staging directories without treating other child directories as cache entries. */
static void cleanupStagingDirectories(Path cacheRoot, Path parent) {
withMaintenanceLock(cacheRoot, () -> deleteUnlockedStagingDirectories(parent));
}

private static void deleteStaleDirectories(Path parent, Instant cutoff, Path excluded) {
if (!Files.isDirectory(parent)) {
return;
}
Expand All @@ -255,7 +300,7 @@ static void deleteStaleDirectories(Path cacheRoot, Path parent, Instant cutoff,
continue;
}

withMaintenanceLock(cacheRoot, () -> deleteIfStale(child, cutoff));
deleteIfStale(child, cutoff);
}
} catch (IOException e) {
throw new UncheckedIOException("Could not clean stale cache directories below '" + parent + "'", e);
Expand All @@ -273,14 +318,14 @@ private static void deleteIfStale(Path path, Instant cutoff) {
}
}

static void deleteUnlockedStagingLocks(Path cacheRoot, Path parent) {
private static void deleteUnlockedStagingDirectories(Path parent) {
if (!Files.isDirectory(parent)) {
return;
}

try (DirectoryStream<Path> locks = Files.newDirectoryStream(parent, ".*.tmp-*.lock")) {
for (Path lock : locks) {
withMaintenanceLock(cacheRoot, () -> deleteIfUnlockedStagingLock(lock));
deleteIfUnlockedStagingLock(lock);
}
} catch (IOException e) {
throw new UncheckedIOException("Could not clean cache staging directories below '" + parent + "'",
Expand Down Expand Up @@ -347,4 +392,8 @@ private static String calculateSha256(File file) {
throw new RuntimeException("Could not calculate SHA-256 for file '" + file + "'", e);
}
}

static String calculateSha256(Path path) {
return calculateSha256(path.toFile());
}
}
90 changes: 30 additions & 60 deletions ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;

Expand Down Expand Up @@ -138,41 +139,30 @@ private boolean isUsable(CachedClangFiles cached) {
return false;
}

return CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> {
if (!cached.files().clangExecutable().isFile()) {
return false;
}

if (cached.files().systemResourceDir() != null
&& !cached.files().systemResourceDir().isDirectory()) {
return false;
}

var includesFolder = cached.includesFolder();
if (includesFolder == null) {
return true;
}

if (!includesFolder.exists()) {
return false;
}
if (!cached.files().clangExecutable().isFile()) {
return false;
}

CacheFiles.touch(includesFolder.toPath());
if (!isIncludesCacheValid(includesFolder)) {
throw invalidIncludesCache(includesFolder, includesFolder.getName());
}
if (cached.files().systemResourceDir() != null
&& !cached.files().systemResourceDir().isDirectory()) {
return false;
}

var includesFolder = cached.includesFolder();
if (includesFolder == null) {
return true;
});
}

return useExistingIncludes(getClangCacheRoot(), includesFolder, includesFolder.getName()) != null;
}

private void touchUse(File resourceFolder, File includesFolder) {
CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> {
CacheFiles.touch(resourceFolder.toPath());
if (includesFolder != null) {
CacheFiles.touch(includesFolder.toPath());
}
});
if (includesFolder == null) {
CacheFiles.touch(getClangCacheRoot().toPath(), resourceFolder.toPath());
return;
}

CacheFiles.touch(getClangCacheRoot().toPath(), resourceFolder.toPath(), includesFolder.toPath());
}

static File getLocalExecutable(File buildFolder) {
Expand Down Expand Up @@ -239,11 +229,9 @@ private void unblockWindowsFile(File executable) {

public File getClangResourceFolder() {
var cacheFolder = getClangCacheRoot();
return CacheFiles.withMaintenanceLock(cacheFolder.toPath(), () -> {
var releaseFolder = SpecsIo.mkdir(getReleasesFolder(), ClangAstWebResource.getReleaseTag());
CacheFiles.touch(releaseFolder.toPath());
return releaseFolder;
});
var releaseFolder = SpecsIo.mkdir(getReleasesFolder(), ClangAstWebResource.getReleaseTag());
CacheFiles.touch(cacheFolder.toPath(), releaseFolder.toPath());
return releaseFolder;
}

public static File getDefaultTempFolder() {
Expand Down Expand Up @@ -400,10 +388,8 @@ static File resolveIncludes(File cacheFolder, ClangDumperManifestAsset includesA
}

var includesRoot = extractedFolder.getParentFile().toPath();
CacheFiles.deleteUnlockedStagingLocks(cacheFolder.toPath(), includesRoot);
var stagingFolder = CacheFiles.createStagingDirectory(cacheFolder.toPath(), includesRoot,
"." + includesAsset.sha256() + ".tmp-");
try {
try (var stagingFolder = CacheFiles.createStagingDirectory(cacheFolder.toPath(), includesRoot,
"." + includesAsset.sha256() + ".tmp-")) {
var downloadFolder = CacheFiles.createTemporaryDirectory(stagingFolder.path(), ".download-");
try {
var archive = archiveResource.write(downloadFolder.toFile());
Expand Down Expand Up @@ -439,32 +425,17 @@ static File resolveIncludes(File cacheFolder, ClangDumperManifestAsset includesA
}

return existingFolder;
} finally {
try {
CacheFiles.delete(stagingFolder.path());
} finally {
stagingFolder.close();
}
}
}

private static File useExistingIncludes(File cacheFolder, File includesFolder, String sha256) {
if (!includesFolder.exists()) {
return null;
}

return CacheFiles.withMaintenanceLock(cacheFolder.toPath(), () -> {
if (!includesFolder.exists()) {
return null;
}

CacheFiles.touch(includesFolder.toPath());
return CacheFiles.useDirectory(cacheFolder.toPath(), includesFolder.toPath(), path -> {
if (!isIncludesCacheValid(includesFolder)) {
throw invalidIncludesCache(includesFolder, sha256);
}

return includesFolder;
});
return Optional.of(includesFolder);
}).orElse(null);
}

private static RuntimeException invalidIncludesCache(File includesFolder, String sha256) {
Expand Down Expand Up @@ -531,12 +502,11 @@ private void deleteStaleVersions(Instant now, File currentVersionFolder, File cu
var cutoff = now.minus(STALE_CACHE_MAX_AGE);
var cacheRoot = getClangCacheRoot().toPath();
try {
CacheFiles.deleteStaleDirectories(cacheRoot, getReleasesFolder().toPath(), cutoff,
CacheFiles.cleanupDirectories(cacheRoot, getReleasesFolder().toPath(), cutoff,
currentVersionFolder.toPath());
CacheFiles.deleteStaleDirectories(cacheRoot, getIncludesRoot().toPath(), cutoff,
CacheFiles.cleanupDirectories(cacheRoot, getIncludesRoot().toPath(), cutoff,
currentIncludesFolder == null ? null : currentIncludesFolder.toPath());
CacheFiles.deleteUnlockedStagingLocks(cacheRoot, currentVersionFolder.toPath());
CacheFiles.deleteUnlockedStagingLocks(cacheRoot, getIncludesRoot().toPath());
CacheFiles.cleanupStagingDirectories(cacheRoot, currentVersionFolder.toPath());
} catch (RuntimeException e) {
SpecsLogs.warn("Could not clean stale clang-dumper cache resources", e);
}
Expand Down
Loading
Loading