Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 69 additions & 1 deletion src/classes/Manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,19 @@ export class Manager extends Base {
if (isNewPackage) this.packages.set(pkg.slug, pkgExisting);
}

// Same as addPackage(), but ingests via Package.addVersionSummary() - see sync().
addPackageSummary(pkg: Package) {
let pkgExisting = this.packages.get(pkg.slug);
const isNewPackage: boolean = !pkgExisting;
if (!pkgExisting) {
pkgExisting = new Package(pkg.slug);
}
for (const [version, pkgVersion] of pkg.versions) {
pkgExisting.addVersionSummary(version, pkgVersion);
}
if (isNewPackage) this.packages.set(pkg.slug, pkgExisting);
}

filter(method: (pkgVersion: PackageVersion, pkg: Package) => boolean): Package[] {
const results: Package[] = [];
for (const [, pkg] of this.packages) {
Expand Down Expand Up @@ -152,7 +165,12 @@ export class Manager extends Base {
// Add one version at a time (rather than the whole package via addPackage() in one
// call) so a single malformed version - from a registry this manager doesn't
// control - can't abort every other version/package still left to sync.
this.addPackage(new Package(slug, { [version]: json[type][slug].versions[version] }));
// The registry root only summarizes each package's latest version, and omits
// `url`/`sha256` from each file (see specification.md "Listing endpoints vs package
// endpoints") - addPackageSummary() ingests that shape without rejecting it for
// fields it never claimed to include. Anything that needs to actually download a file
// (install(), etc.) resolves the full version separately via fetchPackageVersion().
this.addPackageSummary(new Package(slug, { [version]: json[type][slug].versions[version] }));
} catch (err) {
this.syncErrors.push(`${slug}@${version}: ${(err as Error).message}`);
}
Expand All @@ -161,6 +179,56 @@ export class Manager extends Base {
}
}

// Fetches the full per-version payload (every file's `url`/`sha256` included) directly from the
// org/package/version-level endpoint - used whenever a cached Package (populated by sync()'s
// trimmed summary) doesn't have everything an operation needs, e.g. an older version that isn't
// the latest, or the latest version's download data. Tries each configured registry in order,
// the same way sync() combines them; returns undefined if none of them have it.
protected async fetchPackageVersion(slug: string, version: string): Promise<PackageVersion | undefined> {
const registries: ConfigRegistry[] = this.config.get('registries') as ConfigRegistry[];
for (const registry of registries) {
try {
const url = `${registryUrl(registry).replace(/\/$/, '')}/${this.type}/${slug}/${version}`;
return (await apiJson(url)) as PackageVersion;
} catch {
continue;
}
}
return undefined;
}

// A sync()-cached summary version has every file's compatibility fields (architectures,
// systems, contains, type, size) but not `url`/`sha256` - so it's enough for listing/filtering,
// but not for an actual download. Every file having `url` is the reliable tell that this is the
// full payload, not the summary.
private isDownloadable(pkgVersion?: PackageVersion): pkgVersion is PackageVersion {
return (
!!pkgVersion && !!pkgVersion.files && pkgVersion.files.length > 0 && pkgVersion.files.every(file => !!file.url)
);
}

// Resolves a package/version pair to installable data, transparently upgrading a sync()-cached
// summary to the full payload (with `url`/`sha256`) on demand. Returns undefined if the
// package/version genuinely doesn't exist in the registry.
async resolvePackageVersion(
slug: string,
version?: string,
): Promise<{ pkg: Package; pkgVersion: PackageVersion; versionNum: string } | undefined> {
const pkg = this.getPackage(slug);
if (!pkg) return undefined;
const versionNum = version || pkg.latestVersion();
let pkgVersion = pkg.getVersion(versionNum);
if (!this.isDownloadable(pkgVersion)) {
const fetched = await this.fetchPackageVersion(slug, versionNum);
if (fetched) {
pkg.addVersion(versionNum, fetched);
pkgVersion = fetched;
}
}
if (!pkgVersion) return undefined;
return { pkg, pkgVersion, versionNum };
}

toJSON() {
const data: RegistryPackages = {};
for (const [slug, pkg] of this.packages.entries()) {
Expand Down
20 changes: 13 additions & 7 deletions src/classes/ManagerLocal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,16 @@ export class ManagerLocal extends Manager {
// elevated command payload built below.
if (!isValidSlug(slug)) throw new Error(`Invalid package slug: ${slug}`);
if (version && !isValidVersion(version)) throw new Error(`Invalid package version: ${version}`);
// Get package information from registry.
// Get package information from registry. resolvePackageVersion() transparently fetches the
// full per-version payload (every file's url/sha256 included) whenever sync()'s cached
// summary doesn't have it - either because this isn't the latest version, or because each
// file's url/sha256 was omitted from the registry root/list endpoints (see specification.md
// "Listing endpoints vs package endpoints").
const pkg: Package | undefined = this.getPackage(slug);
if (!pkg) throw new Error(`Package ${slug} not found in registry`);
const versionNum: string = version || pkg.latestVersion();
const pkgVersion: PackageVersion | undefined = pkg?.getVersion(versionNum);
if (!pkgVersion) throw new Error(`Package ${slug} version ${versionNum} not found in registry`);
const resolved = await this.resolvePackageVersion(slug, version);
if (!resolved) throw new Error(`Package ${slug} version ${version || pkg.latestVersion()} not found in registry`);
const { pkgVersion, versionNum } = resolved;
if (this.isPackageInstalled(slug, versionNum)) {
this.log(`Package ${slug} version ${versionNum} already installed`);
pkgVersion.installed = true;
Expand Down Expand Up @@ -506,9 +510,11 @@ export class ManagerLocal extends Manager {
manager.scan();
const pkg: Package | undefined = manager.getPackage(slug);
if (!pkg) throw new Error(`Package ${slug} not found in registry`);
const versionNum: string = version || pkg.latestVersion();
const pkgVersion: PackageVersion | undefined = pkg?.getVersion(versionNum);
if (!pkgVersion) throw new Error(`Package ${slug} version ${versionNum} not found in registry`);
// resolvePackageVersion() fetches the full per-version payload on demand when sync()'s cached
// summary doesn't have it - see install()'s comment for why that can happen.
const resolved = await manager.resolvePackageVersion(slug, version);
if (!resolved) throw new Error(`Package ${slug} version ${version || pkg.latestVersion()} not found in registry`);
const { versionNum } = resolved;
// Get local package file.
const pkgFile = packageLoadFile(filePath) as any;
if (pkgFile[type] && pkgFile[type][slug] && pkgFile[type][slug] === versionNum) {
Expand Down
26 changes: 19 additions & 7 deletions src/classes/Package.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as semver from 'semver';
import { PackageVersionReport, PackageVersion, PackageVersions } from '../types/Package.js';
import { packageErrors, packageIsVerified, packageRecommendations } from '../helpers/package.js';
import { packageErrors, packageIsVerified, packageRecommendations, packageSummaryErrors } from '../helpers/package.js';
import { isValidSlug } from '../helpers/utils.js';
import { Base } from './Base.js';

Expand All @@ -19,23 +19,35 @@ export class Package extends Base {
this.version = this.latestVersion();
}

addVersion(num: string, version: PackageVersion) {
// For now allow package versions to be overwritten.
// if (this.versions.has(num)) return this.log(`Package ${version.name} version ${num} already exists`);
const errors = packageErrors(version);
private setVersion(num: string, version: PackageVersion, errors: PackageVersionReport['errors']) {
const recs = packageRecommendations(version);
const report: PackageVersionReport = {
...(errors.length > 0 && { errors }),
...(errors && errors.length > 0 && { errors }),
...(recs.length > 0 && { recs }),
};
if (Object.keys(report).length > 0) this.reports.set(num, report);
if (errors.length > 0)
if (errors && errors.length > 0)
throw new Error(`Package ${version.name} version ${num} has validation errors: ${JSON.stringify(errors)}`);
version.verified = packageIsVerified(this.slug, version);
this.versions.set(num, version);
this.version = this.latestVersion();
}

addVersion(num: string, version: PackageVersion) {
// For now allow package versions to be overwritten.
// if (this.versions.has(num)) return this.log(`Package ${version.name} version ${num} already exists`);
this.setVersion(num, version, packageErrors(version));
}

// Same as addVersion(), but validates against PackageVersionSummaryValidator instead - each
// file's `url`/`sha256` are optional, matching the trimmed payload the registry root/list
// endpoints actually serve (see specification.md "Listing endpoints vs package endpoints").
// Used by Manager.sync(); any caller that needs to actually download a file resolves the full
// version separately (see Manager.fetchPackageVersion()) and stores it via addVersion() instead.
addVersionSummary(num: string, version: PackageVersion) {
this.setVersion(num, version, packageSummaryErrors(version));
}

removeVersion(num: string) {
if (!this.versions.has(num)) return;
this.versions.delete(num);
Expand Down
43 changes: 41 additions & 2 deletions src/helpers/package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,22 @@ export function packageCompatibleFiles(
sys: SystemType[],
excludedFormats?: FileFormat[],
) {
// Defensive - every real PackageVersion has `files`, but a malformed one shouldn't crash a
// filter/listing pass.
if (!pkg.files) return [];
return pkg.files.filter((file: FileInterface) => {
const archMatches = file.architectures.filter(architecture => {
return arch.includes(architecture);
});
const sysMatches = file.systems.filter(system => {
return sys.includes(system.type);
});
const formatAllowed =
excludedFormats && excludedFormats.includes(pathGetExt(file.url).toLowerCase() as FileFormat) ? false : true;
// `url` is omitted from each file at the registry root/list endpoints (see
// specification.md "Listing endpoints vs package endpoints") - format can't be determined
// from it there, so don't exclude on a format we can't check. install() always resolves the
// full version (with `url`) before this matters for real exclusion decisions.
const fileFormat = file.url ? (pathGetExt(file.url).toLowerCase() as FileFormat) : undefined;
const formatAllowed = !(fileFormat && excludedFormats && excludedFormats.includes(fileFormat));
return archMatches.length && sysMatches.length && formatAllowed;
});
}
Expand All @@ -39,6 +46,16 @@ export function packageErrors(pkgVersion: PackageVersion) {
return PackageVersionValidator.safeParse(pkgVersion).error?.issues || [];
}

// The registry root and type-list endpoints (see specification.md "Listing endpoints vs package
// endpoints") only summarize each package's latest version, and omit `url`/`sha256` from each
// file to keep those documents small - used by Manager.sync() so that summary ingestion doesn't
// reject every package over fields it never claimed to include. Org/package/version endpoints
// still return every file in full, validated via packageErrors() when that data is actually
// resolved (see Manager.fetchPackageVersion()).
export function packageSummaryErrors(pkgVersion: PackageVersion) {
return PackageVersionSummaryValidator.safeParse(pkgVersion).error?.issues || [];
}

export function packageFileMap(pkgVersion: PackageVersion) {
return pkgVersion.files.reduce((result: PackageFileMap, file) => {
file.systems.forEach(system => {
Expand Down Expand Up @@ -100,6 +117,16 @@ export const PackageVersionValidator = z.object({
url: z.string().min(8).max(256).startsWith('https://'),
});

// Same shape as PackageFileValidator, but `url`/`sha256` are optional - see
// packageSummaryErrors().
export const PackageFileSummaryValidator = PackageFileValidator.partial({ url: true, sha256: true });

// Same shape as PackageVersionValidator, but each file is validated against
// PackageFileSummaryValidator instead - see packageSummaryErrors().
export const PackageVersionSummaryValidator = PackageVersionValidator.extend({
files: z.array(PackageFileSummaryValidator).min(1).max(256),
});

export const SemverValidator = z
.string()
.regex(
Expand Down Expand Up @@ -143,6 +170,12 @@ export function packageRecommendations(pkgVersion: PackageVersion) {
file.systems.forEach(system => {
supportedSystems[system.type] = true;
});

// `url` is omitted from each file at the registry root/list endpoints (see
// specification.md "Listing endpoints vs package endpoints") - every recommendation below
// is derived from the url itself, so there's nothing to check without it.
if (!file.url) return;

const ext: string = pathGetExt(file.url).toLowerCase();
supportedFileFormats[ext] = true;
packageRecommendationsUrl(file, recs, 'url', 'github');
Expand Down Expand Up @@ -243,8 +276,14 @@ export function packageYamlToJs(pkgYaml: string) {
}

export function packageIsVerified(slug: string, pkgVersion: PackageVersion) {
if (!pkgVersion.files || pkgVersion.files.length === 0) return false;
const org: string = slug.split('/')[0];
return pkgVersion.files.every(file => {
// `url` is omitted from each file at the registry root/list endpoints (see
// specification.md "Listing endpoints vs package endpoints") - nothing to verify against yet,
// so report unverified rather than throwing. install() re-resolves the full version (with
// `url`) before this ever matters for a real decision.
if (!file.url) return false;
const url: string = file.url.toLowerCase();
const root: string = url.startsWith('https://github.com/') ? 'https://github.com/' + org + '/' : `https://${org}.`;
return url.startsWith(root);
Expand Down
102 changes: 101 additions & 1 deletion tests/classes/Manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { SystemType } from '../../src/types/SystemType';
import { Architecture } from '../../src/types/Architecture';
import { PackageVersion } from '../../src/types/Package';
import { packageCompatibleFiles } from '../../src/helpers/package';
import { mockRegistrySync, omitDownloads } from '../testUtils';
import { mockRegistrySync, omitDownloads, toSummaryVersion } from '../testUtils';
import * as apiHelpers from '../../src/helpers/api';

afterEach(() => {
Expand Down Expand Up @@ -134,6 +134,19 @@ test('Manager list packages incompatible', () => {
expect(manager.listPackages(undefined, Architecture.X64, SystemType.Linux)).toEqual([pkgNoWin]);
});

test('Manager list packages by architecture/system works on a sync()-cached summary (url/sha256 omitted)', () => {
// Unlike install(), listing/filtering doesn't need a fetchPackageVersion() round trip - every
// compatibility field (architectures, systems, contains, type, size) is already present on the
// registry root/list endpoints, only url/sha256 are missing (see specification.md "Listing
// endpoints vs package endpoints").
const manager = new Manager(RegistryType.Plugins);
const pkg = new Package(PLUGIN_PACKAGE.slug);
pkg.addVersionSummary(PLUGIN_PACKAGE.version, toSummaryVersion(PLUGIN));
manager.addPackageSummary(pkg);
expect(manager.listPackages(undefined, Architecture.X64, SystemType.Linux)).toEqual([pkg]);
expect(manager.listPackages(undefined, Architecture.Arm32, SystemType.Linux)).toEqual([]);
});

test('Manager filter packages', () => {
const manager = new Manager(RegistryType.Plugins);
const pkg = new Package(PLUGIN_PACKAGE.slug);
Expand Down Expand Up @@ -281,3 +294,90 @@ test('Manager sync with existing package', async () => {
const pkgReturned = manager.getPackage(PLUGIN_PACKAGE.slug);
expect(pkgReturned?.getVersion(PLUGIN_PACKAGE.version)?.name).toEqual('Surge XT');
});

// The registry root/list endpoints only summarize each package's latest version, and omit
// `url`/`sha256` from each file (see specification.md "Listing endpoints vs package endpoints") -
// the following tests cover sync() ingesting that trimmed shape, and resolvePackageVersion()
// transparently fetching the full version (with every file's url/sha256) on demand.
test('Manager sync ingests a trimmed summary (files present, url/sha256 omitted) without rejecting the package', async () => {
const pluginSummary = toSummaryVersion(PLUGIN);
vi.spyOn(apiHelpers, 'apiJson').mockResolvedValue({
name: 'Mock Registry',
url: 'https://example.invalid/mock',
version: '1.0.0',
[RegistryType.Plugins]: {
[PLUGIN_PACKAGE.slug]: {
slug: PLUGIN_PACKAGE.slug,
version: PLUGIN_PACKAGE.version,
versions: { [PLUGIN_PACKAGE.version]: pluginSummary },
},
},
});

const manager = new Manager(RegistryType.Plugins, {
registries: [{ name: 'Mock Registry', url: 'https://example.invalid/mock' }],
});
await manager.sync();

expect(manager.getSyncErrors()).toEqual([]);
const pkgVersion = manager.getPackage(PLUGIN_PACKAGE.slug)?.getVersion(PLUGIN_PACKAGE.version);
expect(pkgVersion?.files).toHaveLength(PLUGIN.files.length);
expect(pkgVersion?.files.every(file => !file.url && !file.sha256)).toEqual(true);
expect(pkgVersion?.name).toEqual(PLUGIN.name);
});

test('Manager resolvePackageVersion fetches the full version when the cached summary is missing url/sha256', async () => {
const pluginSummary = toSummaryVersion(PLUGIN);
const versionUrl = `https://example.invalid/mock/${RegistryType.Plugins}/${PLUGIN_PACKAGE.slug}/${PLUGIN_PACKAGE.version}`;
const apiJsonSpy = vi.spyOn(apiHelpers, 'apiJson').mockImplementation(async (url: string) => {
if (url === versionUrl) return PLUGIN;
return {
name: 'Mock Registry',
url: 'https://example.invalid/mock',
version: '1.0.0',
[RegistryType.Plugins]: {
[PLUGIN_PACKAGE.slug]: {
slug: PLUGIN_PACKAGE.slug,
version: PLUGIN_PACKAGE.version,
versions: { [PLUGIN_PACKAGE.version]: pluginSummary },
},
},
};
});

const manager = new Manager(RegistryType.Plugins, {
registries: [{ name: 'Mock Registry', url: 'https://example.invalid/mock' }],
});
await manager.sync();
const resolved = await manager.resolvePackageVersion(PLUGIN_PACKAGE.slug, PLUGIN_PACKAGE.version);

expect(resolved?.pkgVersion.files).toEqual(PLUGIN.files);
expect(apiJsonSpy).toHaveBeenCalledWith(versionUrl);
});

test('Manager resolvePackageVersion returns undefined for an unknown package', async () => {
const manager = new Manager(RegistryType.Plugins);
expect(await manager.resolvePackageVersion('nonexistent-org/nonexistent-plugin')).toBeUndefined();
});

test('Manager resolvePackageVersion returns undefined when no registry has the version', async () => {
const apiJsonSpy = mockRegistrySync(REGISTRY_PLUGIN_VER);
const manager = new Manager(RegistryType.Plugins, {
registries: [{ name: 'Mock Registry', url: 'https://example.invalid/mock' }],
});
await manager.sync();
apiJsonSpy.mockRejectedValue(new Error('not found'));

expect(await manager.resolvePackageVersion(PLUGIN_PACKAGE.slug, '99.99.99')).toBeUndefined();
});

test('Manager resolvePackageVersion does not refetch when the cached version already has files', async () => {
const apiJsonSpy = mockRegistrySync(REGISTRY_PLUGIN_VER);
const manager = new Manager(RegistryType.Plugins);
await manager.sync();
apiJsonSpy.mockClear();

const resolved = await manager.resolvePackageVersion(PLUGIN_PACKAGE.slug, PLUGIN_PACKAGE.version);
expect(resolved?.pkgVersion.files).toEqual(PLUGIN.files);
expect(apiJsonSpy).not.toHaveBeenCalled();
});
Loading