diff --git a/README.md b/README.md index d1fe6f4..2e952ed 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,21 @@ Returns a list of supported ecosystems. func Ecosystems() []string ``` +### DiscoverManifests + +Discovers root manifests, GitHub Actions workflows, and declared Cargo, Go, +npm/Yarn, and pnpm workspace members. Discovery is repository-aware while +`Parse` remains a pure single-file operation. + +```go +reader := manifests.NewFSReader(os.DirFS(".")) +found, err := manifests.DiscoverManifests(reader) +``` + +Callers reading historical revisions can implement `RepositoryReader` over a +Git tree. Paths and glob patterns are rooted, slash-separated repository paths. +Workspace records set `ParentPath` to the configuration that selected them. + ## Types ### Dependency diff --git a/discovery.go b/discovery.go new file mode 100644 index 0000000..0c7f0cb --- /dev/null +++ b/discovery.go @@ -0,0 +1,237 @@ +package manifests + +import ( + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +// RepositoryReader provides bounded access to files in a repository. Paths and +// glob patterns use forward slashes and are relative to the repository root. +// ReadFile should return an error matching fs.ErrNotExist for absent files. +type RepositoryReader interface { + ReadFile(name string) ([]byte, error) + Glob(pattern string) ([]string, error) +} + +// FSReader adapts an fs.FS to RepositoryReader. It supports recursive ** globs. +type FSReader struct { + fsys fs.FS +} + +// NewFSReader returns a repository reader rooted at fsys. Use os.DirFS to +// discover manifests in a working tree. +func NewFSReader(fsys fs.FS) *FSReader { + return &FSReader{fsys: fsys} +} + +// ReadFile reads a repository-relative file. +func (r *FSReader) ReadFile(name string) ([]byte, error) { + if r == nil || r.fsys == nil { + return nil, errors.New("nil repository filesystem") + } + return fs.ReadFile(r.fsys, name) +} + +// Glob returns files matching a repository-relative doublestar pattern. +func (r *FSReader) Glob(pattern string) ([]string, error) { + if r == nil || r.fsys == nil { + return nil, errors.New("nil repository filesystem") + } + return doublestar.Glob(r.fsys, pattern, + doublestar.WithFilesOnly(), + doublestar.WithFailOnIOErrors(), + ) +} + +// DiscoveredManifest identifies a project or workspace manifest. ParentPath is +// the repository-relative workspace configuration that selected a nested +// manifest; it is empty for root and path-based manifests. +type DiscoveredManifest struct { + Path string + Ecosystem string + Kind Kind + ParentPath string +} + +type manifestDiscovery struct { + reader RepositoryReader + items map[discoveredManifestKey]DiscoveredManifest +} + +type discoveredManifestKey struct { + path string + ecosystem string + kind Kind +} + +// DiscoverManifests returns the project and workspace manifests selected from +// a rooted repository. It does not parse dependency declarations. +func DiscoverManifests(reader RepositoryReader) ([]DiscoveredManifest, error) { + if reader == nil { + return nil, errors.New("nil repository reader") + } + + discovery := &manifestDiscovery{ + reader: reader, + items: make(map[discoveredManifestKey]DiscoveredManifest), + } + for _, pattern := range []string{"*", ".github/workflows/*.yml", ".github/workflows/*.yaml"} { + if err := discovery.addMatches(pattern, ""); err != nil { + return nil, fmt.Errorf("discovering manifests matching %q: %w", pattern, err) + } + } + + if err := discovery.discoverCargoWorkspace(); err != nil { + return nil, err + } + if err := discovery.discoverGoWorkspace(); err != nil { + return nil, err + } + if err := discovery.discoverNPMWorkspace(); err != nil { + return nil, err + } + if err := discovery.discoverPnpmWorkspace(); err != nil { + return nil, err + } + + return discovery.sorted(), nil +} + +func (d *manifestDiscovery) addMatches(pattern, parentPath string) error { + matches, err := d.reader.Glob(pattern) + if err != nil { + return err + } + sort.Strings(matches) + for _, match := range matches { + d.add(match, parentPath) + } + return nil +} + +func (d *manifestDiscovery) add(manifestPath, parentPath string) { + manifestPath, ok := normalizeRepositoryPath(manifestPath) + if !ok { + return + } + ecosystem, kind, ok := Identify(manifestPath) + if !ok { + return + } + if parentPath != "" { + var valid bool + parentPath, valid = normalizeRepositoryPath(parentPath) + if !valid { + return + } + } + + key := discoveredManifestKey{path: manifestPath, ecosystem: ecosystem, kind: kind} + item := DiscoveredManifest{ + Path: manifestPath, + Ecosystem: ecosystem, + Kind: kind, + ParentPath: parentPath, + } + if current, exists := d.items[key]; !exists || current.ParentPath != "" && parentPath == "" { + d.items[key] = item + } +} + +func (d *manifestDiscovery) addWorkspaceManifests( + includePatterns, excludePatterns []string, + manifestName, parentPath string, +) error { + excluded, err := d.workspaceManifestPaths(excludePatterns, manifestName) + if err != nil { + return err + } + included, err := d.workspaceManifestPaths(includePatterns, manifestName) + if err != nil { + return err + } + + paths := make([]string, 0, len(included)) + for manifestPath := range included { + if _, skip := excluded[manifestPath]; !skip { + paths = append(paths, manifestPath) + } + } + sort.Strings(paths) + for _, manifestPath := range paths { + d.add(manifestPath, parentPath) + } + return nil +} + +func (d *manifestDiscovery) workspaceManifestPaths(patterns []string, manifestName string) (map[string]struct{}, error) { + result := make(map[string]struct{}) + for _, pattern := range patterns { + pattern, ok := normalizeRepositoryPattern(pattern) + if !ok { + continue + } + matches, err := d.reader.Glob(path.Join(pattern, manifestName)) + if err != nil { + return nil, fmt.Errorf("expanding workspace pattern %q: %w", pattern, err) + } + for _, match := range matches { + if normalized, valid := normalizeRepositoryPath(match); valid { + result[normalized] = struct{}{} + } + } + } + return result, nil +} + +func (d *manifestDiscovery) readOptional(name string) ([]byte, bool, error) { + content, err := d.reader.ReadFile(name) + if errors.Is(err, fs.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return content, true, nil +} + +func (d *manifestDiscovery) sorted() []DiscoveredManifest { + result := make([]DiscoveredManifest, 0, len(d.items)) + for _, item := range d.items { + result = append(result, item) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Path != result[j].Path { + return result[i].Path < result[j].Path + } + if result[i].Ecosystem != result[j].Ecosystem { + return result[i].Ecosystem < result[j].Ecosystem + } + return result[i].Kind < result[j].Kind + }) + return result +} + +func normalizeRepositoryPath(value string) (string, bool) { + value = strings.TrimSpace(strings.ReplaceAll(value, `\`, "/")) + if value == "" || strings.HasPrefix(value, "/") { + return "", false + } + value = path.Clean(value) + if value == "." || value == ".." || strings.HasPrefix(value, "../") { + return "", false + } + return value, true +} + +func normalizeRepositoryPattern(value string) (string, bool) { + value = strings.TrimSpace(strings.ReplaceAll(value, `\`, "/")) + value = strings.TrimSuffix(value, "/") + return normalizeRepositoryPath(value) +} diff --git a/discovery_test.go b/discovery_test.go new file mode 100644 index 0000000..27c6aef --- /dev/null +++ b/discovery_test.go @@ -0,0 +1,185 @@ +package manifests + +import ( + "errors" + "io/fs" + "slices" + "strings" + "testing" + "testing/fstest" +) + +func TestDiscoverManifestsRootAndKnownPaths(t *testing.T) { + reader := mapFSReader(map[string]string{ + ".github/workflows/ci.yaml": `jobs: {}`, + ".github/workflows/ci.yml": `jobs: {}`, + ".github/workflows/readme": "ignored", + ".gitmodules": "", + "README.md": "ignored", + "nested/requirements.txt": "requests==2.0.0", + "package-lock.json": `{"lockfileVersion": 3}`, + "package.json": `{"name":"root"}`, + }) + + got, err := DiscoverManifests(reader) + if err != nil { + t.Fatalf("DiscoverManifests: %v", err) + } + want := []DiscoveredManifest{ + {Path: ".github/workflows/ci.yaml", Ecosystem: "github-actions", Kind: Manifest}, + {Path: ".github/workflows/ci.yml", Ecosystem: "github-actions", Kind: Manifest}, + {Path: ".gitmodules", Ecosystem: "git", Kind: Manifest}, + {Path: "package-lock.json", Ecosystem: "npm", Kind: Lockfile}, + {Path: "package.json", Ecosystem: "npm", Kind: Manifest}, + } + if !slices.Equal(got, want) { + t.Fatalf("DiscoverManifests() =\n%+v\nwant\n%+v", got, want) + } +} + +func TestDiscoverManifestsWorkspaces(t *testing.T) { + reader := mapFSReader(map[string]string{ + "Cargo.toml": `[workspace] +members = ["crates/*", "tools/cli"] +exclude = ["crates/private"] +`, + "crates/core/Cargo.toml": `[package]`, + "crates/private/Cargo.toml": `[package]`, + "tools/cli/Cargo.toml": `[package]`, + "go.work": `go 1.25.0 + +use ( + "./services/api" + ./libs/shared +) +`, + "services/api/go.mod": `module example.com/api`, + "libs/shared/go.mod": `module example.com/shared`, + "package.json": `{"workspaces":{"packages":["apps/*"],"nohoist":["**"]}}`, + "apps/web/package.json": `{"name":"web"}`, + "pnpm-workspace.yaml": `packages: + - "packages/**" + - "!packages/**/fixtures" +`, + "packages/direct/package.json": `{"name":"direct"}`, + "packages/group/api/package.json": `{"name":"api"}`, + "packages/group/fixtures/package.json": `{"name":"fixture"}`, + }) + + got, err := DiscoverManifests(reader) + if err != nil { + t.Fatalf("DiscoverManifests: %v", err) + } + want := []DiscoveredManifest{ + {Path: "Cargo.toml", Ecosystem: "cargo", Kind: Manifest}, + {Path: "apps/web/package.json", Ecosystem: "npm", Kind: Manifest, ParentPath: "package.json"}, + {Path: "crates/core/Cargo.toml", Ecosystem: "cargo", Kind: Manifest, ParentPath: "Cargo.toml"}, + {Path: "libs/shared/go.mod", Ecosystem: "golang", Kind: Manifest, ParentPath: "go.work"}, + {Path: "package.json", Ecosystem: "npm", Kind: Manifest}, + {Path: "packages/direct/package.json", Ecosystem: "npm", Kind: Manifest, ParentPath: "pnpm-workspace.yaml"}, + {Path: "packages/group/api/package.json", Ecosystem: "npm", Kind: Manifest, ParentPath: "pnpm-workspace.yaml"}, + {Path: "services/api/go.mod", Ecosystem: "golang", Kind: Manifest, ParentPath: "go.work"}, + {Path: "tools/cli/Cargo.toml", Ecosystem: "cargo", Kind: Manifest, ParentPath: "Cargo.toml"}, + } + if !slices.Equal(got, want) { + t.Fatalf("DiscoverManifests() =\n%+v\nwant\n%+v", got, want) + } +} + +func TestDiscoverManifestsNPMWorkspaceForms(t *testing.T) { + tests := []struct { + name string + workspaces string + }{ + {name: "npm array", workspaces: `["packages/*"]`}, + {name: "yarn object", workspaces: `{"packages":["packages/*"],"nohoist":["**"]}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reader := mapFSReader(map[string]string{ + "package.json": `{"workspaces":` + test.workspaces + `}`, + "packages/api/package.json": `{"name":"api"}`, + }) + + got, err := DiscoverManifests(reader) + if err != nil { + t.Fatalf("DiscoverManifests: %v", err) + } + want := []DiscoveredManifest{ + {Path: "package.json", Ecosystem: "npm", Kind: Manifest}, + {Path: "packages/api/package.json", Ecosystem: "npm", Kind: Manifest, ParentPath: "package.json"}, + } + if !slices.Equal(got, want) { + t.Fatalf("DiscoverManifests() = %+v, want %+v", got, want) + } + }) + } +} + +func TestDiscoverManifestsRejectsOutsideWorkspaceMembers(t *testing.T) { + reader := mapFSReader(map[string]string{ + "Cargo.toml": `[workspace]` + "\n" + `members = ["../outside", "/absolute"]`, + "outside/Cargo.toml": `[package]`, + "absolute/Cargo.toml": `[package]`, + "nested/other/Cargo.toml": `[package]`, + }) + + got, err := DiscoverManifests(reader) + if err != nil { + t.Fatalf("DiscoverManifests: %v", err) + } + want := []DiscoveredManifest{{Path: "Cargo.toml", Ecosystem: "cargo", Kind: Manifest}} + if !slices.Equal(got, want) { + t.Fatalf("DiscoverManifests() = %+v, want %+v", got, want) + } +} + +func TestDiscoverManifestsReportsConfigurationErrors(t *testing.T) { + tests := []struct { + name string + path string + content string + wantError string + }{ + {name: "cargo", path: "Cargo.toml", content: `[workspace`, wantError: "parsing Cargo workspace configuration"}, + {name: "go", path: "go.work", content: `use (`, wantError: "parsing Go workspace configuration"}, + {name: "npm", path: "package.json", content: `{"workspaces":true}`, wantError: "parsing npm workspace configuration"}, + {name: "pnpm", path: "pnpm-workspace.yaml", content: `packages: true`, wantError: "parsing pnpm workspace configuration"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := DiscoverManifests(mapFSReader(map[string]string{test.path: test.content})) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("DiscoverManifests() error = %v, want %q", err, test.wantError) + } + }) + } +} + +func TestDiscoverManifestsReaderErrors(t *testing.T) { + wantErr := errors.New("glob failed") + _, err := DiscoverManifests(errorRepositoryReader{err: wantErr}) + if !errors.Is(err, wantErr) { + t.Fatalf("DiscoverManifests() error = %v, want wrapped %v", err, wantErr) + } +} + +func mapFSReader(files map[string]string) *FSReader { + root := make(fstest.MapFS, len(files)) + for name, content := range files { + root[name] = &fstest.MapFile{Data: []byte(content), Mode: 0o644} + } + return NewFSReader(root) +} + +type errorRepositoryReader struct { + err error +} + +func (r errorRepositoryReader) ReadFile(string) ([]byte, error) { + return nil, fs.ErrNotExist +} + +func (r errorRepositoryReader) Glob(string) ([]string, error) { + return nil, r.err +} diff --git a/discovery_workspace.go b/discovery_workspace.go new file mode 100644 index 0000000..53eb81c --- /dev/null +++ b/discovery_workspace.go @@ -0,0 +1,141 @@ +package manifests + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/BurntSushi/toml" + "golang.org/x/mod/modfile" + "gopkg.in/yaml.v3" +) + +func (d *manifestDiscovery) discoverCargoWorkspace() error { + const parentPath = "Cargo.toml" + content, exists, err := d.readOptional(parentPath) + if err != nil { + return fmt.Errorf("reading Cargo workspace configuration: %w", err) + } + if !exists { + return nil + } + + var config struct { + Workspace struct { + Members []string `toml:"members"` + Exclude []string `toml:"exclude"` + } `toml:"workspace"` + } + if _, err := toml.Decode(string(content), &config); err != nil { + return fmt.Errorf("parsing Cargo workspace configuration: %w", err) + } + if err := d.addWorkspaceManifests( + config.Workspace.Members, + config.Workspace.Exclude, + "Cargo.toml", + parentPath, + ); err != nil { + return fmt.Errorf("discovering Cargo workspace members: %w", err) + } + return nil +} + +func (d *manifestDiscovery) discoverGoWorkspace() error { + const parentPath = "go.work" + content, exists, err := d.readOptional(parentPath) + if err != nil { + return fmt.Errorf("reading Go workspace configuration: %w", err) + } + if !exists { + return nil + } + + work, err := modfile.ParseWork(parentPath, content, nil) + if err != nil { + return fmt.Errorf("parsing Go workspace configuration: %w", err) + } + members := make([]string, 0, len(work.Use)) + for _, use := range work.Use { + members = append(members, use.Path) + } + if err := d.addWorkspaceManifests(members, nil, "go.mod", parentPath); err != nil { + return fmt.Errorf("discovering Go workspace members: %w", err) + } + return nil +} + +func (d *manifestDiscovery) discoverNPMWorkspace() error { + const parentPath = "package.json" + content, exists, err := d.readOptional(parentPath) + if err != nil { + return fmt.Errorf("reading npm workspace configuration: %w", err) + } + if !exists { + return nil + } + + patterns, err := npmWorkspacePatterns(content) + if err != nil { + return fmt.Errorf("parsing npm workspace configuration: %w", err) + } + if err := d.addWorkspaceManifests(patterns, nil, "package.json", parentPath); err != nil { + return fmt.Errorf("discovering npm workspace members: %w", err) + } + return nil +} + +func npmWorkspacePatterns(content []byte) ([]string, error) { + var config struct { + Workspaces json.RawMessage `json:"workspaces"` + } + if err := json.Unmarshal(content, &config); err != nil { + return nil, err + } + if len(config.Workspaces) == 0 || bytes.Equal(bytes.TrimSpace(config.Workspaces), []byte("null")) { + return nil, nil + } + + var patterns []string + if err := json.Unmarshal(config.Workspaces, &patterns); err == nil { + return patterns, nil + } + var grouped struct { + Packages []string `json:"packages"` + } + if err := json.Unmarshal(config.Workspaces, &grouped); err != nil { + return nil, err + } + return grouped.Packages, nil +} + +func (d *manifestDiscovery) discoverPnpmWorkspace() error { + const parentPath = "pnpm-workspace.yaml" + content, exists, err := d.readOptional(parentPath) + if err != nil { + return fmt.Errorf("reading pnpm workspace configuration: %w", err) + } + if !exists { + return nil + } + + var config struct { + Packages []string `yaml:"packages"` + } + if err := yaml.Unmarshal(content, &config); err != nil { + return fmt.Errorf("parsing pnpm workspace configuration: %w", err) + } + + includes := make([]string, 0, len(config.Packages)) + excludes := make([]string, 0, len(config.Packages)) + for _, pattern := range config.Packages { + if len(pattern) > 0 && pattern[0] == '!' { + excludes = append(excludes, pattern[1:]) + continue + } + includes = append(includes, pattern) + } + if err := d.addWorkspaceManifests(includes, excludes, "package.json", parentPath); err != nil { + return fmt.Errorf("discovering pnpm workspace members: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index 3366f39..5b8d9dd 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,10 @@ go 1.25.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/git-pkgs/pom v0.1.5 github.com/git-pkgs/purl v0.1.15 + golang.org/x/mod v0.38.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 8492996..c70bec3 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f h1:2mT6QcXmMFtvg7bezs1Fef7nnpJyeCmfWoWNjwtvNZ4= github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f/go.mod h1:PLNUetjLa77TCCziPsz0EI8a6CUxgC+1jgmWv0H25tg= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY= github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4= @@ -10,6 +12,8 @@ github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU= github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo= github.com/package-url/packageurl-go v0.1.6 h1:YO3p6u1XmCUliivUg/qWphaY8vI6hxSnnPv7Bfg3m5M= github.com/package-url/packageurl-go v0.1.6/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=