diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 8bad7b002..ead9fb3e5 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -31,6 +31,7 @@ import ( "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/filament" "github.com/rabbitstack/fibratus/pkg/filter" + "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/handle" "github.com/rabbitstack/fibratus/pkg/ps" "github.com/rabbitstack/fibratus/pkg/rules" @@ -424,6 +425,7 @@ func (f *App) Shutdown() error { } signature.GetSignatures().Close() + fs.GetMetadataStore().Close() return multierror.Wrap(errs...) } diff --git a/internal/etw/processors/chain_windows.go b/internal/etw/processors/chain_windows.go index ea6514f95..5b8009c22 100644 --- a/internal/etw/processors/chain_windows.go +++ b/internal/etw/processors/chain_windows.go @@ -28,7 +28,6 @@ import ( type Chain struct { processors []Processor psnapshotter ps.Snapshotter - fsProcessor Processor } // NewChain constructs the processor chain. It arranges all the processors @@ -49,8 +48,7 @@ func NewChain( chain.addProcessor(newPsProcessor(psnap, vaRegionProber)) if config.EventSource.EnableFileIOEvents { - chain.fsProcessor = newFsProcessor(hsnap, psnap, config) - chain.addProcessor(chain.fsProcessor) + chain.addProcessor(newFsProcessor(hsnap, psnap, config)) } if config.EventSource.EnableRegistryEvents { chain.addProcessor(newRegistryProcessor(hsnap)) diff --git a/internal/etw/processors/fs_windows.go b/internal/etw/processors/fs_windows.go index 4b9104b7f..7a6b02618 100644 --- a/internal/etw/processors/fs_windows.go +++ b/internal/etw/processors/fs_windows.go @@ -29,7 +29,6 @@ import ( htypes "github.com/rabbitstack/fibratus/pkg/handle/types" "github.com/rabbitstack/fibratus/pkg/ps" "github.com/rabbitstack/fibratus/pkg/util/signature" - "golang.org/x/sys/windows" ) var ( @@ -129,11 +128,15 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) { e.AppendEnum(params.FileType, uint32(fileinfo.Type), fs.FileTypes) } - // invalidate signature cache - dispo := e.Params.MustGetUint32(params.FileOperation) - if dispo == windows.FILE_OVERWRITE || dispo == windows.FILE_OVERWRITE_IF { + // invalidate signature cache / file metadata + if e.IsOverwriteDisposition() { + fs.GetMetadataStore().RemoveFile(e.GetParamAsString(params.FilePath)) signature.GetSignatures().RemoveSignature(e.GetParamAsString(params.FilePath)) } + // start async file metadata resolution + if e.IsCreateDisposition() && e.IsSuccess() { + fs.GetMetadataStore().DoRequestAsync(e.GetParamAsString(params.FilePath)) + } return e, nil case event.ReleaseFile: @@ -175,11 +178,13 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) { if e.IsDeleteFile() { delete(f.files, fileObject) if fileinfo != nil { + fs.GetMetadataStore().RemoveFile(fileinfo.Name) signature.GetSignatures().RemoveSignature(fileinfo.Name) } } if e.IsRenameFile() { if fileinfo != nil { + fs.GetMetadataStore().RemoveFile(fileinfo.Name) signature.GetSignatures().RemoveSignature(fileinfo.Name) } } diff --git a/internal/etw/processors/module_windows.go b/internal/etw/processors/module_windows.go index d9191267c..12b2bd9e9 100644 --- a/internal/etw/processors/module_windows.go +++ b/internal/etw/processors/module_windows.go @@ -21,6 +21,7 @@ package processors import ( "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" + "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/ps" "github.com/rabbitstack/fibratus/pkg/util/signature" ) @@ -69,6 +70,9 @@ func (m *moduleProcessor) ProcessEvent(e *event.Event) (*event.Event, bool, erro signature.GetSignatures().DoRequestAsync(key) } + // request module file metadata by queueing async work + fs.GetMetadataStore().DoRequestAsync(e.GetParamAsString(params.ModulePath)) + return e, false, m.psnap.AddModule(e) } diff --git a/pkg/event/event_windows.go b/pkg/event/event_windows.go index a8e63ac98..09593be4c 100644 --- a/pkg/event/event_windows.go +++ b/pkg/event/event_windows.go @@ -258,6 +258,12 @@ func (e *Event) IsCreateDisposition() bool { return e.IsCreateFile() && e.Params.MustGetUint32(params.FileOperation) == windows.FILE_CREATE } +// IsOverwriteDisposition determines if the file disposition leads to file overwriting. +func (e *Event) IsOverwriteDisposition() bool { + o := e.Params.MustGetUint32(params.FileOperation) + return e.IsCreateFile() && (o == windows.FILE_OVERWRITE || o == windows.FILE_OVERWRITE_IF) +} + // IsOpenDisposition determines if the file disposition leads to opening a file object. func (e *Event) IsOpenDisposition() bool { return e.IsCreateFile() && e.Params.MustGetUint32(params.FileOperation) == windows.FILE_OPEN diff --git a/pkg/filter/accessor_windows.go b/pkg/filter/accessor_windows.go index 100064631..25e60ee7e 100644 --- a/pkg/filter/accessor_windows.go +++ b/pkg/filter/accessor_windows.go @@ -628,7 +628,7 @@ func (t *threadAccessor) Get(f Field, e *event.Event) (params.Value, error) { return nil, nil } - sign := requestSignature(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp) + sign := signature.GetSignatures().DoRequest(signature.MakeKey(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp)) if sign == nil { return nil, nil } @@ -655,7 +655,7 @@ func (t *threadAccessor) Get(f Field, e *event.Event) (params.Value, error) { return nil, nil } - sign := requestSignature(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp) + sign := signature.GetSignatures().DoRequest(signature.MakeKey(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp)) if sign == nil { return nil, nil } @@ -727,8 +727,20 @@ func (l *fileAccessor) Get(f Field, e *event.Event) (params.Value, error) { case fields.FileViewProtection: return e.GetParamAsString(params.MemProtect), nil case fields.FileIsDLL, fields.FileIsDriver, fields.FileIsExecutable: + var file *fs.FileInfo if e.IsCreateDisposition() && e.IsSuccess() { - return getFileInfo(f.Name, e) + file = fs.GetMetadataStore().DoRequest(e.GetParamAsString(params.FilePath)) + } + if file == nil { + return false, nil + } + switch f.Name { + case fields.FileIsDLL: + return file.IsDLL, nil + case fields.FileIsDriver: + return file.IsDriver, nil + case fields.FileIsExecutable: + return file.IsExecutable, nil } return false, nil case fields.FilePID: @@ -827,9 +839,25 @@ func (m *moduleAccessor) Get(f Field, e *event.Event) (params.Value, error) { case fields.ImageIsDLL, fields.ModuleIsDLL, fields.ImageIsDriver, fields.ModuleIsDriver, fields.ImageIsExecutable, fields.ModuleIsExecutable, fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: + var file *fs.FileInfo if e.IsLoadModule() { - return getFileInfo(f.Name, e) + file = fs.GetMetadataStore().DoRequest(e.GetParamAsString(params.ModulePath)) + } + if file == nil { + return false, nil } + + switch f.Name { + case fields.ImageIsDLL, fields.ModuleIsDLL: + return file.IsDLL, nil + case fields.ModuleIsDriver, fields.ImageIsDriver: + return file.IsDriver, nil + case fields.ImageIsExecutable, fields.ModuleIsExecutable: + return file.IsExecutable, nil + case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: + return file.IsDotnet, nil + } + return false, nil } diff --git a/pkg/filter/filter_test.go b/pkg/filter/filter_test.go index 491f65bb5..ecf7ac0d4 100644 --- a/pkg/filter/filter_test.go +++ b/pkg/filter/filter_test.go @@ -993,6 +993,8 @@ func TestRegistryFilter(t *testing.T) { } func TestModuleFilter(t *testing.T) { + fs.GetMetadataStore().AddFile(filepath.Join(os.Getenv("windir"), "System32", "kernel32.dll"), &fs.FileInfo{IsDLL: true}) + e1 := &event.Event{ Type: event.LoadModule, Category: event.Module, @@ -1122,7 +1124,7 @@ func TestModuleFilter(t *testing.T) { Type: event.LoadModule, Category: event.Module, Params: event.Params{ - params.ModulePath: {Name: params.ModulePath, Type: params.UnicodeString, Value: "C:\\Windows\\System32\\mscorlib.dll"}, + params.ModulePath: {Name: params.ModulePath, Type: params.UnicodeString, Value: "..\\pe\\_fixtures\\mscorlib.dll"}, params.ProcessID: {Name: params.ProcessID, Type: params.PID, Value: uint32(1023)}, params.ModuleCheckSum: {Name: params.ModuleCheckSum, Type: params.Uint32, Value: uint32(2323432)}, params.ModuleBase: {Name: params.ModuleBase, Type: params.Address, Value: uint64(0xfff313833a3)}, diff --git a/pkg/filter/util.go b/pkg/filter/util.go index 4fdd60d57..f5562a40a 100644 --- a/pkg/filter/util.go +++ b/pkg/filter/util.go @@ -20,71 +20,13 @@ package filter import ( "encoding/hex" - "fmt" "net" "strings" "github.com/rabbitstack/fibratus/pkg/event" - "github.com/rabbitstack/fibratus/pkg/event/params" - "github.com/rabbitstack/fibratus/pkg/filter/fields" - "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/util/bytes" - "github.com/rabbitstack/fibratus/pkg/util/signature" ) -// getFileInfo obtains the file information for created files and loaded modules. -// Appends the file data to the event parameters, so subsequent field extractions -// will already have the needed info. -func getFileInfo(f fields.Field, e *event.Event) (params.Value, error) { - switch f { - case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL: - if e.Params.Contains(params.FileIsDLL) { - return e.Params.GetBool(params.FileIsDLL) - } - case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver: - if e.Params.Contains(params.FileIsDriver) { - return e.Params.GetBool(params.FileIsDriver) - } - case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable: - if e.Params.Contains(params.FileIsExecutable) { - return e.Params.GetBool(params.FileIsExecutable) - } - case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: - if e.Params.Contains(params.FileIsDotnet) { - return e.Params.GetBool(params.FileIsDotnet) - } - } - - fileinfo, err := fs.GetFileInfo(e.GetParamAsString(params.FilePath)) - if err != nil { - return nil, err - } - - e.AppendParam(params.FileIsDLL, params.Bool, fileinfo.IsDLL) - e.AppendParam(params.FileIsDriver, params.Bool, fileinfo.IsDriver) - e.AppendParam(params.FileIsExecutable, params.Bool, fileinfo.IsExecutable) - e.AppendParam(params.FileIsDotnet, params.Bool, fileinfo.IsDotnet) - - switch f { - case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL: - return fileinfo.IsDLL, nil - case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver: - return fileinfo.IsDriver, nil - case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable: - return fileinfo.IsExecutable, nil - case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: - return fileinfo.IsDotnet, nil - } - - return nil, fmt.Errorf("unexpected field: %s", f) -} - -// requestSignature submits the request for the signature check. -func requestSignature(path string, size uint64, checksum, timedatestamp uint32) *signature.Signature { - key := signature.MakeKey(path, size, checksum, timedatestamp) - return signature.GetSignatures().DoRequest(key) -} - // framePID returns the pid associated with the stack frame. func framePID(e *event.Event) uint32 { if !e.Callstack.IsEmpty() && e.Callstack.FrameAt(0).PID != 0 { diff --git a/pkg/filter/util_test.go b/pkg/filter/util_test.go deleted file mode 100644 index eb4e61c7d..000000000 --- a/pkg/filter/util_test.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2021-present by Nedim Sabic Sabic - * https://www.fibratus.io - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package filter - -import ( - "os" - "path/filepath" - "testing" - - "github.com/rabbitstack/fibratus/pkg/event" - "github.com/rabbitstack/fibratus/pkg/event/params" - "github.com/rabbitstack/fibratus/pkg/filter/fields" - "github.com/stretchr/testify/require" -) - -func TestGetFileInfo(t *testing.T) { - path, err := os.Executable() - require.NoError(t, err) - - var tests = []struct { - e *event.Event - f func(*testing.T, *event.Event) - fld fields.Field - }{ - { - e: &event.Event{ - Name: "CreateFile", - Params: map[string]*event.Param{ - params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: path}, - }, - }, - f: func(t *testing.T, e *event.Event) { - require.True(t, e.Params.MustGetBool(params.FileIsExecutable)) - }, - fld: fields.FileIsExecutable, - }, - { - e: &event.Event{ - Name: "CreateFile", - Params: map[string]*event.Param{ - params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: filepath.Join(os.Getenv("SystemRoot"), "System32", "kernel32.dll")}, - }, - }, - f: func(t *testing.T, e *event.Event) { - require.True(t, e.Params.MustGetBool(params.FileIsDLL)) - }, - fld: fields.ModuleIsDLL, - }, - } - - for _, tt := range tests { - t.Run(tt.e.GetParamAsString(params.FilePath), func(t *testing.T) { - v, err := getFileInfo(tt.fld, tt.e) - require.NotNil(t, v) - require.NoError(t, err) - if tt.f != nil { - tt.f(t, tt.e) - } - }) - } -} diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 202d32bcc..e766e2f99 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -27,11 +27,16 @@ import ( "os" "path/filepath" "strings" + "sync" + "sync/atomic" + "time" "unsafe" + "github.com/golang/groupcache/singleflight" "github.com/rabbitstack/fibratus/pkg/pe" "github.com/rabbitstack/fibratus/pkg/sys" "github.com/rabbitstack/fibratus/pkg/util/wildcard" + log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -52,72 +57,462 @@ const ( devConsole = 0x00000050 ) +// metadataStoreQueueSize is the capacity of the async request channel. +const metadataStoreQueueSize = 1500 + +var fsMetadataAsyncRequestDrops = expvar.NewInt("fs.metadata.async.request.drops") + +var fsMetadataCount = expvar.NewInt("fs.metadata.count") + +var fsMetadataFileParseErrors = expvar.NewMap("fs.metadata.file.parse.errors") + +var fsMetadataCacheHits = expvar.NewInt("fs.metadata.cache.hits") + +var fsMetadataCacheMisses = expvar.NewInt("fs.metadata.cache.misses") + +var fsMetadataEvictions = expvar.NewInt("fs.metadata.evictions") + +// ErrSkippedFile signals the file processing is skipped. +var ErrSkippedFile = func(path string) error { return fmt.Errorf("skipped file: %s", path) } + +var metadataStore *FileMetadataStore +var onceMetadataStore sync.Once + +// metadataTTL is the maximum age of an untouched cache entry before the GC evicts it. +var metadataTTL = 10 * time.Minute + +// windowsUpdateWildcards lists system directories commonly touched during +// updates, servicing, recovery, and component store operations. +// Excluded from metastore because they generate a lot of legitimate file +// activity. +type windowsUpdateWildcards []string + +func (w *windowsUpdateWildcards) Accept(path string) bool { + for _, wc := range *w { + if wildcard.Match(wc, path, false) { + return true + } + } + return false +} + +// moduleWildcards accepts well-known system executable or DLL paths. +type moduleWildcards map[string]struct{} + +var sysroot string +var sysrootOnce sync.Once + +func (w *moduleWildcards) Accept(path string) bool { + sysrootOnce.Do(func() { + sysroot = os.Getenv("SystemRoot") + if sysroot == "" { + sysroot = os.Getenv("SYSTEMROOT") + } + if sysroot == "" { + sysroot = "C:\\Windows" + } + }) + + n := strings.ToLower(filepath.Base(path)) + _, ok := (*w)[n] + if !ok { + return false + } + + return wildcard.Match(filepath.Join(sysroot, "System32", n), path, false) || + wildcard.Match(filepath.Join(sysroot, "Syswow64", n), path, false) +} + // FileInfo represents file metadata. type FileInfo struct { IsExecutable bool IsDLL bool IsDriver bool IsDotnet bool + + // accessed is updated on every cache lookup to drive TTL-based eviction. + accessed atomic.Int64 } -var skippedPatterns = []string{ - `?:\$WinREAgent\Scratch\*`, - `?:\WINDOWS\WinSxS\*`, - `?:\Windows\WinSxS\*`, - `?:\WINDOWS\CbsTemp\*`, - `?:\Windows\CbsTemp\*`, - `?:\WINDOWS\SoftwareDistribution\*`, - `?:\Windows\SoftwareDistribution\*`, +func (f *FileInfo) keepalive() { + f.accessed.Store(time.Now().UnixNano()) } -// ErrSkippedFile signals the file processing is skipped. -var ErrSkippedFile = func(path string) error { return fmt.Errorf("skipped file: %s", path) } +func (f *FileInfo) lastAccessed() time.Time { + return time.Unix(0, f.accessed.Load()) +} + +type Request struct { + Path string + Response chan *FileInfo +} + +func GetMetadataStore() *FileMetadataStore { + onceMetadataStore.Do(func() { + metadataStore = newFileMetadataStore() + }) + + return metadataStore +} + +// FileMetadataStore contains metainfo of PE files derived +// from file creation or DLL loading. Metadata store is +// invalidated on various signals such as file overwriting +// deletion or renaming. +// +// Metadata resolution is asynchronous. File and module events +// insert a pending entry immediately and enqueue a worker job. +type FileMetadataStore struct { + mux sync.RWMutex + files map[string]*FileInfo + + requests chan Request + + stop chan struct{} + group singleflight.Group + + purger *time.Ticker -var parserOpts = []pe.Option{ - pe.WithSections(), - pe.WithSymbols(), - pe.WithCLR(), + windowsUpdateWildcards windowsUpdateWildcards + wellKnownDLLs moduleWildcards + wellKnownExecutables moduleWildcards } -// GetFileInfo returns file metadata for the given path. -// The file metadata consists of information extracted -// from the Portable Executable headers. -func GetFileInfo(path string) (*FileInfo, error) { - for _, pat := range skippedPatterns { - if wildcard.Match(pat, path, false) { - return nil, ErrSkippedFile(path) +func newFileMetadataStore() *FileMetadataStore { + s := &FileMetadataStore{ + files: make(map[string]*FileInfo, 1024), + requests: make(chan Request, metadataStoreQueueSize), + stop: make(chan struct{}), + purger: time.NewTicker(time.Minute), + windowsUpdateWildcards: []string{ + `?:\$winreagent\scratch\*`, + `?:\windows\winsxs\*`, + `?:\windows\cbstemp\*`, + `?:\windows\softwaredistribution\*`, + }, + wellKnownDLLs: map[string]struct{}{ + "ntdll.dll": {}, + "kernel32.dll": {}, + "kernelbase.dll": {}, + "kernel.appcore.dll": {}, + "user32.dll": {}, + "gdi32.dll": {}, + "gdi32full.dll": {}, + "advapi32.dll": {}, + "msvcrt.dll": {}, + "msvcp_win.dll": {}, + "sechost.dll": {}, + "rpcrt4.dll": {}, + "combase.dll": {}, + "ucrtbase.dll": {}, + "win32u.dll": {}, + "bcryptprimitives.dll": {}, + "ole32.dll": {}, + "oleacc.dll": {}, + "oleaut32.dll": {}, + "shell32.dll": {}, + "shlwapi.dll": {}, + "shcore.dll": {}, + "imm32.dll": {}, + "ntmarta.dll": {}, + "setupapi.dll": {}, + "crypt32.dll": {}, + "cryptbase.dll": {}, + "bcrypt.dll": {}, + "ws2_32.dll": {}, + "wintrust.dll": {}, + "netapi32.dll": {}, + "powrprof.dll": {}, + "psapi.dll": {}, + "userenv.dll": {}, + "profapi.dll": {}, + "clbcatq.dll": {}, + "windows.storage.dll": {}, + "uxtheme.dll": {}, + "dwmapi.dll": {}, + "mscoree.dll": {}, + "wintypes.dll": {}, + }, + wellKnownExecutables: map[string]struct{}{ + // core OS / boot processes + "smss.exe": {}, + "csrss.exe": {}, + "wininit.exe": {}, + "winlogon.exe": {}, + "services.exe": {}, + "lsass.exe": {}, + "svchost.exe": {}, + "lsm.exe": {}, + + // desktop / shell + "explorer.exe": {}, + "dwm.exe": {}, + "sihost.exe": {}, + "taskhostw.exe": {}, + "fontdrvhost.exe": {}, + "ctfmon.exe": {}, + "shellexperiencehost.exe": {}, + "startmenuexperiencehost.exe": {}, + "searchhost.exe": {}, + "searchapp.exe": {}, + "searchindexer.exe": {}, + + // common subsystem / broker processes + "runtimebroker.exe": {}, + "dllhost.exe": {}, + "conhost.exe": {}, + "wmiprvse.exe": {}, + "spoolsv.exe": {}, + "taskeng.exe": {}, + "taskhost.exe": {}, + "backgroundtaskhost.exe": {}, + + // security / update related + "smartscreen.exe": {}, + "securityhealthservice.exe": {}, + "securityhealthsystray.exe": {}, + "msmpeng.exe": {}, + "nissrv.exe": {}, + "mpcmdrun.exe": {}, + "trustedinstaller.exe": {}, + "tiworker.exe": {}, + "wuauclt.exe": {}, + "usoclient.exe": {}, + + // remote/session infra + "logonui.exe": {}, + "userinit.exe": {}, + }, + } + + const numWorkers = 6 + for range numWorkers { + go s.runWorker() + } + + go s.runGC() + + return s +} + +// DoRequest submits a file meta request and blocks until the result is ready. +// Use this when the caller must make an allow/deny decision such as +// in the field accessors. +func (s *FileMetadataStore) DoRequest(path string) *FileInfo { + p := s.normalizePath(path) + if f := s.get(p); f != nil { + return f + } + + if s.windowsUpdateWildcards.Accept(p) { + return nil + } + + if s.wellKnownDLLs.Accept(p) { + s.addDLL(p) + return s.get(p) + } + if s.wellKnownExecutables.Accept(p) { + s.addExecutable(p) + return s.get(p) + } + + ch := make(chan *FileInfo, 1) + select { + case s.requests <- Request{Path: p, Response: ch}: + default: + // queue full: fall back to inline check rather than + // dropping a decision that has a security consequence. + return s.getOrParse(p) + } + r := <-ch + return r +} + +func (s *FileMetadataStore) DoRequestAsync(path string) { + p := s.normalizePath(path) + if s.contains(p) { + return + } + + if s.windowsUpdateWildcards.Accept(p) { + return + } + + if s.wellKnownDLLs.Accept(p) { + s.addDLL(p) + return + } + if s.wellKnownExecutables.Accept(p) { + s.addExecutable(p) + return + } + + select { + case s.requests <- Request{Path: p}: + default: + // queue is full + fsMetadataAsyncRequestDrops.Add(1) + } +} + +func (s *FileMetadataStore) Close() { + s.purger.Stop() + close(s.stop) +} + +func (s *FileMetadataStore) AddFile(path string, f *FileInfo) { + s.mux.Lock() + defer s.mux.Unlock() + fsMetadataCount.Add(1) + s.files[path] = f +} + +func (s *FileMetadataStore) RemoveFile(path string) { + p := s.normalizePath(path) + s.mux.Lock() + defer s.mux.Unlock() + delete(s.files, p) + fsMetadataCount.Add(-1) +} + +func (s *FileMetadataStore) runWorker() { + for { + select { + case r := <-s.requests: + s.processRequest(r) + case <-s.stop: + return } } +} + +// gc removes entries that have not been accessed within sigTTL. +// It reads the accessed timestamp via an atomic load, so it does not +// need to hold the write lock while computing ages. +func (s *FileMetadataStore) runGC() { + for { + select { + case <-s.purger.C: + s.gc() + case <-s.stop: + return + } + } +} + +func (s *FileMetadataStore) gc() { + now := time.Now() + + // collect stale files under a read lock to minimize write-lock hold time + s.mux.RLock() + var paths []string + for path, file := range s.files { + if now.Sub(file.lastAccessed()) > metadataTTL { + paths = append(paths, path) + } + } + s.mux.RUnlock() + + if len(paths) == 0 { + return + } + + s.mux.Lock() + for _, path := range paths { + file := s.files[path] + // re-check under the write lock: the entry may have been + // refreshed between the RUnlock above and this Lock + if file != nil && now.Sub(file.lastAccessed()) > metadataTTL { + log.Debugf("evicting file metadata for %s", path) + fsMetadataCount.Add(-1) + fsMetadataEvictions.Add(1) + delete(s.files, path) + } + } + s.mux.Unlock() +} - ext := filepath.Ext(path) - switch strings.ToLower(ext) { - case ".exe": - return &FileInfo{IsExecutable: true}, nil - case ".sys": - return &FileInfo{IsDriver: true}, nil - case ".dll": - pefile, err := pe.ParseFile(path, parserOpts...) +func (s *FileMetadataStore) processRequest(r Request) { + f := s.getOrParse(r.Path) + if r.Response != nil { + r.Response <- f + } +} + +func (s *FileMetadataStore) contains(path string) bool { + return s.get(path) != nil +} + +func (s *FileMetadataStore) addDLL(path string) { + f := &FileInfo{IsDLL: true} + s.AddFile(path, f) +} + +func (s *FileMetadataStore) addExecutable(path string) { + f := &FileInfo{IsExecutable: true} + s.AddFile(path, f) +} + +func (s *FileMetadataStore) get(path string) *FileInfo { + s.mux.RLock() + f := s.files[path] + s.mux.RUnlock() + if f != nil { + fsMetadataCacheHits.Add(1) + f.keepalive() + } + return f +} + +func (s *FileMetadataStore) getOrParse(path string) *FileInfo { + if f := s.get(path); f != nil { + return f + } + + v, err := s.group.Do(path, func() (any, error) { + pe, err := s.parsePE(path) if err != nil { - return &FileInfo{IsDLL: true}, nil + return nil, err } - return &FileInfo{IsDLL: true, IsDotnet: pefile.IsDotnet}, nil + return pe, nil + }) + + if err != nil { + return nil } - pefile, err := pe.ParseFile(path, parserOpts...) + p := v.(*pe.PE) + + f := &FileInfo{} + f.keepalive() + f.IsDLL, f.IsDriver, f.IsExecutable, f.IsDotnet = p.IsDLL, p.IsDriver, p.IsExecutable, p.IsDotnet + s.AddFile(path, f) + fsMetadataCacheMisses.Add(1) + + return f +} + +func (s *FileMetadataStore) parsePE(path string) (*pe.PE, error) { + const size = 8 * 1024 * 1024 // 8MB + data, err := sys.ReadFile(path, size, time.Millisecond*500) if err != nil { + fsMetadataFileParseErrors.Add(err.Error(), 1) return nil, err } - return &FileInfo{ - IsExecutable: pefile.IsExecutable, - IsDLL: pefile.IsDLL, - IsDriver: pefile.IsDriver, - IsDotnet: pefile.IsDotnet, - }, nil + pe, err := pe.ParseBytes(data, pe.WithSections(), pe.WithSymbols(), pe.WithCLR()) + if err != nil { + fsMetadataFileParseErrors.Add(err.Error(), 1) + return nil, err + } + + return pe, err } -// queryVolumeCalls represents the number of times the query volume function was called -var queryVolumeCalls = expvar.NewInt("file.query.volume.info.calls") +func (s *FileMetadataStore) normalizePath(path string) string { + return strings.ToLower(path) +} // GetFileType returns the underlying file type. The opts parameter corresponds to the NtCreateFile CreateOptions argument // that specifies the options to be applied when creating or opening the file. @@ -154,6 +549,9 @@ func GetFileType(filename string, opts uint32) FileType { return getFileTypeFromVolumeInfo(filename) } +// queryVolumeCalls represents the number of times the query volume function was called +var queryVolumeCalls = expvar.NewInt("file.query.volume.info.calls") + func getFileTypeFromVolumeInfo(filename string) FileType { f, err := os.Open(filename) if err != nil { diff --git a/pkg/fs/file_test.go b/pkg/fs/file_test.go index a3aa94ce5..f6e2080ef 100644 --- a/pkg/fs/file_test.go +++ b/pkg/fs/file_test.go @@ -22,12 +22,552 @@ package fs import ( + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) +func sysRoot(t *testing.T) string { + t.Helper() + root := os.Getenv("SystemRoot") + if root == "" { + root = os.Getenv("SYSTEMROOT") + } + if root == "" { + t.Skip("SystemRoot/SYSTEMROOT not set; skipping test that requires a real Windows install") + } + return root +} + +func system32(t *testing.T, name string) string { + t.Helper() + p := filepath.Join(sysRoot(t), "System32", name) + if _, err := os.Stat(p); err != nil { + t.Skipf("required system file not present: %s (%v)", p, err) + } + return p +} + +// newTestStore builds a store without relying on the process-wide singleton +// (GetMetadataStore), so tests don't interfere with each other. +func newTestStore() *FileMetadataStore { + return newFileMetadataStore() +} + +// --------------------------------------------------------------------- +// moduleWildcards (wellKnownDLLs / wellKnownExecutables) — pure string +// matching, no disk I/O required, so these run against synthetic paths. +// --------------------------------------------------------------------- + +func TestModuleWildcardsAcceptTrustedDLL(t *testing.T) { + s := newTestStore() + defer s.Close() + + tests := []struct { + name string + path string + want bool + }{ + {"well-known name, System32", `c:\windows\system32\kernel32.dll`, true}, + {"well-known name, SysWOW64", `c:\windows\syswow64\ntdll.dll`, true}, + {"unknown name, System32", `c:\windows\system32\totally-unknown.dll`, false}, + {"well-known name, wrong directory", `c:\temp\kernel32.dll`, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := s.wellKnownDLLs.Accept(tt.path); got != tt.want { + t.Errorf("Accept(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} + +func TestModuleWildcardsAcceptTrustedExecutable(t *testing.T) { + s := newTestStore() + defer s.Close() + + tests := []struct { + name string + path string + want bool + }{ + {"well-known name, System32", `c:\windows\system32\svchost.exe`, true}, + { + "well-known name but lives outside System32 (explorer.exe is under %SystemRoot%, not System32)", + `c:\windows\explorer.exe`, + false, + }, + {"unknown name, System32", `c:\windows\system32\notepad.exe`, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := s.wellKnownExecutables.Accept(tt.path); got != tt.want { + t.Errorf("Accept(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} + +func TestModuleWildcardsRejectsSiblingDirectorySpoof(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := `c:\windows\system32fake\kernel32.dll` + if s.wellKnownDLLs.Accept(path) { + t.Error("expected sibling-directory path (system32fake) to be rejected") + } +} + +// --------------------------------------------------------------------- +// windowsUpdateWildcards +// --------------------------------------------------------------------- + +func TestWindowsUpdateWildcardsAccept(t *testing.T) { + w := windowsUpdateWildcards{ + `?:\$winreagent\scratch\*`, + `?:\windows\winsxs\*`, + `?:\windows\cbstemp\*`, + `?:\windows\softwaredistribution\*`, + } + + tests := []struct { + path string + want bool + }{ + {`c:\windows\winsxs\amd64_foo\file.dll`, true}, + {`c:\windows\softwaredistribution\download\update.cab`, true}, + {`c:\windows\system32\kernel32.dll`, false}, + {`c:\users\bob\downloads\evil.exe`, false}, + } + + for _, tt := range tests { + if got := w.Accept(tt.path); got != tt.want { + t.Errorf("Accept(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestNormalizePath(t *testing.T) { + s := newTestStore() + defer s.Close() + + got := s.normalizePath(`C:\Windows\System32\KERNEL32.DLL`) + want := `c:\windows\system32\kernel32.dll` + if got != want { + t.Errorf("normalizePath() = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------- +// FileInfo bookkeeping +// --------------------------------------------------------------------- + +func TestFileInfoKeepaliveAndLastAccessed(t *testing.T) { + f := &FileInfo{} + before := time.Now() + f.keepalive() + after := time.Now() + + got := f.lastAccessed() + if got.Before(before.Add(-time.Second)) || got.After(after.Add(time.Second)) { + t.Errorf("lastAccessed() = %v, expected to be between %v and %v", got, before, after) + } +} + +func TestFileMetadataStoreAddAndGet(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + + f := s.get(path) + if f == nil { + t.Fatal("expected entry after addDLL, got nil") + } + if !f.IsDLL { + t.Error("expected IsDLL to be true") + } + if !s.contains(path) { + t.Error("expected contains() to be true") + } +} + +func TestFileMetadataStoreAddExecutable(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\svchost.exe`) + s.addExecutable(path) + + f := s.get(path) + if f == nil { + t.Fatal("expected entry after addExecutable, got nil") + } + if !f.IsExecutable { + t.Error("expected IsExecutable to be true") + } +} + +func TestFileMetadataStoreRemoveFile(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := `C:\Windows\System32\kernel32.dll` + s.addDLL(s.normalizePath(path)) + + if !s.contains(s.normalizePath(path)) { + t.Fatal("expected entry to be present before removal") + } + + s.RemoveFile(path) + + if s.contains(s.normalizePath(path)) { + t.Error("expected entry to be gone after RemoveFile") + } +} + +func TestFileMetadataStoreGetKeepsAlive(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + + f := s.get(path) + old := f.lastAccessed() + + time.Sleep(10 * time.Millisecond) + f2 := s.get(path) + if !f2.lastAccessed().After(old) { + t.Error("expected lastAccessed to advance on repeated get()") + } +} + +func TestDoRequestFastPathTrustedDLL(t *testing.T) { + s := newTestStore() + defer s.Close() + + // Relies on the real SystemRoot literally being named "Windows" + // (true for the overwhelming majority of installs) since the fast + // path no longer derives the root from the environment -- see the + // any-drive-letter/hardcoded-root design flaw noted separately. + path := system32(t, "kernel32.dll") + + f := s.DoRequest(path) + if f == nil { + t.Fatal("expected non-nil FileInfo for trusted DLL fast path") + } + if !f.IsDLL { + t.Error("expected IsDLL true via fast path (well-known name + trusted dir), no PE parsing needed") + } + if f.IsExecutable || f.IsDriver || f.IsDotnet { + t.Errorf("fast path should only set IsDLL, got %+v", f) + } +} + +func TestDoRequestFastPathTrustedExecutable(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := system32(t, "svchost.exe") + + f := s.DoRequest(path) + if f == nil { + t.Fatal("expected non-nil FileInfo for trusted executable fast path") + } + if !f.IsExecutable { + t.Error("expected IsExecutable true via fast path") + } +} + +func TestDoRequestWindowsUpdateWildcardIsSkipped(t *testing.T) { + s := newTestStore() + defer s.Close() + + root := sysRoot(t) + path := filepath.Join(root, "WinSxS", "some-component", "file.dll") + + f := s.DoRequest(path) + if f != nil { + t.Errorf("expected nil for a path matching the WinSxS wildcard, got %+v", f) + } + if s.contains(s.normalizePath(path)) { + t.Error("wildcard-matched paths should never be cached") + } +} + +func TestDoRequestRealPEParsingUnlistedDLL(t *testing.T) { + path := system32(t, "version.dll") + + s := newTestStore() + defer s.Close() + + if _, known := s.wellKnownDLLs[strings.ToLower(filepath.Base(path))]; known { + t.Skip("version.dll is now in wellKnownDLLs; pick another unlisted DLL to keep testing the slow path") + } + + f := s.DoRequest(path) + if f == nil { + t.Fatal("expected a resolved FileInfo from real PE parsing") + } + if !f.IsDLL { + t.Error("expected IsDLL true from parsed PE headers") + } + if f.IsExecutable { + t.Error("did not expect IsExecutable true for a DLL") + } +} + +func TestDoRequestRealPEParsingUnlistedExecutable(t *testing.T) { + path := system32(t, "notepad.exe") + + s := newTestStore() + defer s.Close() + + if _, known := s.wellKnownExecutables[strings.ToLower(filepath.Base(path))]; known { + t.Skip("notepad.exe is now in wellKnownExecutables; pick another unlisted exe to keep testing the slow path") + } + + f := s.DoRequest(path) + if f == nil { + t.Fatal("expected a resolved FileInfo from real PE parsing") + } + if !f.IsExecutable { + t.Error("expected IsExecutable true from parsed PE headers") + } + if f.IsDLL { + t.Error("did not expect IsDLL true for a plain executable") + } +} + +func TestDoRequestRealPEParsingDotnetAssembly(t *testing.T) { + root := sysRoot(t) + + matches, err := filepath.Glob(filepath.Join(root, "Microsoft.NET", "Framework64", "*", "System.IO.dll")) + if err != nil || len(matches) == 0 { + t.Skip("no .NET Framework csc.exe found on this machine") + } + + var path string + for _, c := range matches { + if _, err := os.Stat(c); err == nil { + path = c + break + } + } + if path == "" { + t.Skip("no .NET Framework csc.exe found on this machine") + } + + s := newTestStore() + defer s.Close() + + f := s.DoRequest(path) + if f == nil { + t.Fatal("expected a resolved FileInfo") + } + if !f.IsDotnet { + t.Error("expected IsDotnet true for a managed .NET executable") + } +} + +func TestDoRequestNonexistentFileReturnsNil(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := filepath.Join(sysRoot(t), "System32", "this-file-does-not-exist-12345.dll") + + f := s.DoRequest(path) + if f != nil { + t.Errorf("expected nil FileInfo for a nonexistent file, got %+v", f) + } + if s.contains(s.normalizePath(path)) { + t.Error("a failed parse should not populate the cache") + } +} + +func TestDoRequestAsyncPopulatesCacheEventually(t *testing.T) { + path := system32(t, "version.dll") + + s := newTestStore() + defer s.Close() + + s.DoRequestAsync(path) + + deadline := time.Now().Add(2 * time.Second) + var f *FileInfo + for time.Now().Before(deadline) { + if f = s.get(s.normalizePath(path)); f != nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if f == nil { + t.Fatal("expected DoRequestAsync to eventually populate the cache") + } + if !f.IsDLL { + t.Error("expected IsDLL true") + } +} + +func TestDoRequestAsyncIsNoopWhenAlreadyCached(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + original := s.get(path) + + s.DoRequestAsync(`C:\Windows\System32\kernel32.dll`) + time.Sleep(50 * time.Millisecond) + + if got := s.get(path); got != original { + t.Error("expected DoRequestAsync to leave an already-cached entry untouched") + } +} + +func TestDoRequestConcurrentSameFileSingleflight(t *testing.T) { + path := system32(t, "version.dll") + + s := newTestStore() + defer s.Close() + + const n = 32 + var wg sync.WaitGroup + results := make([]*FileInfo, n) + + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + results[idx] = s.DoRequest(path) + }(i) + } + wg.Wait() + + for i, f := range results { + if f == nil { + t.Fatalf("goroutine %d got nil FileInfo", i) + } + if !f.IsDLL { + t.Errorf("goroutine %d: expected IsDLL true", i) + } + } + if final := s.get(s.normalizePath(path)); final == nil { + t.Fatal("expected a cached entry after concurrent resolution") + } +} + +func TestDoRequestConcurrentDifferentFilesNoRace(t *testing.T) { + root := sysRoot(t) + names := []string{"version.dll", "notepad.exe", "kernel32.dll", "svchost.exe"} + + var paths []string + for _, n := range names { + p := filepath.Join(root, "System32", n) + if _, err := os.Stat(p); err == nil { + paths = append(paths, p) + } + } + if len(paths) == 0 { + t.Skip("none of the candidate files are present") + } + + s := newTestStore() + defer s.Close() + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + for _, p := range paths { + wg.Add(1) + go func(p string) { + defer wg.Done() + _ = s.DoRequest(p) + }(p) + } + } + wg.Wait() +} + +// --------------------------------------------------------------------- +// GC / TTL eviction +// --------------------------------------------------------------------- + +func TestGCEvictsStaleEntries(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + + f := s.get(path) + f.accessed.Store(time.Now().Add(-metadataTTL - time.Minute).UnixNano()) + + s.gc() + + if s.contains(path) { + t.Error("expected stale entry to be evicted by gc()") + } +} + +func TestGCKeepsFreshEntries(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + s.get(path) // refresh accessed timestamp + + s.gc() + + if !s.contains(path) { + t.Error("expected freshly-accessed entry to survive gc()") + } +} + +func TestGCRacesWithConcurrentAccess(t *testing.T) { + s := newTestStore() + defer s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + + var stop atomic.Bool + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + s.get(path) + } + }() + + for i := 0; i < 50; i++ { + s.gc() + } + stop.Store(true) + wg.Wait() +} + +func TestCloseStopsWorkersAndPurger(t *testing.T) { + s := newTestStore() + s.Close() + + path := s.normalizePath(`C:\Windows\System32\kernel32.dll`) + s.addDLL(path) + if !s.contains(path) { + t.Error("fast-path cache writes should still work after Close()") + } +} + func TestGetFileType(t *testing.T) { var tests = []struct { filename string @@ -57,50 +597,3 @@ func TestGetFileType(t *testing.T) { }) } } - -func TestGetFileInfo(t *testing.T) { - var tests = []struct { - path string - fileinfo *FileInfo - err error - }{ - { - `C:\System32\cmd.exe`, - &FileInfo{IsExecutable: true}, - nil, - }, - { - `C:\System32\kernel32.dll`, - &FileInfo{IsDLL: true}, - nil, - }, - { - `C:\Temp\afs.sys`, - &FileInfo{IsDriver: true}, - nil, - }, - { - `../pe/_fixtures/054299e09cea38df2b84e6b29348b418.bin`, - &FileInfo{IsDriver: true}, - nil, - }, - { - `C:\WINDOWS\SoftwareDistribution\Temp\combase.dll`, - nil, - ErrSkippedFile(`C:\WINDOWS\SoftwareDistribution\Temp\combase.dll`), - }, - { - `../pe/_fixtures/mscorlib.dll`, - &FileInfo{IsDLL: true, IsDotnet: true}, - nil, - }, - } - - for _, tt := range tests { - t.Run(tt.path, func(t *testing.T) { - fileinfo, err := GetFileInfo(tt.path) - require.Equal(t, tt.err, err) - assert.Equal(t, tt.fileinfo, fileinfo) - }) - } -} diff --git a/pkg/pe/parser.go b/pkg/pe/parser.go index 9c425062c..d8a42be33 100644 --- a/pkg/pe/parser.go +++ b/pkg/pe/parser.go @@ -402,7 +402,7 @@ func parse(path string, data []byte, options ...Option) (*PE, error) { p.IsDLL = pe.IsDLL() p.IsDriver = p.isDriver() p.IsExecutable = pe.IsEXE() - p.IsDotnet = pe.HasCLR + p.IsDotnet = pe.HasCLR || p.hasManagedImport() p.Anomalies = pe.Anomalies return p, nil @@ -458,3 +458,35 @@ func (pe *PE) isDriver() bool { } return false } + +const ( + mscoreeDLL = "mscoree.dll" + corExeMain = "_CorExeMain" + corDllMain = "_CorDllMain" +) + +// hasManagedImport walks the import descriptor table looking for +// mscoree.dll with the classic unmanaged entry point thunk. This is +// the import that every IJW/Framework-hosted managed PE must carry +// because the OS loader needs a native entry point to bootstrap the +// CLR, even when the COR20 directory has been stripped or relocated +// by a protector. +func (pe *PE) hasManagedImport() bool { + hasMscoree := false + for _, imp := range pe.Imports { + if strings.EqualFold(imp, mscoreeDLL) { + hasMscoree = true + break + } + } + if !hasMscoree { + return false + } + for _, fn := range pe.Symbols { + if fn == corExeMain || fn == corDllMain { + return true + } + } + + return false +} diff --git a/pkg/util/signature/types.go b/pkg/util/signature/types.go index 4c3b51e06..0db5a19a3 100644 --- a/pkg/util/signature/types.go +++ b/pkg/util/signature/types.go @@ -186,6 +186,7 @@ func newSignature(path string, sigType Type, sigLevel Level) *Signature { s := &Signature{ Path: path, } + s.keepalive() s.setType(sigType) s.setStatus(sys.SignatureNotTrusted)