refactor(filemetadata): Implement file metadata store

internal/bootstrap/bootstrap.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rabbitstack/fibratus/pkg/config"
 	"github.com/rabbitstack/fibratus/pkg/filament"
 	"github.com/rabbitstack/fibratus/pkg/filter"
+	"github.com/rabbitstack/fibratus/pkg/fs"
 	"github.com/rabbitstack/fibratus/pkg/handle"
 	"github.com/rabbitstack/fibratus/pkg/ps"
 	"github.com/rabbitstack/fibratus/pkg/rules"
@@ -424,6 +425,7 @@ func (f *App) Shutdown() error {
 	}
 
 	signature.GetSignatures().Close()
+	fs.GetMetadataStore().Close()
 
 	return multierror.Wrap(errs...)
 }

internal/etw/processors/chain_windows.go

@@ -28,7 +28,6 @@ import (
 type Chain struct {
 	processors   []Processor
 	psnapshotter ps.Snapshotter
-	fsProcessor  Processor
 }
 
 // NewChain constructs the processor chain. It arranges all the processors
@@ -49,8 +48,7 @@ func NewChain(
 	chain.addProcessor(newPsProcessor(psnap, vaRegionProber))
 
 	if config.EventSource.EnableFileIOEvents {
-		chain.fsProcessor = newFsProcessor(hsnap, psnap, config)
-		chain.addProcessor(chain.fsProcessor)
+		chain.addProcessor(newFsProcessor(hsnap, psnap, config))
 	}
 	if config.EventSource.EnableRegistryEvents {
 		chain.addProcessor(newRegistryProcessor(hsnap))

internal/etw/processors/fs_windows.go

@@ -29,7 +29,6 @@ import (
 	htypes "github.com/rabbitstack/fibratus/pkg/handle/types"
 	"github.com/rabbitstack/fibratus/pkg/ps"
 	"github.com/rabbitstack/fibratus/pkg/util/signature"
-	"golang.org/x/sys/windows"
 )
 
 var (
@@ -129,11 +128,15 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) {
 			e.AppendEnum(params.FileType, uint32(fileinfo.Type), fs.FileTypes)
 		}
 
-		// invalidate signature cache
-		dispo := e.Params.MustGetUint32(params.FileOperation)
-		if dispo == windows.FILE_OVERWRITE || dispo == windows.FILE_OVERWRITE_IF {
+		// invalidate signature cache / file metadata
+		if e.IsOverwriteDisposition() {
+			fs.GetMetadataStore().RemoveFile(e.GetParamAsString(params.FilePath))
 			signature.GetSignatures().RemoveSignature(e.GetParamAsString(params.FilePath))
 		}
+		// start async file metadata resolution
+		if e.IsCreateDisposition() && e.IsSuccess() {
+			fs.GetMetadataStore().DoRequestAsync(e.GetParamAsString(params.FilePath))
+		}
 
 		return e, nil
 	case event.ReleaseFile:
@@ -175,11 +178,13 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) {
 		if e.IsDeleteFile() {
 			delete(f.files, fileObject)
 			if fileinfo != nil {
+				fs.GetMetadataStore().RemoveFile(fileinfo.Name)
 				signature.GetSignatures().RemoveSignature(fileinfo.Name)
 			}
 		}
 		if e.IsRenameFile() {
 			if fileinfo != nil {
+				fs.GetMetadataStore().RemoveFile(fileinfo.Name)
 				signature.GetSignatures().RemoveSignature(fileinfo.Name)
 			}
 		}

internal/etw/processors/module_windows.go

@@ -21,6 +21,7 @@ package processors
 import (
 	"github.com/rabbitstack/fibratus/pkg/event"
 	"github.com/rabbitstack/fibratus/pkg/event/params"
+	"github.com/rabbitstack/fibratus/pkg/fs"
 	"github.com/rabbitstack/fibratus/pkg/ps"
 	"github.com/rabbitstack/fibratus/pkg/util/signature"
 )
@@ -69,6 +70,9 @@ func (m *moduleProcessor) ProcessEvent(e *event.Event) (*event.Event, bool, erro
 			signature.GetSignatures().DoRequestAsync(key)
 		}
 
+		// request module file metadata by queueing async work
+		fs.GetMetadataStore().DoRequestAsync(e.GetParamAsString(params.ModulePath))
+
 		return e, false, m.psnap.AddModule(e)
 	}
 

pkg/event/event_windows.go

@@ -258,6 +258,12 @@ func (e *Event) IsCreateDisposition() bool {
 	return e.IsCreateFile() && e.Params.MustGetUint32(params.FileOperation) == windows.FILE_CREATE
 }
 
+// IsOverwriteDisposition determines if the file disposition leads to file overwriting.
+func (e *Event) IsOverwriteDisposition() bool {
+	o := e.Params.MustGetUint32(params.FileOperation)
+	return e.IsCreateFile() && (o == windows.FILE_OVERWRITE || o == windows.FILE_OVERWRITE_IF)
+}
+
 // IsOpenDisposition determines if the file disposition leads to opening a file object.
 func (e *Event) IsOpenDisposition() bool {
 	return e.IsCreateFile() && e.Params.MustGetUint32(params.FileOperation) == windows.FILE_OPEN

pkg/filter/accessor_windows.go

@@ -628,7 +628,7 @@ func (t *threadAccessor) Get(f Field, e *event.Event) (params.Value, error) {
 			return nil, nil
 		}
 
-		sign := requestSignature(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp)
+		sign := signature.GetSignatures().DoRequest(signature.MakeKey(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp))
 		if sign == nil {
 			return nil, nil
 		}
@@ -655,7 +655,7 @@ func (t *threadAccessor) Get(f Field, e *event.Event) (params.Value, error) {
 			return nil, nil
 		}
 
-		sign := requestSignature(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp)
+		sign := signature.GetSignatures().DoRequest(signature.MakeKey(mod.Name, mod.Size, mod.Checksum, mod.TimedateStamp))
 		if sign == nil {
 			return nil, nil
 		}
@@ -727,8 +727,20 @@ func (l *fileAccessor) Get(f Field, e *event.Event) (params.Value, error) {
 	case fields.FileViewProtection:
 		return e.GetParamAsString(params.MemProtect), nil
 	case fields.FileIsDLL, fields.FileIsDriver, fields.FileIsExecutable:
+		var file *fs.FileInfo
 		if e.IsCreateDisposition() && e.IsSuccess() {
-			return getFileInfo(f.Name, e)
+			file = fs.GetMetadataStore().DoRequest(e.GetParamAsString(params.FilePath))
+		}
+		if file == nil {
+			return false, nil
+		}
+		switch f.Name {
+		case fields.FileIsDLL:
+			return file.IsDLL, nil
+		case fields.FileIsDriver:
+			return file.IsDriver, nil
+		case fields.FileIsExecutable:
+			return file.IsExecutable, nil
 		}
 		return false, nil
 	case fields.FilePID:
@@ -827,9 +839,25 @@ func (m *moduleAccessor) Get(f Field, e *event.Event) (params.Value, error) {
 	case fields.ImageIsDLL, fields.ModuleIsDLL, fields.ImageIsDriver,
 		fields.ModuleIsDriver, fields.ImageIsExecutable, fields.ModuleIsExecutable,
 		fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet:
+		var file *fs.FileInfo
 		if e.IsLoadModule() {
-			return getFileInfo(f.Name, e)
+			file = fs.GetMetadataStore().DoRequest(e.GetParamAsString(params.ModulePath))
+		}
+		if file == nil {
+			return false, nil
 		}
+
+		switch f.Name {
+		case fields.ImageIsDLL, fields.ModuleIsDLL:
+			return file.IsDLL, nil
+		case fields.ModuleIsDriver, fields.ImageIsDriver:
+			return file.IsDriver, nil
+		case fields.ImageIsExecutable, fields.ModuleIsExecutable:
+			return file.IsExecutable, nil
+		case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet:
+			return file.IsDotnet, nil
+		}
+
 		return false, nil
 	}
 

pkg/filter/filter_test.go

@@ -993,6 +993,8 @@ func TestRegistryFilter(t *testing.T) {
 }
 
 func TestModuleFilter(t *testing.T) {
+	fs.GetMetadataStore().AddFile(filepath.Join(os.Getenv("windir"), "System32", "kernel32.dll"), &fs.FileInfo{IsDLL: true})
+
 	e1 := &event.Event{
 		Type:     event.LoadModule,
 		Category: event.Module,
@@ -1122,7 +1124,7 @@ func TestModuleFilter(t *testing.T) {
 		Type:     event.LoadModule,
 		Category: event.Module,
 		Params: event.Params{
-			params.ModulePath:           {Name: params.ModulePath, Type: params.UnicodeString, Value: "C:\\Windows\\System32\\mscorlib.dll"},
+			params.ModulePath:           {Name: params.ModulePath, Type: params.UnicodeString, Value: "..\\pe\\_fixtures\\mscorlib.dll"},
 			params.ProcessID:            {Name: params.ProcessID, Type: params.PID, Value: uint32(1023)},
 			params.ModuleCheckSum:       {Name: params.ModuleCheckSum, Type: params.Uint32, Value: uint32(2323432)},
 			params.ModuleBase:           {Name: params.ModuleBase, Type: params.Address, Value: uint64(0xfff313833a3)},

pkg/filter/util.go

@@ -20,71 +20,13 @@ package filter
 
 import (
 	"encoding/hex"
-	"fmt"
 	"net"
 	"strings"
 
 	"github.com/rabbitstack/fibratus/pkg/event"
-	"github.com/rabbitstack/fibratus/pkg/event/params"
-	"github.com/rabbitstack/fibratus/pkg/filter/fields"
-	"github.com/rabbitstack/fibratus/pkg/fs"
 	"github.com/rabbitstack/fibratus/pkg/util/bytes"
-	"github.com/rabbitstack/fibratus/pkg/util/signature"
 )
 
-// getFileInfo obtains the file information for created files and loaded modules.
-// Appends the file data to the event parameters, so subsequent field extractions
-// will already have the needed info.
-func getFileInfo(f fields.Field, e *event.Event) (params.Value, error) {
-	switch f {
-	case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL:
-		if e.Params.Contains(params.FileIsDLL) {
-			return e.Params.GetBool(params.FileIsDLL)
-		}
-	case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver:
-		if e.Params.Contains(params.FileIsDriver) {
-			return e.Params.GetBool(params.FileIsDriver)
-		}
-	case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable:
-		if e.Params.Contains(params.FileIsExecutable) {
-			return e.Params.GetBool(params.FileIsExecutable)
-		}
-	case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet:
-		if e.Params.Contains(params.FileIsDotnet) {
-			return e.Params.GetBool(params.FileIsDotnet)
-		}
-	}
-
-	fileinfo, err := fs.GetFileInfo(e.GetParamAsString(params.FilePath))
-	if err != nil {
-		return nil, err
-	}
-
-	e.AppendParam(params.FileIsDLL, params.Bool, fileinfo.IsDLL)
-	e.AppendParam(params.FileIsDriver, params.Bool, fileinfo.IsDriver)
-	e.AppendParam(params.FileIsExecutable, params.Bool, fileinfo.IsExecutable)
-	e.AppendParam(params.FileIsDotnet, params.Bool, fileinfo.IsDotnet)
-
-	switch f {
-	case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL:
-		return fileinfo.IsDLL, nil
-	case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver:
-		return fileinfo.IsDriver, nil
-	case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable:
-		return fileinfo.IsExecutable, nil
-	case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet:
-		return fileinfo.IsDotnet, nil
-	}
-
-	return nil, fmt.Errorf("unexpected field: %s", f)
-}
-
-// requestSignature submits the request for the signature check.
-func requestSignature(path string, size uint64, checksum, timedatestamp uint32) *signature.Signature {
-	key := signature.MakeKey(path, size, checksum, timedatestamp)
-	return signature.GetSignatures().DoRequest(key)
-}
-
 // framePID returns the pid associated with the stack frame.
 func framePID(e *event.Event) uint32 {
 	if !e.Callstack.IsEmpty() && e.Callstack.FrameAt(0).PID != 0 {