diff --git a/Makefile b/Makefile index 08a7959..9530d24 100644 --- a/Makefile +++ b/Makefile @@ -69,15 +69,15 @@ generate-o11y-check: generate-o11y generate-o11y: @echo "Discovering metric registries..." - @for manifest in $$(find . -name "registry_manifest.yaml" -path "*/monitoring/registry_manifest.yaml" | sort); do \ + @set -e; for manifest in $$(find . -name "registry_manifest.yaml" -path "*/monitoring/registry_manifest.yaml" | sort); do \ registry_path=$$(dirname $$(dirname $$manifest) | sed 's|^\./||'); \ depth=$$(echo "$$registry_path" | tr '/' '\n' | wc -l | tr -d ' '); \ rel_path=""; \ for i in $$(seq 1 $$depth); do rel_path="../$$rel_path"; done; \ echo "Generating metrics for $$registry_path..."; \ - docker run --rm -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry check --registry=./monitoring; \ - docker run --rm -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry generate --registry=./monitoring --templates=$${rel_path}weaver/templates --config=$${rel_path}weaver-go.yaml go .; \ - docker run --rm -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry generate --registry=./monitoring --templates=$${rel_path}weaver/templates --config=$${rel_path}weaver-markdown.yaml markdown .; \ + docker run --rm --user $$(id -u):$$(id -g) -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry check --registry=./monitoring; \ + docker run --rm --user $$(id -u):$$(id -g) -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry generate --registry=./monitoring --templates=$${rel_path}weaver/templates --config=$${rel_path}weaver-go.yaml go .; \ + docker run --rm --user $$(id -u):$$(id -g) -v ${PWD}:/workspace -w /workspace/$$registry_path $(WEAVER_IMAGE) registry generate --registry=./monitoring --templates=$${rel_path}weaver/templates --config=$${rel_path}weaver-markdown.yaml markdown .; \ echo "✓ Generated $$registry_path/monitoring.go and $$registry_path/monitoring.md"; \ done @go fmt ./... diff --git a/cmd/blitz/main.go b/cmd/blitz/main.go index c631130..1f45c61 100644 --- a/cmd/blitz/main.go +++ b/cmd/blitz/main.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/generator/count" gennop "github.com/observiq/blitz/generator/nop" "github.com/observiq/blitz/generator/winevt" @@ -142,6 +143,11 @@ func run(cmd *cobra.Command, args []string) error { cancel() }() + // Blitz routes its own self-telemetry through this bundle. Standalone + // leaves the providers nil so they fall back to the process-global + // provider configured by setupMetrics (Prometheus). + tel := embed.TelemetrySettings{Logger: logger} + // Configure output first var outputInstance output.Output switch cfg.Output.Type { @@ -175,6 +181,7 @@ func run(cmd *cobra.Command, args []string) error { strconv.Itoa(cfg.Output.TCP.Port), cfg.Output.TCP.Workers, tlsConfig, + tel, ) if err != nil { logger.Error("Failed to create TCP output", zap.Error(err)) @@ -186,6 +193,7 @@ func run(cmd *cobra.Command, args []string) error { cfg.Output.UDP.Host, strconv.Itoa(cfg.Output.UDP.Port), cfg.Output.UDP.Workers, + tel, ) if err != nil { logger.Error("Failed to create UDP output", zap.Error(err)) @@ -214,6 +222,7 @@ func run(cmd *cobra.Command, args []string) error { MsgID: cfg.Output.Syslog.MsgID, MaxDatagramBytes: cfg.Output.Syslog.MaxDatagramBytes, TLSConfig: tlsConfig, + Telemetry: tel, } outputInstance, err = syslogout.New(logger, sysCfg) if err != nil { @@ -240,6 +249,7 @@ func run(cmd *cobra.Command, args []string) error { } // Set insecure flag opts = append(opts, otlpgrpc.WithInsecure(cfg.Output.OTLPGrpc.Insecure)) + opts = append(opts, otlpgrpc.WithTelemetry(tel)) // If TLS is enabled and not insecure, set up TLS if cfg.Output.OTLPGrpc.EnableTLS && !cfg.Output.OTLPGrpc.Insecure { var tlsConfig *tls.Config @@ -268,6 +278,7 @@ func run(cmd *cobra.Command, args []string) error { cfg.Output.File.Path, cfg.Output.File.Workers, rot, + tel, ) if err != nil { logger.Error("Failed to create File output", zap.Error(err)) @@ -290,6 +301,7 @@ func run(cmd *cobra.Command, args []string) error { hecout.WithSourceType(cfg.Output.HEC.SourceType), hecout.WithIndex(cfg.Output.HEC.Index), hecout.WithEnableTLS(cfg.Output.HEC.EnableTLS), + hecout.WithTelemetry(tel), } if cfg.Output.HEC.EnableTLS { var tlsConfig *tls.Config @@ -316,7 +328,7 @@ func run(cmd *cobra.Command, args []string) error { var tracker *count.Tracker for _, genCfg := range effectiveGens { - gen, genErr := createGenerator(logger, genCfg, outputInstance) + gen, genErr := createGenerator(logger, genCfg, outputInstance, tel) if genErr != nil { logger.Error("Failed to create generator", zap.String("type", string(genCfg.Type)), @@ -395,7 +407,7 @@ shutdown: return nil } -func createGenerator(logger *zap.Logger, genCfg config.Generator, out output.Output) (any, error) { +func createGenerator(logger *zap.Logger, genCfg config.Generator, out output.Output, tel embed.TelemetrySettings) (any, error) { // Standalone-CLI-only generator types that dispatch.ForEmbed does not // construct (winevt is deprecated for embed; nop yields no records). // All other generators delegate to dispatch.ForEmbed so the @@ -404,7 +416,7 @@ func createGenerator(logger *zap.Logger, genCfg config.Generator, out output.Out case config.GeneratorTypeNop: return gennop.New(logger) case config.GeneratorTypeWinevt: - return winevt.New(logger, genCfg.Winevt.Workers, genCfg.Winevt.Rate) + return winevt.New(logger, genCfg.Winevt.Workers, genCfg.Winevt.Rate, tel) } // All remaining (embed-eligible) types go through the canonical @@ -423,7 +435,7 @@ func createGenerator(logger *zap.Logger, genCfg config.Generator, out output.Out if tw, ok := out.(output.TraceWriter); ok { consumers.TraceConsumer = output.WriterAsTraceConsumer(tw) } - mod, err := dispatch.ForEmbed(logger, genCfg, consumers, nil) + mod, err := dispatch.ForEmbed(logger, genCfg, consumers, nil, tel) if err != nil { return nil, err } diff --git a/config/loader.go b/config/loader.go index 96d35ea..9ab5eb9 100644 --- a/config/loader.go +++ b/config/loader.go @@ -122,6 +122,11 @@ type EmbedOpts struct { // the CLI's BLITZ_* env-var path; blitz never reads os.Environ() // itself in embedded mode. EnvOverrides map[string]string + + // Telemetry carries the OTel providers blitz routes its own + // self-telemetry through, forwarded to every constructed generator. The + // zero value falls back to the process-global providers. + Telemetry embed.TelemetrySettings } // LoadModules parses blitz YAML bytes, constructs the corresponding @@ -156,7 +161,7 @@ func LoadModules(yamlBytes []byte, opts EmbedOpts) ([]embed.ProducerModule, erro } modules := make([]embed.ProducerModule, 0, len(gens)) for i, gen := range gens { - mod, err := dispatch.ForEmbed(logger, gen, consumers, opts.FileGenLibrary) + mod, err := dispatch.ForEmbed(logger, gen, consumers, opts.FileGenLibrary, opts.Telemetry) if err != nil { return nil, fmt.Errorf("generator[%d] type=%q: %w", i, gen.Type, err) } diff --git a/embed/host.go b/embed/host.go index 9a5e137..451b54f 100644 --- a/embed/host.go +++ b/embed/host.go @@ -1,6 +1,10 @@ package embed -import "go.uber.org/zap" +import ( + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) // Host is the bundle of consumers and ambient resources a host process // supplies to an embedded blitz runner. @@ -31,6 +35,21 @@ type Host struct { // treat their own reference as frozen once they hand the Host off. // See cloneResource in this package. Resource map[string]string + + // MeterProvider is the OTel MeterProvider blitz routes its own internal + // metrics through (generator and output self-telemetry). Nil falls back + // to the process-global provider, matching standalone behavior. + MeterProvider metric.MeterProvider + + // TracerProvider is the OTel TracerProvider blitz routes its own internal + // spans through. Nil falls back to the process-global provider. Reserved + // for the self-tracing phase; blitz emits no internal spans yet. + TracerProvider trace.TracerProvider + + // PerBatchSpans enables the higher-volume per-emit-cycle spans once + // self-tracing lands. Off by default; the coarse spans do not depend on + // it. + PerBatchSpans bool } // cloneResource returns a defensive copy of m. Runner.Start uses it so diff --git a/embed/integration_test.go b/embed/integration_test.go index 5a64a2b..67c85fd 100644 --- a/embed/integration_test.go +++ b/embed/integration_test.go @@ -67,7 +67,7 @@ func TestEmbed_ApacheRecordsFlowToMemoryConsumer(t *testing.T) { logger := zaptest.NewLogger(t) consumer := &memoryLogConsumer{} - gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer) + gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) runner, err := embed.New(embed.Config{ @@ -232,7 +232,7 @@ func TestEmbed_HostMetricsPointsFlowToMemoryConsumer(t *testing.T) { func TestEmbed_RunnerRejectsDoubleStart(t *testing.T) { logger := zaptest.NewLogger(t) consumer := &memoryLogConsumer{} - gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer) + gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) runner, err := embed.New(embed.Config{Modules: []embed.ProducerModule{gen}}) diff --git a/embed/new_race_test.go b/embed/new_race_test.go index 09659bd..1b91071 100644 --- a/embed/new_race_test.go +++ b/embed/new_race_test.go @@ -24,7 +24,7 @@ func TestRunner_ConcurrentStartIsSerialized(t *testing.T) { logger := zaptest.NewLogger(t) consumer := &memoryLogConsumer{} - gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer) + gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) runner, err := embed.New(embed.Config{Modules: []embed.ProducerModule{gen}}) @@ -72,7 +72,7 @@ func TestRunner_ConcurrentStopIsIdempotent(t *testing.T) { logger := zaptest.NewLogger(t) consumer := &memoryLogConsumer{} - gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer) + gen, err := apache.New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) runner, err := embed.New(embed.Config{Modules: []embed.ProducerModule{gen}}) diff --git a/generator/apache/apache.go b/generator/apache/apache.go index a52f6fc..d3d267a 100644 --- a/generator/apache/apache.go +++ b/generator/apache/apache.go @@ -46,11 +46,12 @@ type ApacheLogGenerator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Apache log generator. The consumer receives each // generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*ApacheLogGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*ApacheLogGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -63,11 +64,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &ApacheLogGenerator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -83,7 +90,7 @@ func (g *ApacheLogGenerator) Start(_ context.Context) error { zap.Duration("rate", g.rate)) // Record initial active workers count - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) for i := 0; i < g.workers; i++ { g.wg.Add(1) @@ -99,7 +106,7 @@ func (g *ApacheLogGenerator) Stop(ctx context.Context) error { g.logger.Info("Stopping Apache log generator") // Record zero active workers - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -181,7 +188,7 @@ func (g *ApacheLogGenerator) generateAndWriteLog(_ int) error { } // Record logs generated counter - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) // Push as a size-1 batch with timeout ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -393,7 +400,7 @@ func parseApacheCLF(line string) (map[string]any, error) { // recordWriteError records metrics for write errors func (g *ApacheLogGenerator) recordWriteError(errorType string, _ error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) } diff --git a/generator/apache/apache_test.go b/generator/apache/apache_test.go index 675b7a6..75603d4 100644 --- a/generator/apache/apache_test.go +++ b/generator/apache/apache_test.go @@ -85,7 +85,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockConsumer()) + generator, err := New(logger, workers, rate, newMockConsumer(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -97,7 +97,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -106,7 +106,7 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_NilConsumer(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 5, 100*time.Millisecond, nil) + generator, err := New(logger, 5, 100*time.Millisecond, nil, embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -117,13 +117,13 @@ func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) // Test zero workers - generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") // Test negative workers - generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -131,7 +131,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestApacheGenerator_Name(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Equal(t, componentName, generator.Name()) } @@ -144,7 +144,7 @@ var _ embed.ProducerModule = (*ApacheLogGenerator)(nil) func TestApacheGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 2, 50*time.Millisecond, consumer) + generator, err := New(logger, 2, 50*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -176,7 +176,7 @@ func TestApacheGenerator_Start(t *testing.T) { func TestApacheGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 3, 10*time.Millisecond, consumer) + generator, err := New(logger, 3, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -207,7 +207,7 @@ func TestApacheGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() consumer.setConsumeError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -232,7 +232,7 @@ func TestApacheGenerator_WriteErrors_Backoff(t *testing.T) { func TestApacheGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 5, 20*time.Millisecond, consumer) + generator, err := New(logger, 5, 20*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -257,7 +257,7 @@ func TestApacheGenerator_ConcurrentWorkers(t *testing.T) { func TestFormatAsApacheCLF_DefaultLog(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -292,7 +292,7 @@ func TestFormatAsApacheCLF_DefaultLog(t *testing.T) { func TestFormatAsApacheCLF_ParseFunc(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -336,7 +336,7 @@ func (d *discardConsumer) ConsumeLogs(_ context.Context, _ []embed.LogRecord) er func TestApacheLogGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -350,7 +350,7 @@ func TestApacheLogGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - gen, err := New(logger, 2, 10*time.Millisecond, consumer) + gen, err := New(logger, 2, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) @@ -383,7 +383,7 @@ func TestApacheLogGenerator_CountLimited(t *testing.T) { func BenchmarkApacheGenerator(b *testing.B) { logger := zaptest.NewLogger(b) consumer := &discardConsumer{} - generator, err := New(logger, 1, 1*time.Millisecond, consumer) + generator, err := New(logger, 1, 1*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(b, err) err = generator.Start(context.Background()) diff --git a/generator/apache_combined/apache_combined.go b/generator/apache_combined/apache_combined.go index 0deb344..c9727c4 100644 --- a/generator/apache_combined/apache_combined.go +++ b/generator/apache_combined/apache_combined.go @@ -48,11 +48,12 @@ type ApacheCombinedLogGenerator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Apache Combined log generator. The consumer receives // each generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*ApacheCombinedLogGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*ApacheCombinedLogGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -65,11 +66,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &ApacheCombinedLogGenerator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -123,8 +130,8 @@ func (g *ApacheCombinedLogGenerator) SetCountTracker(t *count.Tracker) { func (g *ApacheCombinedLogGenerator) worker(workerID int) { defer g.wg.Done() - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -178,7 +185,7 @@ func (g *ApacheCombinedLogGenerator) generateAndWriteLog(_ int) error { } // Record logs generated counter - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) // Push as a size-1 batch with timeout ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -344,7 +351,7 @@ func formatAsApacheCombined(data *apacheCombinedLogData) (embed.LogRecord, error // recordWriteError records a write error metric func (g *ApacheCombinedLogGenerator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/apache_combined/apache_combined_test.go b/generator/apache_combined/apache_combined_test.go index 34936e9..049612b 100644 --- a/generator/apache_combined/apache_combined_test.go +++ b/generator/apache_combined/apache_combined_test.go @@ -83,7 +83,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockConsumer()) + generator, err := New(logger, workers, rate, newMockConsumer(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -95,7 +95,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -104,7 +104,7 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_NilConsumer(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 5, 100*time.Millisecond, nil) + generator, err := New(logger, 5, 100*time.Millisecond, nil, embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -115,13 +115,13 @@ func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) // Test zero workers - generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") // Test negative workers - generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -129,7 +129,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestApacheCombinedGenerator_Name(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Equal(t, componentName, generator.Name()) } @@ -140,7 +140,7 @@ var _ embed.ProducerModule = (*ApacheCombinedLogGenerator)(nil) func TestApacheCombinedGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 2, 50*time.Millisecond, consumer) + generator, err := New(logger, 2, 50*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -174,7 +174,7 @@ func TestApacheCombinedGenerator_Start(t *testing.T) { func TestApacheCombinedGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 3, 10*time.Millisecond, consumer) + generator, err := New(logger, 3, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -205,7 +205,7 @@ func TestApacheCombinedGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() consumer.setConsumeError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -230,7 +230,7 @@ func TestApacheCombinedGenerator_WriteErrors_Backoff(t *testing.T) { func TestApacheCombinedGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 5, 20*time.Millisecond, consumer) + generator, err := New(logger, 5, 20*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -255,7 +255,7 @@ func TestApacheCombinedGenerator_ConcurrentWorkers(t *testing.T) { func TestFormatAsApacheCombined_Structure(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -293,7 +293,7 @@ func TestFormatAsApacheCombined_Structure(t *testing.T) { func TestFormatAsApacheCombined_ParseFunc(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -327,7 +327,7 @@ func (d *discardConsumer) ConsumeLogs(_ context.Context, _ []embed.LogRecord) er func TestApacheCombinedLogGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -341,7 +341,7 @@ func TestApacheCombinedLogGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - gen, err := New(logger, 2, 10*time.Millisecond, consumer) + gen, err := New(logger, 2, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) @@ -374,7 +374,7 @@ func TestApacheCombinedLogGenerator_CountLimited(t *testing.T) { func BenchmarkApacheCombinedGenerator(b *testing.B) { logger := zaptest.NewLogger(b) consumer := &discardConsumer{} - generator, err := New(logger, 1, 1*time.Millisecond, consumer) + generator, err := New(logger, 1, 1*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(b, err) err = generator.Start(context.Background()) diff --git a/generator/apache_error/apache_error.go b/generator/apache_error/apache_error.go index 9150324..8a61e18 100644 --- a/generator/apache_error/apache_error.go +++ b/generator/apache_error/apache_error.go @@ -45,11 +45,12 @@ type ApacheErrorLogGenerator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Apache Error log generator. The consumer receives // each generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*ApacheErrorLogGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*ApacheErrorLogGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -62,11 +63,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &ApacheErrorLogGenerator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -120,8 +127,8 @@ func (g *ApacheErrorLogGenerator) SetCountTracker(t *count.Tracker) { func (g *ApacheErrorLogGenerator) worker(workerID int) { defer g.wg.Done() - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -175,7 +182,7 @@ func (g *ApacheErrorLogGenerator) generateAndWriteLog(_ int) error { } // Record logs generated counter - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) // Push as a size-1 batch with timeout ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -450,7 +457,7 @@ func formatAsApacheError(data *apacheErrorLogData) (embed.LogRecord, error) { // recordWriteError records a write error metric func (g *ApacheErrorLogGenerator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/apache_error/apache_error_test.go b/generator/apache_error/apache_error_test.go index 5f1bd97..e481161 100644 --- a/generator/apache_error/apache_error_test.go +++ b/generator/apache_error/apache_error_test.go @@ -82,7 +82,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockConsumer()) + generator, err := New(logger, workers, rate, newMockConsumer(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -94,7 +94,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -103,7 +103,7 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_NilConsumer(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 5, 100*time.Millisecond, nil) + generator, err := New(logger, 5, 100*time.Millisecond, nil, embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -114,13 +114,13 @@ func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) // Test zero workers - generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") // Test negative workers - generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -128,7 +128,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestApacheErrorGenerator_Name(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer()) + generator, err := New(logger, 1, 100*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Equal(t, componentName, generator.Name()) } @@ -139,7 +139,7 @@ var _ embed.ProducerModule = (*ApacheErrorLogGenerator)(nil) func TestApacheErrorGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 2, 50*time.Millisecond, consumer) + generator, err := New(logger, 2, 50*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -170,7 +170,7 @@ func TestApacheErrorGenerator_Start(t *testing.T) { func TestApacheErrorGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 3, 10*time.Millisecond, consumer) + generator, err := New(logger, 3, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -201,7 +201,7 @@ func TestApacheErrorGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() consumer.setConsumeError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -226,7 +226,7 @@ func TestApacheErrorGenerator_WriteErrors_Backoff(t *testing.T) { func TestApacheErrorGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 5, 20*time.Millisecond, consumer) + generator, err := New(logger, 5, 20*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -251,7 +251,7 @@ func TestApacheErrorGenerator_ConcurrentWorkers(t *testing.T) { func TestFormatAsApacheError_Structure(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -294,7 +294,7 @@ func TestFormatAsApacheError_Structure(t *testing.T) { func TestFormatAsApacheError_ParseFunc(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - generator, err := New(logger, 1, 10*time.Millisecond, consumer) + generator, err := New(logger, 1, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -322,7 +322,7 @@ func TestFormatAsApacheError_ParseFunc(t *testing.T) { func TestApacheErrorLogGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockConsumer(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -336,7 +336,7 @@ func TestApacheErrorLogGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) consumer := newMockConsumer() - gen, err := New(logger, 2, 10*time.Millisecond, consumer) + gen, err := New(logger, 2, 10*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) @@ -376,7 +376,7 @@ func (d *discardConsumer) ConsumeLogs(_ context.Context, _ []embed.LogRecord) er func BenchmarkApacheErrorGenerator(b *testing.B) { logger := zaptest.NewLogger(b) consumer := &discardConsumer{} - generator, err := New(logger, 1, 1*time.Millisecond, consumer) + generator, err := New(logger, 1, 1*time.Millisecond, consumer, embed.NopTelemetry()) require.NoError(b, err) err = generator.Start(context.Background()) diff --git a/generator/filegen/filegen.go b/generator/filegen/filegen.go index 7f1ec0f..c23ece4 100644 --- a/generator/filegen/filegen.go +++ b/generator/filegen/filegen.go @@ -108,6 +108,8 @@ type FileLogGenerator struct { // File cache cache *Cache + + metrics *generator.Metrics } // New creates a new File log generator. The consumer receives each @@ -119,7 +121,7 @@ type FileLogGenerator struct { // when running blitz embedded inside another binary). When nil, those // sources resolve against the on-disk ./data_library/ directory, which // is the standalone CLI behavior. -func New(logger *zap.Logger, workers int, rate time.Duration, source string, cacheEnabled bool, cacheTTL time.Duration, consumer embed.LogConsumer, dataLibrary fs.FS) (*FileLogGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, source string, cacheEnabled bool, cacheTTL time.Duration, consumer embed.LogConsumer, dataLibrary fs.FS, tel embed.TelemetrySettings) (*FileLogGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -145,6 +147,11 @@ func New(logger *zap.Logger, workers int, rate time.Duration, source string, cac return nil, fmt.Errorf("create cache: %w", err) } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &FileLogGenerator{ logger: logger, workers: workers, @@ -154,6 +161,7 @@ func New(logger *zap.Logger, workers int, rate time.Duration, source string, cac dataLibrary: dataLibrary, stopCh: make(chan struct{}), cache: cache, + metrics: metrics, }, nil } @@ -169,7 +177,7 @@ func (g *FileLogGenerator) Start(_ context.Context) error { zap.Duration("rate", g.rate)) // Record initial active workers count - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) // Get list of files to read files, err := g.getFiles() @@ -196,7 +204,7 @@ func (g *FileLogGenerator) Start(_ context.Context) error { func (g *FileLogGenerator) Stop(ctx context.Context) error { g.logger.Info("Stopping File log generator") - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -470,7 +478,7 @@ func (g *FileLogGenerator) worker(id int, files []string) { err := g.readAndWriteFile(file) if err != nil { g.logger.Error("Error reading file", zap.String("file", file), zap.Error(err)) - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName) // On error, backoff will automatically handle retry timing continue } @@ -532,11 +540,11 @@ func (g *FileLogGenerator) readAndWriteFile(filename string) error { cancel() if err != nil { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName) return fmt.Errorf("write: %w", err) } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) return nil } diff --git a/generator/filegen/filegen_embedded_test.go b/generator/filegen/filegen_embedded_test.go index b9694ed..e8b2c3f 100644 --- a/generator/filegen/filegen_embedded_test.go +++ b/generator/filegen/filegen_embedded_test.go @@ -59,6 +59,7 @@ func TestFilegen_EmbeddedLibrarySource(t *testing.T) { 0, // cache TTL consumer, embeddedlibrary.FS(), + embed.NopTelemetry(), ) require.NoError(t, err) @@ -107,6 +108,7 @@ func TestFilegen_BareNameFallsBackToLibrary(t *testing.T) { 0, consumer, libFS, + embed.NopTelemetry(), ) require.NoError(t, err) @@ -149,6 +151,7 @@ func TestFilegen_PackagePrefixSkipsDiskFallback(t *testing.T) { 0, consumer, libFS, + embed.NopTelemetry(), ) require.NoError(t, err) diff --git a/generator/filegen/filegen_test.go b/generator/filegen/filegen_test.go index 731ffcb..130727e 100644 --- a/generator/filegen/filegen_test.go +++ b/generator/filegen/filegen_test.go @@ -86,14 +86,14 @@ func TestFileLogGenerator_Name(t *testing.T) { defer os.Remove(tmpfile.Name()) tmpfile.Close() - gen, err := New(logger, 1, 100*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil) + gen, err := New(logger, 1, 100*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil, embed.NopTelemetry()) require.NoError(t, err) assert.Equal(t, componentName, gen.Name()) } func TestFileLogGenerator_NilConsumer(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 100*time.Millisecond, "/tmp/test.log", true, 0, nil, nil) + gen, err := New(logger, 1, 100*time.Millisecond, "/tmp/test.log", true, 0, nil, nil, embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, gen) assert.Contains(t, err.Error(), "consumer cannot be nil") @@ -141,7 +141,7 @@ func TestNewFileLogGenerator(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gen, err := New(logger, tt.workers, tt.rate, tt.source, true, 0, newMockConsumer(), nil) + gen, err := New(logger, tt.workers, tt.rate, tt.source, true, 0, newMockConsumer(), nil, embed.NopTelemetry()) if tt.wantErr { assert.Error(t, err) assert.Nil(t, gen) @@ -174,7 +174,7 @@ func TestFileLogGeneratorFileMode(t *testing.T) { tmpfile.Close() consumer := newMockConsumer() - gen, err := New(logger, 1, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil) + gen, err := New(logger, 1, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -220,7 +220,7 @@ func TestFileLogGeneratorDirectoryMode(t *testing.T) { // Test with directory mode (auto-detected) consumer := newMockConsumer() - gen, err := New(logger, 1, 10*time.Millisecond, tmpdir, true, 0, consumer, nil) + gen, err := New(logger, 1, 10*time.Millisecond, tmpdir, true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -243,7 +243,7 @@ func TestFileLogGeneratorDirectoryMode(t *testing.T) { func TestFileLogGeneratorNonexistentFile(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 100*time.Millisecond, "/nonexistent/path/file.log", true, 0, newMockConsumer(), nil) + gen, err := New(logger, 1, 100*time.Millisecond, "/nonexistent/path/file.log", true, 0, newMockConsumer(), nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -262,7 +262,7 @@ func TestFileLogGeneratorStop(t *testing.T) { require.NoError(t, err) tmpfile.Close() - gen, err := New(logger, 1, 100*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil) + gen, err := New(logger, 1, 100*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -288,7 +288,7 @@ func TestFileLogGeneratorWriteError(t *testing.T) { consumer := newMockConsumer() consumer.setConsumeError(errors.New("write failed")) - gen, err := New(logger, 1, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil) + gen, err := New(logger, 1, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -319,7 +319,7 @@ func TestFileLogGeneratorMultipleWorkers(t *testing.T) { tmpfile.Close() consumer := newMockConsumer() - gen, err := New(logger, 3, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil) + gen, err := New(logger, 3, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -347,7 +347,7 @@ func TestTimestampProcessing(t *testing.T) { require.NoError(t, err) tmpFile.Close() - gen, err := New(logger, 1, 100*time.Millisecond, tmpFile.Name(), true, 0, newMockConsumer(), nil) + gen, err := New(logger, 1, 100*time.Millisecond, tmpFile.Name(), true, 0, newMockConsumer(), nil, embed.NopTelemetry()) require.NoError(t, err) testCases := []struct { @@ -518,7 +518,7 @@ func TestFileLogGeneratorGlobPatterns(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { consumer := newMockConsumer() - gen, err := New(logger, 1, 10*time.Millisecond, tc.pattern, true, 0, consumer, nil) + gen, err := New(logger, 1, 10*time.Millisecond, tc.pattern, true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -590,7 +590,7 @@ func TestFileLogGeneratorGlobDirectories(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { consumer := newMockConsumer() - gen, err := New(logger, 1, 10*time.Millisecond, tc.pattern, true, 0, consumer, nil) + gen, err := New(logger, 1, 10*time.Millisecond, tc.pattern, true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) err = gen.Start(context.Background()) @@ -711,7 +711,7 @@ func TestFileLogGenerator_SetCountTracker(t *testing.T) { } tmpfile.Close() - gen, err := New(logger, 1, 50*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil) + gen, err := New(logger, 1, 50*time.Millisecond, tmpfile.Name(), true, 0, newMockConsumer(), nil, embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -736,7 +736,7 @@ func TestFileLogGenerator_CountLimited(t *testing.T) { consumer := newMockConsumer() - gen, err := New(logger, 2, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil) + gen, err := New(logger, 2, 10*time.Millisecond, tmpfile.Name(), true, 0, consumer, nil, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/generator/hostmetrics/hostmetrics.go b/generator/hostmetrics/hostmetrics.go index 0cd072c..967cc9e 100644 --- a/generator/hostmetrics/hostmetrics.go +++ b/generator/hostmetrics/hostmetrics.go @@ -47,6 +47,10 @@ type Config struct { // data by default; pass a positive value explicitly for // reproducibility. Seed int64 + // Telemetry supplies the OTel providers blitz routes its own + // self-telemetry through. The zero value falls back to the process + // global providers. + Telemetry embed.TelemetrySettings } // Generator implements embed.ProducerModule for host metrics. @@ -61,6 +65,7 @@ type Generator struct { scrapers []Scraper consumer embed.MetricConsumer seed int64 + metrics *generator.Metrics wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker @@ -87,6 +92,11 @@ func New(cfg Config) (*Generator, error) { return nil, fmt.Errorf("rate must be greater than 0, got %s", cfg.Rate) } + metrics, err := generator.NewMetrics(cfg.Telemetry.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + hostname := cfg.Hostname if hostname == "" { // Hostname-only RNG; intentionally seeded once at construction @@ -115,6 +125,7 @@ func New(cfg Config) (*Generator, error) { scrapers: buildScrapers(cfg.ScraperNames), consumer: cfg.Consumer, seed: cfg.Seed, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -137,7 +148,7 @@ func (g *Generator) Start(_ context.Context) error { zap.Int("scrapers", len(g.scrapers)), ) - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), generatorType) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), generatorType) for i := range g.workers { g.wg.Add(1) @@ -153,7 +164,7 @@ func (g *Generator) Stop(ctx context.Context) error { close(g.stopCh) - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, generatorType) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, generatorType) done := make(chan struct{}) go func() { @@ -231,12 +242,12 @@ func (g *Generator) scrape(r *rand.Rand) { zap.String("scraper", scraper.Name()), zap.Error(err), ) - generator.BlitzGeneratorWriteErrorsCounter.Add(ctx, 1, generatorType, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(ctx, 1, generatorType, metric.WithAttributeSet(attribute.NewSet(attribute.String("scraper", scraper.Name()))), ) continue } - generator.BlitzGeneratorEntriesCounter.Add(ctx, int64(len(points)), generatorType) + g.metrics.BlitzGeneratorEntriesCounter.Add(ctx, int64(len(points)), generatorType) } } diff --git a/generator/json/json.go b/generator/json/json.go index f54549b..c946be6 100644 --- a/generator/json/json.go +++ b/generator/json/json.go @@ -84,11 +84,12 @@ type JSONLogGenerator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new JSON log generator. The consumer receives each // generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, logType string, consumer embed.LogConsumer) (*JSONLogGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, logType string, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*JSONLogGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -111,12 +112,18 @@ func New(logger *zap.Logger, workers int, rate time.Duration, logType string, co return nil, fmt.Errorf("logType must be one of: %s, %s, got %q", LogTypeDefault, LogTypePII, logType) } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &JSONLogGenerator{ logger: logger, workers: workers, rate: rate, logType: logType, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -132,7 +139,7 @@ func (g *JSONLogGenerator) Start(_ context.Context) error { zap.Duration("rate", g.rate)) // Record initial active workers count - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) for i := 0; i < g.workers; i++ { g.wg.Add(1) @@ -148,7 +155,7 @@ func (g *JSONLogGenerator) Stop(ctx context.Context) error { g.logger.Info("Stopping JSON log generator") // Record zero active workers - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -246,7 +253,7 @@ func (g *JSONLogGenerator) generateAndWriteLog(_ int) error { } // Record logs generated counter - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) // Write the data with timeout ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -327,7 +334,7 @@ func formatAsJSON(data logtypes.LogData) (embed.LogRecord, error) { // recordWriteError records metrics for write errors func (g *JSONLogGenerator) recordWriteError(errorType string, _ error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) } diff --git a/generator/json/json_test.go b/generator/json/json_test.go index 1a5b5c4..027d0e8 100644 --- a/generator/json/json_test.go +++ b/generator/json/json_test.go @@ -97,7 +97,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, "default", newMockWriter()) + generator, err := New(logger, workers, rate, "default", newMockWriter(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -108,7 +108,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, "default", newMockWriter()) + generator, err := New(nil, 5, 100*time.Millisecond, "default", newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -119,13 +119,13 @@ func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) // Test zero workers - generator, err := New(logger, 0, 100*time.Millisecond, "default", newMockWriter()) + generator, err := New(logger, 0, 100*time.Millisecond, "default", newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") // Test negative workers - generator, err = New(logger, -1, 100*time.Millisecond, "default", newMockWriter()) + generator, err = New(logger, -1, 100*time.Millisecond, "default", newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -134,7 +134,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestJSONGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 2, 50*time.Millisecond, "default", writer) + generator, err := New(logger, 2, 50*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -173,7 +173,7 @@ func TestJSONGenerator_Start(t *testing.T) { func TestJSONGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 3, 10*time.Millisecond, "default", writer) + generator, err := New(logger, 3, 10*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -203,7 +203,7 @@ func TestJSONGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() writer.setWriteError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, "default", writer) + generator, err := New(logger, 1, 10*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -228,7 +228,7 @@ func TestJSONGenerator_WriteErrors_Backoff(t *testing.T) { func TestJSONGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 5, 20*time.Millisecond, "default", writer) + generator, err := New(logger, 5, 20*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -263,7 +263,7 @@ func TestJSONGenerator_ConcurrentWorkers(t *testing.T) { func TestJSONGenerator_LogMessageVariety(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 5*time.Millisecond, "default", writer) + generator, err := New(logger, 1, 5*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -310,7 +310,7 @@ func TestJSONGenerator_LogMessageVariety(t *testing.T) { func TestJSONGenerator_LogMessageSize(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 10*time.Millisecond, "default", writer) + generator, err := New(logger, 1, 10*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -349,7 +349,7 @@ func TestJSONGenerator_MultipleStartStop(t *testing.T) { // Start and stop multiple times with new generator instances for range 3 { - generator, err := New(logger, 2, 20*time.Millisecond, "default", writer) + generator, err := New(logger, 2, 20*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -374,7 +374,7 @@ func TestJSONGenerator_MultipleStartStop(t *testing.T) { func TestNew_ZeroWorkers(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 0, 10*time.Millisecond, "default", newMockWriter()) + generator, err := New(logger, 0, 10*time.Millisecond, "default", newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -384,7 +384,7 @@ func TestNew_ZeroWorkers(t *testing.T) { func TestJSONGenerator_VeryFastRate(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 1*time.Millisecond, "default", writer) + generator, err := New(logger, 1, 1*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -408,7 +408,7 @@ func TestJSONGenerator_VeryFastRate(t *testing.T) { func TestJSONGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, "default", newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, "default", newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -422,7 +422,7 @@ func TestJSONGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, "default", writer) + gen, err := New(logger, 2, 10*time.Millisecond, "default", writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/generator/kubernetes/kubernetes.go b/generator/kubernetes/kubernetes.go index 80bd1ad..1a6f9dc 100644 --- a/generator/kubernetes/kubernetes.go +++ b/generator/kubernetes/kubernetes.go @@ -60,11 +60,12 @@ type Generator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Kubernetes container log generator. The consumer // receives each generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, format string, consumer embed.LogConsumer) (*Generator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, format string, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -85,12 +86,18 @@ func New(logger *zap.Logger, workers int, rate time.Duration, format string, con return nil, fmt.Errorf("unsupported container log format: %s, must be one of: %s", format, formatCRIO) } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &Generator{ logger: logger, workers: workers, rate: rate, format: logFormat, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -144,8 +151,8 @@ func (g *Generator) SetCountTracker(t *count.Tracker) { func (g *Generator) worker(workerID int) { defer g.wg.Done() - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -202,7 +209,7 @@ func (g *Generator) generateAndWriteLog(_ int) error { }, } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -435,7 +442,7 @@ func parseContainerLog(message string) (map[string]any, error) { // recordWriteError records a write error metric func (g *Generator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/kubernetes/kubernetes_test.go b/generator/kubernetes/kubernetes_test.go index f24aa3d..bc60db7 100644 --- a/generator/kubernetes/kubernetes_test.go +++ b/generator/kubernetes/kubernetes_test.go @@ -42,14 +42,14 @@ func (m *mockWriter) getWrites() [][]byte { func TestGenerator_Name(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Equal(t, componentName, gen.Name()) } func TestGenerator_NilConsumer(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", nil) + gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", nil, embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, gen) assert.Contains(t, err.Error(), "consumer cannot be nil") @@ -57,7 +57,7 @@ func TestGenerator_NilConsumer(t *testing.T) { func TestGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, "cri-o", newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -71,7 +71,7 @@ func TestGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, "cri-o", writer) + gen, err := New(logger, 2, 10*time.Millisecond, "cri-o", writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/generator/metrics_routing_test.go b/generator/metrics_routing_test.go new file mode 100644 index 0000000..39b1f15 --- /dev/null +++ b/generator/metrics_routing_test.go @@ -0,0 +1,55 @@ +package generator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// TestNewMetrics_recordsToProvidedProvider is the core contract of the +// self-telemetry refactor: instruments built by NewMetrics record to the +// caller-supplied MeterProvider, not the process global. +func TestNewMetrics_recordsToProvidedProvider(t *testing.T) { + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + + m, err := NewMetrics(mp) + require.NoError(t, err) + + m.BlitzGeneratorEntriesCounter.Add(context.Background(), 5, "json") + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + + var got int64 + found := false + for _, sm := range rm.ScopeMetrics { + for _, md := range sm.Metrics { + if md.Name == "blitz.generator.entries" { + sum, ok := md.Data.(metricdata.Sum[int64]) + require.True(t, ok, "blitz.generator.entries should be an int64 sum") + require.Len(t, sum.DataPoints, 1) + got = sum.DataPoints[0].Value + found = true + } + } + } + require.True(t, found, "expected blitz.generator.entries in the provided provider's output") + require.Equal(t, int64(5), got) +} + +// TestNewMetrics_nilProviderFallsBackToGlobal documents that a nil provider is +// safe and uses the process global, preserving standalone behavior. +func TestNewMetrics_nilProviderFallsBackToGlobal(t *testing.T) { + m, err := NewMetrics(nil) + require.NoError(t, err) + require.NotNil(t, m) + + // Recording must not panic against the global provider. + require.NotPanics(t, func() { + m.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, "json") + }) +} diff --git a/generator/monitoring.go b/generator/monitoring.go index 8b201df..a145532 100644 --- a/generator/monitoring.go +++ b/generator/monitoring.go @@ -7,7 +7,6 @@ package generator import ( "context" "errors" - "fmt" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -20,63 +19,63 @@ const ( attrGeneratorType = "generator_type" ) -// ErrorTypeValue is the type of the error_type enum attribute -type ErrorTypeValue string +// error_typeValue is the type of the error_type enum attribute +type errorTypeValue string // Constants for enum members const ( - ErrorTypeUnknown ErrorTypeValue = "unknown" - ErrorTypeTimeout ErrorTypeValue = "timeout" + errorTypeUnknown errorTypeValue = "unknown" + errorTypeTimeout errorTypeValue = "timeout" ) // Wrapper types for metrics with required attributes -// BlitzGeneratorEntriesCounterType wraps the blitz.generator.entries metric with type-safe required attributes -type BlitzGeneratorEntriesCounterType struct { - counter metric.Int64Counter +// blitzGeneratorActiveWorkersGaugeType wraps the blitz.generator.active_workers metric with type-safe required attributes +type blitzGeneratorActiveWorkersGaugeType struct { + gauge metric.Int64Gauge } -// Add records a value for blitz.generator.entries with required attributes -func (m BlitzGeneratorEntriesCounterType) Add(ctx context.Context, value int64, generatorType string, opts ...metric.AddOption) { +// Record records a value for blitz.generator.active_workers with required attributes +func (m blitzGeneratorActiveWorkersGaugeType) Record(ctx context.Context, value int64, generatorType string, opts ...metric.RecordOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrGeneratorType, generatorType), )) if len(opts) == 0 { - m.counter.Add(ctx, value, attrs) + m.gauge.Record(ctx, value, attrs) } else { - allOpts := append([]metric.AddOption{attrs}, opts...) - m.counter.Add(ctx, value, allOpts...) + allOpts := append([]metric.RecordOption{attrs}, opts...) + m.gauge.Record(ctx, value, allOpts...) } } -// BlitzGeneratorActiveWorkersGaugeType wraps the blitz.generator.active_workers metric with type-safe required attributes -type BlitzGeneratorActiveWorkersGaugeType struct { - gauge metric.Int64Gauge +// blitzGeneratorEntriesCounterType wraps the blitz.generator.entries metric with type-safe required attributes +type blitzGeneratorEntriesCounterType struct { + counter metric.Int64Counter } -// Record records a value for blitz.generator.active_workers with required attributes -func (m BlitzGeneratorActiveWorkersGaugeType) Record(ctx context.Context, value int64, generatorType string, opts ...metric.RecordOption) { +// Add records a value for blitz.generator.entries with required attributes +func (m blitzGeneratorEntriesCounterType) Add(ctx context.Context, value int64, generatorType string, opts ...metric.AddOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrGeneratorType, generatorType), )) if len(opts) == 0 { - m.gauge.Record(ctx, value, attrs) + m.counter.Add(ctx, value, attrs) } else { - allOpts := append([]metric.RecordOption{attrs}, opts...) - m.gauge.Record(ctx, value, allOpts...) + allOpts := append([]metric.AddOption{attrs}, opts...) + m.counter.Add(ctx, value, allOpts...) } } -// BlitzGeneratorWriteErrorsCounterType wraps the blitz.generator.write_errors metric with type-safe required attributes -type BlitzGeneratorWriteErrorsCounterType struct { +// blitzGeneratorWriteErrorsCounterType wraps the blitz.generator.write_errors metric with type-safe required attributes +type blitzGeneratorWriteErrorsCounterType struct { counter metric.Int64Counter } // Add records a value for blitz.generator.write_errors with required attributes -func (m BlitzGeneratorWriteErrorsCounterType) Add(ctx context.Context, value int64, generatorType string, opts ...metric.AddOption) { +func (m blitzGeneratorWriteErrorsCounterType) Add(ctx context.Context, value int64, generatorType string, opts ...metric.AddOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrGeneratorType, generatorType), )) @@ -89,51 +88,60 @@ func (m BlitzGeneratorWriteErrorsCounterType) Add(ctx context.Context, value int } } +// Attribute value options (provider-independent) var ( - generatorMeter = otel.Meter("generator") + attrErrorTypeUnknown = attribute.String(attrErrorType, string(errorTypeUnknown)) + attrErrorTypeTimeout = attribute.String(attrErrorType, string(errorTypeTimeout)) +) - // BlitzGeneratorEntriesCounter tracks the total number of telemetry entries generated - BlitzGeneratorEntriesCounter BlitzGeneratorEntriesCounterType +// Metrics holds blitz's generator self-telemetry instruments. Build one +// with NewMetrics from a caller-supplied MeterProvider so metrics can be routed +// to any provider (an embedding host's, or the process global). +type Metrics struct { + generatorMeter metric.Meter + + // number of active worker goroutines + BlitzGeneratorActiveWorkersGauge blitzGeneratorActiveWorkersGaugeType + // total number of telemetry entries generated + BlitzGeneratorEntriesCounter blitzGeneratorEntriesCounterType + // total number of write errors + BlitzGeneratorWriteErrorsCounter blitzGeneratorWriteErrorsCounterType +} - // BlitzGeneratorActiveWorkersGauge tracks the number of active worker goroutines - BlitzGeneratorActiveWorkersGauge BlitzGeneratorActiveWorkersGaugeType +// NewMetrics builds the generator instruments from mp. A nil mp falls +// back to the process-global MeterProvider, preserving standalone behavior. +func NewMetrics(mp metric.MeterProvider) (*Metrics, error) { + if mp == nil { + mp = otel.GetMeterProvider() + } + m := &Metrics{} + var errs error - // BlitzGeneratorWriteErrorsCounter tracks the total number of write errors - BlitzGeneratorWriteErrorsCounter BlitzGeneratorWriteErrorsCounterType + m.generatorMeter = mp.Meter("generator") - // error_type - AttrErrorTypeUnknown = attribute.String(attrErrorType, string(ErrorTypeUnknown)) - AttrErrorTypeTimeout = attribute.String(attrErrorType, string(ErrorTypeTimeout)) -) - -func init() { - var err, errs error + BlitzGeneratorActiveWorkersGaugeRaw, err := m.generatorMeter.Int64Gauge( + "blitz.generator.active_workers", + metric.WithDescription("number of active worker goroutines"), + metric.WithUnit("{worker}"), + ) + errs = errors.Join(errs, err) + m.BlitzGeneratorActiveWorkersGauge = blitzGeneratorActiveWorkersGaugeType{gauge: BlitzGeneratorActiveWorkersGaugeRaw} - blitzGeneratorEntriesCounterRaw, err := generatorMeter.Int64Counter( + BlitzGeneratorEntriesCounterRaw, err := m.generatorMeter.Int64Counter( "blitz.generator.entries", metric.WithDescription("total number of telemetry entries generated"), metric.WithUnit("{entry}"), ) errs = errors.Join(errs, err) - BlitzGeneratorEntriesCounter = BlitzGeneratorEntriesCounterType{counter: blitzGeneratorEntriesCounterRaw} + m.BlitzGeneratorEntriesCounter = blitzGeneratorEntriesCounterType{counter: BlitzGeneratorEntriesCounterRaw} - blitzGeneratorActiveWorkersGaugeRaw, err := generatorMeter.Int64Gauge( - "blitz.generator.active_workers", - metric.WithDescription("number of active worker goroutines"), - metric.WithUnit("{worker}"), - ) - errs = errors.Join(errs, err) - BlitzGeneratorActiveWorkersGauge = BlitzGeneratorActiveWorkersGaugeType{gauge: blitzGeneratorActiveWorkersGaugeRaw} - - blitzGeneratorWriteErrorsCounterRaw, err := generatorMeter.Int64Counter( + BlitzGeneratorWriteErrorsCounterRaw, err := m.generatorMeter.Int64Counter( "blitz.generator.write_errors", metric.WithDescription("total number of write errors"), metric.WithUnit("{error}"), ) errs = errors.Join(errs, err) - BlitzGeneratorWriteErrorsCounter = BlitzGeneratorWriteErrorsCounterType{counter: blitzGeneratorWriteErrorsCounterRaw} + m.BlitzGeneratorWriteErrorsCounter = blitzGeneratorWriteErrorsCounterType{counter: BlitzGeneratorWriteErrorsCounterRaw} - if errs != nil { - panic(fmt.Sprintf("Initialize generator metrics: %s", errs)) - } + return m, errs } diff --git a/generator/monitoring.md b/generator/monitoring.md index a6ae90f..58bfa40 100644 --- a/generator/monitoring.md +++ b/generator/monitoring.md @@ -4,24 +4,24 @@ | Metric | Type | Unit | Description | |--------|------|------|-------------| +| [`blitz.generator.active_workers`](#blitzgeneratoractive-workers) | Gauge | `{worker}` | number of active worker goroutines | | [`blitz.generator.entries`](#blitzgeneratorentries) | Counter | `{entry}` | total number of telemetry entries generated | -| [`blitz.generator.active_workers`](#blitzgeneratoractive-workers) | Counter | `{worker}` | number of active worker goroutines | | [`blitz.generator.write_errors`](#blitzgeneratorwrite-errors) | Counter | `{error}` | total number of write errors | --- ## Metrics Detail -### blitz.generator.entries +### blitz.generator.active_workers | Property | Value | |----------|-------| -| **Type** | Counter | -| **Unit** | `{entry}` | +| **Type** | Gauge | +| **Unit** | `{worker}` | | **Meter** | `generator` | | **Stability** | Stable | -| **Description** | total number of telemetry entries generated | -| **Attributes** | `generator_type` | +| **Description** | number of active worker goroutines | +| **Attributes** |`generator_type` | **Attributes:** @@ -32,21 +32,21 @@ **Usage:** ```go // Type-safe wrapper with required attributes -blitzGeneratorEntriesCounter.Add(ctx, 1, generatorTypeValue) +blitzGeneratorActiveWorkersGauge.Add(ctx, 1, generatorTypeValue) ``` --- -### blitz.generator.active_workers +### blitz.generator.entries | Property | Value | |----------|-------| -| **Type** | Gauge | -| **Unit** | `{worker}` | +| **Type** | Counter | +| **Unit** | `{entry}` | | **Meter** | `generator` | | **Stability** | Stable | -| **Description** | number of active worker goroutines | -| **Attributes** | `generator_type` | +| **Description** | total number of telemetry entries generated | +| **Attributes** |`generator_type` | **Attributes:** @@ -57,7 +57,7 @@ blitzGeneratorEntriesCounter.Add(ctx, 1, generatorTypeValue) **Usage:** ```go // Type-safe wrapper with required attributes -blitzGeneratorActiveWorkersGauge.Record(ctx, 1, generatorTypeValue) +blitzGeneratorEntriesCounter.Add(ctx, 1, generatorTypeValue) ``` --- @@ -71,7 +71,7 @@ blitzGeneratorActiveWorkersGauge.Record(ctx, 1, generatorTypeValue) | **Meter** | `generator` | | **Stability** | Stable | | **Description** | total number of write errors | -| **Attributes** | `generator_type`, `error_type` | +| **Attributes** |`generator_type`, `error_type` | **Attributes:** @@ -89,6 +89,7 @@ blitzGeneratorWriteErrorsCounter.Add(ctx, 1, generatorTypeValue) --- + --- -**Generated:** `make generate-o11y` | **Registry:** `generator/monitoring/` | **Templates:** `weaver/templates/` +**Generated:** `make generate-o11y` | **Registry:** `generator/monitoring/` | **Templates:** `weaver/templates/` \ No newline at end of file diff --git a/generator/nginx/nginx.go b/generator/nginx/nginx.go index 58226a2..12b6068 100644 --- a/generator/nginx/nginx.go +++ b/generator/nginx/nginx.go @@ -68,6 +68,7 @@ type Generator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // remoteUsers is nginx-specific user values used in the $remote_user log field; @@ -76,7 +77,7 @@ var remoteUsers = []string{"-", "admin", "user1", "user2", "guest"} // New creates a new NGINX log generator. The consumer receives each // generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*Generator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -89,11 +90,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &Generator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -147,8 +154,8 @@ func (g *Generator) SetCountTracker(t *count.Tracker) { func (g *Generator) worker(workerID int) { defer g.wg.Done() - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -199,7 +206,7 @@ func (g *Generator) generateAndWriteLog(_ int) error { return fmt.Errorf("format log as NGINX Combined: %w", err) } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -354,7 +361,7 @@ func formatAsNginxCombined(data *nginxLogData) (embed.LogRecord, error) { // recordWriteError records a write error metric func (g *Generator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/nginx/nginx_test.go b/generator/nginx/nginx_test.go index 107af4f..e560379 100644 --- a/generator/nginx/nginx_test.go +++ b/generator/nginx/nginx_test.go @@ -88,7 +88,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockWriter()) + generator, err := New(logger, workers, rate, newMockWriter(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -99,7 +99,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -109,12 +109,12 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") - generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -123,7 +123,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestNginxGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 2, 50*time.Millisecond, writer) + generator, err := New(logger, 2, 50*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -153,7 +153,7 @@ func TestNginxGenerator_Start(t *testing.T) { func TestNginxGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 3, 10*time.Millisecond, writer) + generator, err := New(logger, 3, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -182,7 +182,7 @@ func TestNginxGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() writer.setWriteError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -205,7 +205,7 @@ func TestNginxGenerator_WriteErrors_Backoff(t *testing.T) { func TestNginxGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 5, 20*time.Millisecond, writer) + generator, err := New(logger, 5, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -228,7 +228,7 @@ func TestNginxGenerator_ConcurrentWorkers(t *testing.T) { func TestFormatAsNginxCombined_Structure(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -262,7 +262,7 @@ func TestFormatAsNginxCombined_Structure(t *testing.T) { func TestFormatAsNginxCombined_ParseFunc(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -289,7 +289,7 @@ func TestFormatAsNginxCombined_ParseFunc(t *testing.T) { // discardWriter implements output.Writer for benchmarking - discards all data func TestGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -303,7 +303,7 @@ func TestGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, writer) + gen, err := New(logger, 2, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) @@ -342,7 +342,7 @@ func (d *discardWriter) ConsumeLogs(_ context.Context, _ []embed.LogRecord) erro func BenchmarkNginxGenerator(b *testing.B) { logger := zaptest.NewLogger(b) writer := &discardWriter{} - generator, err := New(logger, 1, 1*time.Millisecond, writer) + generator, err := New(logger, 1, 1*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(b, err) err = generator.Start(context.Background()) diff --git a/generator/okta/okta.go b/generator/okta/okta.go index f313a84..8bfe212 100644 --- a/generator/okta/okta.go +++ b/generator/okta/okta.go @@ -37,6 +37,7 @@ type Generator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // Predefined data for realistic Okta log generation @@ -189,7 +190,7 @@ var ( // New creates a new Okta log generator. The consumer receives each // generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*Generator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -202,11 +203,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &Generator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -261,8 +268,8 @@ func (g *Generator) worker(workerID int) { r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(workerID))) // #nosec G404 - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -305,7 +312,7 @@ func (g *Generator) generateAndWriteLog(r *rand.Rand, _ int) error { return fmt.Errorf("generate Okta log: %w", err) } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -465,7 +472,7 @@ func generateRandomIP(r *rand.Rand) string { } func (g *Generator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/okta/okta_test.go b/generator/okta/okta_test.go index 532f27e..8cdb042 100644 --- a/generator/okta/okta_test.go +++ b/generator/okta/okta_test.go @@ -73,7 +73,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockWriter()) + generator, err := New(logger, workers, rate, newMockWriter(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -84,7 +84,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -94,12 +94,12 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") - generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -108,7 +108,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestOktaGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 2, 50*time.Millisecond, writer) + generator, err := New(logger, 2, 50*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -155,7 +155,7 @@ func TestOktaGenerator_Start(t *testing.T) { func TestOktaGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 3, 10*time.Millisecond, writer) + generator, err := New(logger, 3, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -185,7 +185,7 @@ func TestOktaGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() writer.setWriteError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -208,7 +208,7 @@ func TestOktaGenerator_WriteErrors_Backoff(t *testing.T) { func TestOktaGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 5, 20*time.Millisecond, writer) + generator, err := New(logger, 5, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -231,7 +231,7 @@ func TestOktaGenerator_ConcurrentWorkers(t *testing.T) { func TestOktaGenerator_EventTypeVariety(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 5*time.Millisecond, writer) + generator, err := New(logger, 1, 5*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -274,7 +274,7 @@ func TestOktaGenerator_FailureOutcomeHasReason(t *testing.T) { logger := zaptest.NewLogger(t) r := rand.New(rand.NewSource(42)) // #nosec G404 - generator, err := New(logger, 1, time.Second, newMockWriter()) + generator, err := New(logger, 1, time.Second, newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) // Generate enough logs to get some failures @@ -304,7 +304,7 @@ func TestOktaGenerator_MultipleStartStop(t *testing.T) { writer := newMockWriter() for range 3 { - generator, err := New(logger, 2, 20*time.Millisecond, writer) + generator, err := New(logger, 2, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -367,7 +367,7 @@ func TestGenerateRandomIP(t *testing.T) { func TestGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -381,7 +381,7 @@ func TestGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, writer) + gen, err := New(logger, 2, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/generator/paloalto/paloalto.go b/generator/paloalto/paloalto.go index 37a1f98..8451299 100644 --- a/generator/paloalto/paloalto.go +++ b/generator/paloalto/paloalto.go @@ -37,11 +37,12 @@ type Generator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Palo Alto generator. The consumer receives each // generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*Generator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -52,11 +53,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &Generator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -72,7 +79,7 @@ func (g *Generator) Start(_ context.Context) error { zap.Duration("rate", g.rate), ) - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) for i := 0; i < g.workers; i++ { g.wg.Add(1) @@ -85,7 +92,7 @@ func (g *Generator) Start(_ context.Context) error { func (g *Generator) Stop(ctx context.Context) error { g.logger.Info("Stopping Palo Alto generator") - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -147,7 +154,7 @@ func (g *Generator) worker(workerID int) { func (g *Generator) generateAndWrite(_ int) error { line := generatePaloAltoLog() - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -172,7 +179,7 @@ func (g *Generator) generateAndWrite(_ int) error { } func (g *Generator) recordWriteError(errorType string, _ error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) } diff --git a/generator/paloalto/paloalto_test.go b/generator/paloalto/paloalto_test.go index 584beb4..172bb68 100644 --- a/generator/paloalto/paloalto_test.go +++ b/generator/paloalto/paloalto_test.go @@ -87,7 +87,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockWriter()) + generator, err := New(logger, workers, rate, newMockWriter(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -98,7 +98,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -109,13 +109,13 @@ func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) // Test zero workers - generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") // Test negative workers - generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -124,7 +124,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 2, 50*time.Millisecond, writer) + generator, err := New(logger, 2, 50*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -145,7 +145,7 @@ func TestGenerator_Start(t *testing.T) { func TestGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 3, 10*time.Millisecond, writer) + generator, err := New(logger, 3, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -175,7 +175,7 @@ func TestGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() writer.setWriteError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -197,7 +197,7 @@ func TestGenerator_WriteErrors_Backoff(t *testing.T) { func TestGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 5, 20*time.Millisecond, writer) + generator, err := New(logger, 5, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -296,7 +296,7 @@ func TestGenerator_MultipleStartStop(t *testing.T) { // cycle must produce at least one additional record before the // generator is stopped — otherwise the cycle proved nothing. for range 3 { - generator, err := New(logger, 2, 20*time.Millisecond, writer) + generator, err := New(logger, 2, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -317,7 +317,7 @@ func TestGenerator_MultipleStartStop(t *testing.T) { func TestGenerator_VeryFastRate(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 1*time.Millisecond, writer) + generator, err := New(logger, 1, 1*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -338,7 +338,7 @@ func TestGenerator_VeryFastRate(t *testing.T) { func TestGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -352,7 +352,7 @@ func TestGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, writer) + gen, err := New(logger, 2, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/generator/postgres/postgres.go b/generator/postgres/postgres.go index 9eba378..cf5b5a2 100644 --- a/generator/postgres/postgres.go +++ b/generator/postgres/postgres.go @@ -64,6 +64,7 @@ type Generator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // Predefined lists for fast random generation @@ -228,7 +229,7 @@ var ( // New creates a new PostgreSQL log generator. The consumer receives // each generated record as a size-1 batch via ConsumeLogs. -func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer) (*Generator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.LogConsumer, tel embed.TelemetrySettings) (*Generator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -241,11 +242,17 @@ func New(logger *zap.Logger, workers int, rate time.Duration, consumer embed.Log return nil, fmt.Errorf("consumer cannot be nil") } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &Generator{ logger: logger, workers: workers, rate: rate, consumer: consumer, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -299,8 +306,8 @@ func (g *Generator) SetCountTracker(t *count.Tracker) { func (g *Generator) worker(workerID int) { defer g.wg.Done() - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) - defer generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 1, componentName) + defer g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), 0, componentName) backoffConfig := backoff.NewExponentialBackOff() backoffConfig.InitialInterval = g.rate @@ -350,7 +357,7 @@ func (g *Generator) generateAndWriteLog(_ int) error { return fmt.Errorf("format log as PostgreSQL: %w", err) } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -504,7 +511,7 @@ func formatAsPostgres(data *postgresLogData) (embed.LogRecord, error) { // recordWriteError records a write error metric func (g *Generator) recordWriteError(errorType string, err error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) g.logger.Debug("Recorded write error", diff --git a/generator/postgres/postgres_test.go b/generator/postgres/postgres_test.go index b2c80ae..20eb32a 100644 --- a/generator/postgres/postgres_test.go +++ b/generator/postgres/postgres_test.go @@ -88,7 +88,7 @@ func TestNew(t *testing.T) { workers := 5 rate := 100 * time.Millisecond - generator, err := New(logger, workers, rate, newMockWriter()) + generator, err := New(logger, workers, rate, newMockWriter(), embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, generator) @@ -99,7 +99,7 @@ func TestNew(t *testing.T) { } func TestNew_NilLogger(t *testing.T) { - generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter()) + generator, err := New(nil, 5, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) @@ -109,12 +109,12 @@ func TestNew_NilLogger(t *testing.T) { func TestNew_InvalidWorkers(t *testing.T) { logger := zaptest.NewLogger(t) - generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter()) + generator, err := New(logger, 0, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") - generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter()) + generator, err = New(logger, -1, 100*time.Millisecond, newMockWriter(), embed.NopTelemetry()) assert.Error(t, err) assert.Nil(t, generator) assert.Contains(t, err.Error(), "workers must be 1 or greater") @@ -123,7 +123,7 @@ func TestNew_InvalidWorkers(t *testing.T) { func TestPostgresGenerator_Start(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 2, 50*time.Millisecond, writer) + generator, err := New(logger, 2, 50*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -154,7 +154,7 @@ func TestPostgresGenerator_Start(t *testing.T) { func TestPostgresGenerator_Stop_GracefulShutdown(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 3, 10*time.Millisecond, writer) + generator, err := New(logger, 3, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -179,7 +179,7 @@ func TestPostgresGenerator_WriteErrors_Backoff(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() writer.setWriteError(errors.New("write failed")) - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -198,7 +198,7 @@ func TestPostgresGenerator_WriteErrors_Backoff(t *testing.T) { func TestPostgresGenerator_ConcurrentWorkers(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 5, 20*time.Millisecond, writer) + generator, err := New(logger, 5, 20*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -217,7 +217,7 @@ func TestPostgresGenerator_ConcurrentWorkers(t *testing.T) { func TestFormatAsPostgres_Structure(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -246,7 +246,7 @@ func TestFormatAsPostgres_Structure(t *testing.T) { func TestFormatAsPostgres_ParseFunc(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - generator, err := New(logger, 1, 10*time.Millisecond, writer) + generator, err := New(logger, 1, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) err = generator.Start(context.Background()) @@ -272,7 +272,7 @@ func TestFormatAsPostgres_ParseFunc(t *testing.T) { // discardWriter implements output.Writer for benchmarking - discards all data func TestGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter()) + gen, err := New(logger, 1, 50*time.Millisecond, newMockWriter(), embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -286,7 +286,7 @@ func TestGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond, writer) + gen, err := New(logger, 2, 10*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) @@ -324,7 +324,7 @@ func (d *discardWriter) ConsumeLogs(_ context.Context, _ []embed.LogRecord) erro func BenchmarkPostgresGenerator(b *testing.B) { logger := zaptest.NewLogger(b) writer := &discardWriter{} - generator, err := New(logger, 1, 1*time.Millisecond, writer) + generator, err := New(logger, 1, 1*time.Millisecond, writer, embed.NopTelemetry()) require.NoError(b, err) err = generator.Start(context.Background()) diff --git a/generator/traces/traces.go b/generator/traces/traces.go index ed64582..5922209 100644 --- a/generator/traces/traces.go +++ b/generator/traces/traces.go @@ -67,6 +67,10 @@ type Config struct { // uniqueness across blitz instances that share a Seed but // participate in the same downstream pipeline. Seed int64 + // Telemetry supplies the OTel providers blitz routes its own + // self-telemetry through. The zero value falls back to the process + // global providers. + Telemetry embed.TelemetrySettings } // Generator implements embed.ProducerModule for synthetic distributed @@ -92,6 +96,7 @@ type Generator struct { hostname string consumer embed.TraceConsumer seed int64 + metrics *generator.Metrics wg sync.WaitGroup stopCh chan struct{} @@ -118,6 +123,11 @@ func New(cfg Config) (*Generator, error) { return nil, fmt.Errorf("rate must be greater than 0, got %s", cfg.Rate) } + metrics, err := generator.NewMetrics(cfg.Telemetry.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + hostname := cfg.Hostname if hostname == "" { // Hostname-only RNG; seeded once at construction since the @@ -140,6 +150,7 @@ func New(cfg Config) (*Generator, error) { hostname: hostname, consumer: cfg.Consumer, seed: cfg.Seed, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -166,7 +177,7 @@ func (g *Generator) Start(_ context.Context) error { zap.String("hostname", g.hostname), ) - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), generatorType) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), generatorType) for i := range g.workers { g.wg.Add(1) @@ -204,7 +215,7 @@ func (g *Generator) Stop(ctx context.Context) error { close(g.stopCh) - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, generatorType) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, generatorType) drainCtx, cancel := context.WithTimeout(ctx, stopDrainTimeout) defer cancel() @@ -437,12 +448,12 @@ func (g *Generator) emitSpan(sp embed.Span) { ctx := context.Background() if err := g.consumer.ConsumeTraces(ctx, []embed.Span{sp}); err != nil { g.logger.Debug("Consume trace error", zap.Error(err)) - generator.BlitzGeneratorWriteErrorsCounter.Add(ctx, 1, generatorType, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(ctx, 1, generatorType, metric.WithAttributeSet(attribute.NewSet(attribute.String("span_kind", string(sp.Kind)))), ) return } - generator.BlitzGeneratorEntriesCounter.Add(ctx, 1, generatorType) + g.metrics.BlitzGeneratorEntriesCounter.Add(ctx, 1, generatorType) } // cloneResource returns a defensive copy so per-span mutations (e.g. a diff --git a/generator/wel/wel.go b/generator/wel/wel.go index 6fc3ab9..574678f 100644 --- a/generator/wel/wel.go +++ b/generator/wel/wel.go @@ -42,6 +42,8 @@ type Generator struct { state *catalog.StateTracker opts *catalog.GenerateOpts + metrics *generator.Metrics + wg sync.WaitGroup stopCh chan struct{} @@ -61,6 +63,11 @@ type Config struct { Channels []string Consumer embed.LogConsumer + // Telemetry supplies the OTel providers blitz routes its own + // self-telemetry through. The zero value falls back to the process + // global providers. + Telemetry embed.TelemetrySettings + // Environment data — caller-owned slices. The generator stores the // reference; callers must not mutate the underlying arrays after // passing them in. @@ -101,6 +108,11 @@ func New(cfg Config) (*Generator, error) { return nil, fmt.Errorf("no events available for role %q with channels %v", cfg.Role, cfg.Channels) } + metrics, err := generator.NewMetrics(cfg.Telemetry.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + state := catalog.NewStateTracker(1000) // Defensive clone of the caller-supplied slices. Workers read from @@ -128,6 +140,7 @@ func New(cfg Config) (*Generator, error) { registry: reg, state: state, opts: opts, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -145,7 +158,7 @@ func (g *Generator) Start(_ context.Context) error { zap.Strings("channels", g.channels), ) - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) for i := 0; i < g.workers; i++ { g.wg.Add(1) @@ -158,7 +171,7 @@ func (g *Generator) Start(_ context.Context) error { func (g *Generator) Stop(ctx context.Context) error { g.logger.Info("Stopping WEL generator") - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -228,7 +241,7 @@ func (g *Generator) generateAndWrite(rng *rand.Rand) error { // Render as XML for the consumer xml := record.ToXML() - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("channel", record.Channel))), ) @@ -253,7 +266,7 @@ func (g *Generator) generateAndWrite(rng *rand.Rand) error { if ctx.Err() == context.DeadlineExceeded { errorType = "timeout" } - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) return err diff --git a/generator/winevt/winevt.go b/generator/winevt/winevt.go index dfc4bec..f123e60 100644 --- a/generator/winevt/winevt.go +++ b/generator/winevt/winevt.go @@ -31,10 +31,11 @@ type WinevtGenerator struct { wg sync.WaitGroup stopCh chan struct{} tracker *count.Tracker + metrics *generator.Metrics } // New creates a new Windows Event generator. -func New(logger *zap.Logger, workers int, rate time.Duration) (*WinevtGenerator, error) { +func New(logger *zap.Logger, workers int, rate time.Duration, tel embed.TelemetrySettings) (*WinevtGenerator, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -42,10 +43,16 @@ func New(logger *zap.Logger, workers int, rate time.Duration) (*WinevtGenerator, return nil, fmt.Errorf("workers must be 1 or greater, got %d", workers) } + metrics, err := generator.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build generator metrics: %w", err) + } + return &WinevtGenerator{ logger: logger, workers: workers, rate: rate, + metrics: metrics, stopCh: make(chan struct{}), }, nil } @@ -57,7 +64,7 @@ func (g *WinevtGenerator) Start(writer output.Writer) error { zap.Duration("rate", g.rate), ) - generator.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(context.Background(), int64(g.workers), componentName) for i := 0; i < g.workers; i++ { g.wg.Add(1) @@ -70,7 +77,7 @@ func (g *WinevtGenerator) Start(writer output.Writer) error { func (g *WinevtGenerator) Stop(ctx context.Context) error { g.logger.Info("Stopping Windows Event generator") - generator.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) + g.metrics.BlitzGeneratorActiveWorkersGauge.Record(ctx, 0, componentName) close(g.stopCh) @@ -136,7 +143,7 @@ func (g *WinevtGenerator) generateAndWrite(writer output.Writer, workerID int) e return fmt.Errorf("render template: %w", err) } - generator.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) + g.metrics.BlitzGeneratorEntriesCounter.Add(context.Background(), 1, componentName) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -160,7 +167,7 @@ func (g *WinevtGenerator) generateAndWrite(writer output.Writer, workerID int) e } func (g *WinevtGenerator) recordWriteError(errorType string, _ error) { - generator.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, + g.metrics.BlitzGeneratorWriteErrorsCounter.Add(context.Background(), 1, componentName, metric.WithAttributeSet(attribute.NewSet(attribute.String("error_type", errorType))), ) } diff --git a/generator/winevt/winevt_test.go b/generator/winevt/winevt_test.go index 9de1021..83a3d2b 100644 --- a/generator/winevt/winevt_test.go +++ b/generator/winevt/winevt_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/generator/count" "github.com/observiq/blitz/internal/generators/winevt/templates" "github.com/observiq/blitz/output" @@ -42,7 +43,7 @@ func (m *mockWriter) getWrites() [][]byte { func TestNew(t *testing.T) { logger := zaptest.NewLogger(t) - g, err := New(logger, 2, 50*time.Millisecond) + g, err := New(logger, 2, 50*time.Millisecond, embed.NopTelemetry()) assert.NoError(t, err) assert.NotNil(t, g) } @@ -50,7 +51,7 @@ func TestNew(t *testing.T) { func TestWinevtGenerator_GeneratesAndWrites(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - g, err := New(logger, 2, 20*time.Millisecond) + g, err := New(logger, 2, 20*time.Millisecond, embed.NopTelemetry()) require.NoError(t, err) err = g.Start(writer) @@ -124,7 +125,7 @@ func TestRenderTemplate_RandomSelection(t *testing.T) { func TestWinevtGenerator_SetCountTracker(t *testing.T) { logger := zaptest.NewLogger(t) - gen, err := New(logger, 1, 50*time.Millisecond) + gen, err := New(logger, 1, 50*time.Millisecond, embed.NopTelemetry()) require.NoError(t, err) assert.Nil(t, gen.tracker, "tracker should be nil initially") @@ -138,7 +139,7 @@ func TestWinevtGenerator_CountLimited(t *testing.T) { logger := zaptest.NewLogger(t) writer := newMockWriter() - gen, err := New(logger, 2, 10*time.Millisecond) + gen, err := New(logger, 2, 10*time.Millisecond, embed.NopTelemetry()) require.NoError(t, err) tracker := count.NewTracker(5) diff --git a/internal/dispatch/embed.go b/internal/dispatch/embed.go index 557cc1d..81a297d 100644 --- a/internal/dispatch/embed.go +++ b/internal/dispatch/embed.go @@ -75,7 +75,7 @@ func (c EmbedConsumers) requireTrace(typ config.GeneratorType) error { // consumer that is nil in `consumers` (e.g. hostmetrics without a // MetricConsumer, traces without a TraceConsumer), and for generator // types that are not embed-eligible at all (nop, winevt — see PIPE-1032). -func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsumers, fileGenLibrary fs.FS) (embed.ProducerModule, error) { +func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsumers, fileGenLibrary fs.FS, tel embed.TelemetrySettings) (embed.ProducerModule, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -84,52 +84,52 @@ func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsum if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return jsongen.New(logger, genCfg.JSON.Workers, genCfg.JSON.Rate, genCfg.JSON.Type, consumers.LogConsumer) + return jsongen.New(logger, genCfg.JSON.Workers, genCfg.JSON.Rate, genCfg.JSON.Type, consumers.LogConsumer, tel) case config.GeneratorTypePaloAlto: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return paloalto.New(logger, genCfg.PaloAlto.Workers, genCfg.PaloAlto.Rate, consumers.LogConsumer) + return paloalto.New(logger, genCfg.PaloAlto.Workers, genCfg.PaloAlto.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeApache: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return apachegen.New(logger, genCfg.Apache.Workers, genCfg.Apache.Rate, consumers.LogConsumer) + return apachegen.New(logger, genCfg.Apache.Workers, genCfg.Apache.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeApacheCombined: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return apachecombinedgen.New(logger, genCfg.ApacheCombined.Workers, genCfg.ApacheCombined.Rate, consumers.LogConsumer) + return apachecombinedgen.New(logger, genCfg.ApacheCombined.Workers, genCfg.ApacheCombined.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeApacheError: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return apacheerrorgen.New(logger, genCfg.ApacheError.Workers, genCfg.ApacheError.Rate, consumers.LogConsumer) + return apacheerrorgen.New(logger, genCfg.ApacheError.Workers, genCfg.ApacheError.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeNginx: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return nginx.New(logger, genCfg.Nginx.Workers, genCfg.Nginx.Rate, consumers.LogConsumer) + return nginx.New(logger, genCfg.Nginx.Workers, genCfg.Nginx.Rate, consumers.LogConsumer, tel) case config.GeneratorTypePostgres: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return postgres.New(logger, genCfg.Postgres.Workers, genCfg.Postgres.Rate, consumers.LogConsumer) + return postgres.New(logger, genCfg.Postgres.Workers, genCfg.Postgres.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeKubernetes: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return kubernetes.New(logger, genCfg.Kubernetes.Workers, genCfg.Kubernetes.Rate, genCfg.Kubernetes.Format, consumers.LogConsumer) + return kubernetes.New(logger, genCfg.Kubernetes.Workers, genCfg.Kubernetes.Rate, genCfg.Kubernetes.Format, consumers.LogConsumer, tel) case config.GeneratorTypeFile: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return filegen.New(logger, genCfg.Filegen.Workers, genCfg.Filegen.Rate, genCfg.Filegen.Source, genCfg.Filegen.CacheEnabled, genCfg.Filegen.CacheTTL, consumers.LogConsumer, fileGenLibrary) + return filegen.New(logger, genCfg.Filegen.Workers, genCfg.Filegen.Rate, genCfg.Filegen.Source, genCfg.Filegen.CacheEnabled, genCfg.Filegen.CacheTTL, consumers.LogConsumer, fileGenLibrary, tel) case config.GeneratorTypeOkta: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err } - return okta.New(logger, genCfg.Okta.Workers, genCfg.Okta.Rate, consumers.LogConsumer) + return okta.New(logger, genCfg.Okta.Workers, genCfg.Okta.Rate, consumers.LogConsumer, tel) case config.GeneratorTypeWel: if err := consumers.requireLog(genCfg.Type); err != nil { return nil, err @@ -139,14 +139,15 @@ func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsum role = welcatalog.RoleMember } return wel.New(wel.Config{ - Logger: logger, - Workers: genCfg.Wel.Workers, - Rate: genCfg.Wel.Rate, - Computer: genCfg.Wel.Computer, - Domain: genCfg.Wel.Domain, - Role: role, - Channels: genCfg.Wel.Channels, - Consumer: consumers.LogConsumer, + Logger: logger, + Workers: genCfg.Wel.Workers, + Rate: genCfg.Wel.Rate, + Computer: genCfg.Wel.Computer, + Domain: genCfg.Wel.Domain, + Role: role, + Channels: genCfg.Wel.Channels, + Consumer: consumers.LogConsumer, + Telemetry: tel, }) case config.GeneratorTypeFIX: if err := consumers.requireLog(genCfg.Type); err != nil { @@ -166,18 +167,20 @@ func ForEmbed(logger *zap.Logger, genCfg config.Generator, consumers EmbedConsum ScraperNames: genCfg.HostMetrics.Scrapers, Consumer: consumers.MetricConsumer, Seed: yamlSeedDefault(genCfg.HostMetrics.Seed), + Telemetry: tel, }) case config.GeneratorTypeTraces: if err := consumers.requireTrace(genCfg.Type); err != nil { return nil, err } return tracesgen.New(tracesgen.Config{ - Logger: logger, - Workers: genCfg.Traces.Workers, - Rate: genCfg.Traces.Rate, - Hostname: genCfg.Traces.Hostname, - Consumer: consumers.TraceConsumer, - Seed: yamlSeedDefault(genCfg.Traces.Seed), + Logger: logger, + Workers: genCfg.Traces.Workers, + Rate: genCfg.Traces.Rate, + Hostname: genCfg.Traces.Hostname, + Consumer: consumers.TraceConsumer, + Seed: yamlSeedDefault(genCfg.Traces.Seed), + Telemetry: tel, }) case config.GeneratorTypeNop: return nil, fmt.Errorf("generator type %q does not produce records; not embed-eligible", genCfg.Type) diff --git a/internal/dispatch/embed_test.go b/internal/dispatch/embed_test.go index fa27c70..69f34e9 100644 --- a/internal/dispatch/embed_test.go +++ b/internal/dispatch/embed_test.go @@ -40,7 +40,7 @@ func TestForEmbedWelReturnsProducerModule(t *testing.T) { Role: "member", }, } - mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.NoError(t, err) require.NotNil(t, mod) assert.Equal(t, "wel", mod.Name()) @@ -54,14 +54,14 @@ func TestForEmbedWelDefaultsEmptyRole(t *testing.T) { Rate: 50 * time.Millisecond, }, } - mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.NoError(t, err) require.NotNil(t, mod) } func TestForEmbedWinevtRejectionMentionsWel(t *testing.T) { cfg := config.Generator{Type: config.GeneratorTypeWinevt} - _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "DEPRECATED") assert.Contains(t, err.Error(), "`wel` generator") @@ -77,7 +77,7 @@ func TestForEmbedFIXReturnsProducerModule(t *testing.T) { Version: "4.4", }, } - mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + mod, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.NoError(t, err) require.NotNil(t, mod) assert.Equal(t, "fix", mod.Name()) @@ -92,7 +92,7 @@ func TestForEmbedFIXRejectsUnknownVersion(t *testing.T) { Version: "4.3", }, } - _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "unknown version") } @@ -106,18 +106,18 @@ func TestForEmbedFIXRejectsUnknownCategory(t *testing.T) { EnabledCategories: []string{"crypto"}, }, } - _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil) + _, err := ForEmbed(zap.NewNop(), cfg, logsOnly(), nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "unknown asset category") } func TestForEmbedRejectsNilLogger(t *testing.T) { - _, err := ForEmbed(nil, config.Generator{Type: config.GeneratorTypeFIX}, logsOnly(), nil) + _, err := ForEmbed(nil, config.Generator{Type: config.GeneratorTypeFIX}, logsOnly(), nil, embed.NopTelemetry()) require.Error(t, err) } func TestForEmbedRejectsMissingLogConsumerForLogType(t *testing.T) { - _, err := ForEmbed(zap.NewNop(), config.Generator{Type: config.GeneratorTypeFIX}, EmbedConsumers{}, nil) + _, err := ForEmbed(zap.NewNop(), config.Generator{Type: config.GeneratorTypeFIX}, EmbedConsumers{}, nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "LogConsumer") } @@ -133,7 +133,7 @@ func TestForEmbedHostMetricsReturnsProducerModule(t *testing.T) { Hostname: "test-host", }, } - mod, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{MetricConsumer: noopMetricConsumer{}}, nil) + mod, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{MetricConsumer: noopMetricConsumer{}}, nil, embed.NopTelemetry()) require.NoError(t, err) require.NotNil(t, mod) assert.Equal(t, "hostmetrics", mod.Name()) @@ -148,7 +148,7 @@ func TestForEmbedHostMetricsRejectsMissingMetricConsumer(t *testing.T) { OS: "linux", }, } - _, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{}, nil) + _, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{}, nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "MetricConsumer") } @@ -162,7 +162,7 @@ func TestForEmbedTracesReturnsProducerModule(t *testing.T) { Rate: 50 * time.Millisecond, }, } - mod, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{TraceConsumer: noopTraceConsumer{}}, nil) + mod, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{TraceConsumer: noopTraceConsumer{}}, nil, embed.NopTelemetry()) require.NoError(t, err) require.NotNil(t, mod) assert.Equal(t, "traces", mod.Name()) @@ -176,7 +176,7 @@ func TestForEmbedTracesRejectsMissingTraceConsumer(t *testing.T) { Rate: time.Second, }, } - _, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{}, nil) + _, err := ForEmbed(zap.NewNop(), cfg, EmbedConsumers{}, nil, embed.NopTelemetry()) require.Error(t, err) assert.Contains(t, err.Error(), "TraceConsumer") } diff --git a/output/file/file.go b/output/file/file.go index 58d1107..261be83 100644 --- a/output/file/file.go +++ b/output/file/file.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/internal/workermanager" "github.com/observiq/blitz/output" "github.com/observiq/blitz/telemetry" @@ -43,10 +44,11 @@ type File struct { cancel context.CancelFunc workerManager *workermanager.WorkerManager writer *lumberjack.Logger + metrics *output.Metrics } // New creates a new File output instance -func New(logger *zap.Logger, path string, workers int, rotation RotationOptions) (*File, error) { +func New(logger *zap.Logger, path string, workers int, rotation RotationOptions, tel embed.TelemetrySettings) (*File, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -57,6 +59,11 @@ func New(logger *zap.Logger, path string, workers int, rotation RotationOptions) workers = DefaultFileWorkers } + m, err := output.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build output metrics: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) writer := &lumberjack.Logger{ @@ -76,6 +83,7 @@ func New(logger *zap.Logger, path string, workers int, rotation RotationOptions) ctx: ctx, cancel: cancel, writer: writer, + metrics: m, } f.logger.Info("Starting File output", @@ -85,13 +93,15 @@ func New(logger *zap.Logger, path string, workers int, rotation RotationOptions) ) // Register observable metrics (queue_size) - output.InitObservableMetrics(f) + if err := f.metrics.InitObservable(f); err != nil { + return nil, fmt.Errorf("init observable metrics: %w", err) + } // Worker manager f.workerManager = workermanager.NewWorkerManager(f.logger, workers, f.fileWorker) // Record initial active workers count - output.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) + f.metrics.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) f.workerManager.Start() @@ -108,7 +118,7 @@ func (f *File) ObserveBlitzOutputQueueSize(_ context.Context, observer metric.In func (f *File) Write(ctx context.Context, data output.LogRecord) error { select { case f.dataChan <- data.Message: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") + f.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write data: %w", ctx.Err()) @@ -121,7 +131,7 @@ func (f *File) Write(ctx context.Context, data output.LogRecord) error { func (f *File) Stop(ctx context.Context) error { f.logger.Info("Stopping File output") - output.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) + f.metrics.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) close(f.dataChan) f.cancel() @@ -175,8 +185,8 @@ func (f *File) writeData(data string) error { } latency := time.Since(start).Seconds() - output.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") + f.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") + f.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") // Record latency as a histogram using Float64Histogram like TCP for symmetry // Use a separate metric name if needed in the future; omitted here to reduce metric cardinality @@ -186,7 +196,7 @@ func (f *File) writeData(data string) error { } func (f *File) recordWriteError(_ string, _ error) { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") + f.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") } // SupportedTelemetry returns the telemetry types this output can consume. diff --git a/output/file/file_test.go b/output/file/file_test.go index 689d381..31d4b50 100644 --- a/output/file/file_test.go +++ b/output/file/file_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -19,7 +20,7 @@ func TestFileOutputWriteSingleLine(t *testing.T) { logger := zap.NewNop() rot := RotationOptions{MaxSizeMB: 100, MaxBackups: 1, MaxAgeDays: 1, Compress: false, LocalTime: false} - f, err := New(logger, path, 1, rot) + f, err := New(logger, path, 1, rot, embed.NopTelemetry()) if err != nil { t.Fatalf("new file output: %v", err) } @@ -54,7 +55,7 @@ func TestFileOutputWriteMultipleLines(t *testing.T) { logger := zap.NewNop() rot := RotationOptions{MaxSizeMB: 100, MaxBackups: 2, MaxAgeDays: 2, Compress: false, LocalTime: false} - f, err := New(logger, path, 2, rot) + f, err := New(logger, path, 2, rot, embed.NopTelemetry()) if err != nil { t.Fatalf("new file output: %v", err) } diff --git a/output/hec/hec.go b/output/hec/hec.go index c5cba55..844624f 100644 --- a/output/hec/hec.go +++ b/output/hec/hec.go @@ -124,7 +124,7 @@ func New(logger *zap.Logger, opts ...Option) (*HEC, error) { hostname = "unknown" } - m, err := newHECMetrics() + m, err := newHECMetrics(cfg.tel) if err != nil { return nil, fmt.Errorf("create metrics: %w", err) } diff --git a/output/hec/metrics.go b/output/hec/metrics.go index 063c748..54505d1 100644 --- a/output/hec/metrics.go +++ b/output/hec/metrics.go @@ -3,69 +3,82 @@ package hec import ( "context" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" ) // outputType is the output_type attribute value for HEC metrics. const outputType = "hec" -// hecMetrics provides convenience methods that delegate to generated metric wrappers. -// Shared output metrics use the output package-level wrappers (with output_type="hec"). -// HEC-specific metrics use the package-level variables from monitoring.go. -type hecMetrics struct{} +// hecMetrics provides convenience methods that delegate to the per-instance +// metric structs. Shared output metrics record through *output.Metrics (with +// output_type="hec"); HEC-specific metrics record through this package's +// generated *Metrics. +type hecMetrics struct { + out *output.Metrics + hec *Metrics +} -func newHECMetrics() (*hecMetrics, error) { - return &hecMetrics{}, nil +func newHECMetrics(tel embed.TelemetrySettings) (*hecMetrics, error) { + out, err := output.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, err + } + hecM, err := NewMetrics(tel.MeterProvider) + if err != nil { + return nil, err + } + return &hecMetrics{out: out, hec: hecM}, nil } func (m *hecMetrics) recordLogsReceived(ctx context.Context, count int64) { - output.BlitzOutputEntriesReceivedCounter.Add(ctx, count, outputType, "logs") + m.out.BlitzOutputEntriesReceivedCounter.Add(ctx, count, outputType, "logs") } func (m *hecMetrics) recordActiveWorkers(ctx context.Context, count int64) { - output.BlitzOutputActiveWorkersGauge.Record(ctx, count, outputType) + m.out.BlitzOutputActiveWorkersGauge.Record(ctx, count, outputType) } func (m *hecMetrics) recordLogRate(ctx context.Context, count float64) { - output.BlitzOutputEntryRateCounter.Add(ctx, count, outputType, "logs") + m.out.BlitzOutputEntryRateCounter.Add(ctx, count, outputType, "logs") } func (m *hecMetrics) recordRequestSize(ctx context.Context, bytes int64) { - output.BlitzOutputRequestSizeHistogram.Record(ctx, bytes, outputType, "logs") + m.out.BlitzOutputRequestSizeHistogram.Record(ctx, bytes, outputType, "logs") } func (m *hecMetrics) recordRequestLatency(ctx context.Context, seconds float64) { - output.BlitzOutputRequestLatencyHistogram.Record(ctx, seconds, outputType, "logs") + m.out.BlitzOutputRequestLatencyHistogram.Record(ctx, seconds, outputType, "logs") } func (m *hecMetrics) recordSendError(ctx context.Context, _ string) { - output.BlitzOutputSendErrorsCounter.Add(ctx, 1, outputType, "logs") + m.out.BlitzOutputSendErrorsCounter.Add(ctx, 1, outputType, "logs") } func (m *hecMetrics) recordBatchSize(ctx context.Context, size int64) { - blitzOutputHecBatchSizeHistogram.Record(ctx, size) + m.hec.blitzOutputHecBatchSizeHistogram.Record(ctx, size) } func (m *hecMetrics) recordACKPending(ctx context.Context, count int64) { - blitzOutputHecAckPendingGauge.Record(ctx, count) + m.hec.blitzOutputHecAckPendingGauge.Record(ctx, count) } func (m *hecMetrics) recordACKConfirmed(ctx context.Context, count int64) { - blitzOutputHecAckConfirmedCounter.Add(ctx, count) + m.hec.blitzOutputHecAckConfirmedCounter.Add(ctx, count) } func (m *hecMetrics) recordACKExpired(ctx context.Context, count int64) { - blitzOutputHecAckExpiredCounter.Add(ctx, count) + m.hec.blitzOutputHecAckExpiredCounter.Add(ctx, count) } func (m *hecMetrics) recordACKRetried(ctx context.Context, count int64) { - blitzOutputHecAckRetriedCounter.Add(ctx, count) + m.hec.blitzOutputHecAckRetriedCounter.Add(ctx, count) } func (m *hecMetrics) recordACKDropped(ctx context.Context, count int64) { - blitzOutputHecAckDroppedCounter.Add(ctx, count) + m.hec.blitzOutputHecAckDroppedCounter.Add(ctx, count) } func (m *hecMetrics) recordACKPollLatency(ctx context.Context, seconds float64) { - blitzOutputHecAckPollLatencyHistogram.Record(ctx, seconds) + m.hec.blitzOutputHecAckPollLatencyHistogram.Record(ctx, seconds) } diff --git a/output/hec/monitoring.go b/output/hec/monitoring.go index aa0c411..dc55350 100644 --- a/output/hec/monitoring.go +++ b/output/hec/monitoring.go @@ -6,90 +6,103 @@ package hec import ( "errors" - "fmt" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" ) -var ( - hecMeter = otel.Meter("hec") +// Constants for enum members - // number of entries per HEC batch - blitzOutputHecBatchSizeHistogram metric.Int64Histogram +// Wrapper types for metrics with required attributes - // number of ACKs currently pending confirmation - blitzOutputHecAckPendingGauge metric.Int64Gauge +// Metrics holds blitz's hec self-telemetry instruments. Build one +// with NewMetrics from a caller-supplied MeterProvider so metrics can be routed +// to any provider (an embedding host's, or the process global). +type Metrics struct { + hecMeter metric.Meter // total number of ACKs confirmed by the server blitzOutputHecAckConfirmedCounter metric.Int64Counter - - // total number of ACKs that expired without confirmation - blitzOutputHecAckExpiredCounter metric.Int64Counter - - // total number of batches retried due to ACK failure - blitzOutputHecAckRetriedCounter metric.Int64Counter - // total number of batches dropped after max retries blitzOutputHecAckDroppedCounter metric.Int64Counter - + // total number of ACKs that expired without confirmation + blitzOutputHecAckExpiredCounter metric.Int64Counter + // number of ACKs currently pending confirmation + blitzOutputHecAckPendingGauge metric.Int64Gauge // latency of ACK polling requests blitzOutputHecAckPollLatencyHistogram metric.Float64Histogram -) + // total number of batches retried due to ACK failure + blitzOutputHecAckRetriedCounter metric.Int64Counter + // number of entries per HEC batch + blitzOutputHecBatchSizeHistogram metric.Int64Histogram +} -func init() { - var err, errs error +// NewMetrics builds the hec instruments from mp. A nil mp falls +// back to the process-global MeterProvider, preserving standalone behavior. +func NewMetrics(mp metric.MeterProvider) (*Metrics, error) { + if mp == nil { + mp = otel.GetMeterProvider() + } + m := &Metrics{} + var errs error - blitzOutputHecBatchSizeHistogram, err = hecMeter.Int64Histogram( - "blitz.output.hec.batch_size", - metric.WithDescription("number of entries per HEC batch"), - metric.WithUnit("{entry}"), - ) - errs = errors.Join(errs, err) + m.hecMeter = mp.Meter("hec") - blitzOutputHecAckPendingGauge, err = hecMeter.Int64Gauge( - "blitz.output.hec.ack_pending", - metric.WithDescription("number of ACKs currently pending confirmation"), + blitzOutputHecAckConfirmedCounterRaw, err := m.hecMeter.Int64Counter( + "blitz.output.hec.ack_confirmed", + metric.WithDescription("total number of ACKs confirmed by the server"), metric.WithUnit("{ack}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecAckConfirmedCounter = blitzOutputHecAckConfirmedCounterRaw - blitzOutputHecAckConfirmedCounter, err = hecMeter.Int64Counter( - "blitz.output.hec.ack_confirmed", - metric.WithDescription("total number of ACKs confirmed by the server"), + blitzOutputHecAckDroppedCounterRaw, err := m.hecMeter.Int64Counter( + "blitz.output.hec.ack_dropped", + metric.WithDescription("total number of batches dropped after max retries"), metric.WithUnit("{ack}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecAckDroppedCounter = blitzOutputHecAckDroppedCounterRaw - blitzOutputHecAckExpiredCounter, err = hecMeter.Int64Counter( + blitzOutputHecAckExpiredCounterRaw, err := m.hecMeter.Int64Counter( "blitz.output.hec.ack_expired", metric.WithDescription("total number of ACKs that expired without confirmation"), metric.WithUnit("{ack}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecAckExpiredCounter = blitzOutputHecAckExpiredCounterRaw - blitzOutputHecAckRetriedCounter, err = hecMeter.Int64Counter( - "blitz.output.hec.ack_retried", - metric.WithDescription("total number of batches retried due to ACK failure"), + blitzOutputHecAckPendingGaugeRaw, err := m.hecMeter.Int64Gauge( + "blitz.output.hec.ack_pending", + metric.WithDescription("number of ACKs currently pending confirmation"), metric.WithUnit("{ack}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecAckPendingGauge = blitzOutputHecAckPendingGaugeRaw - blitzOutputHecAckDroppedCounter, err = hecMeter.Int64Counter( - "blitz.output.hec.ack_dropped", - metric.WithDescription("total number of batches dropped after max retries"), + blitzOutputHecAckPollLatencyHistogramRaw, err := m.hecMeter.Float64Histogram( + "blitz.output.hec.ack_poll_latency", + metric.WithDescription("latency of ACK polling requests"), + metric.WithUnit("s"), + ) + errs = errors.Join(errs, err) + m.blitzOutputHecAckPollLatencyHistogram = blitzOutputHecAckPollLatencyHistogramRaw + + blitzOutputHecAckRetriedCounterRaw, err := m.hecMeter.Int64Counter( + "blitz.output.hec.ack_retried", + metric.WithDescription("total number of batches retried due to ACK failure"), metric.WithUnit("{ack}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecAckRetriedCounter = blitzOutputHecAckRetriedCounterRaw - blitzOutputHecAckPollLatencyHistogram, err = hecMeter.Float64Histogram( - "blitz.output.hec.ack_poll_latency", - metric.WithDescription("latency of ACK polling requests"), - metric.WithUnit("s"), + blitzOutputHecBatchSizeHistogramRaw, err := m.hecMeter.Int64Histogram( + "blitz.output.hec.batch_size", + metric.WithDescription("number of entries per HEC batch"), + metric.WithUnit("{entry}"), ) errs = errors.Join(errs, err) + m.blitzOutputHecBatchSizeHistogram = blitzOutputHecBatchSizeHistogramRaw - if errs != nil { - panic(fmt.Sprintf("Initialize hec metrics: %s", errs)) - } + return m, errs } diff --git a/output/hec/monitoring.md b/output/hec/monitoring.md new file mode 100644 index 0000000..eb6060a --- /dev/null +++ b/output/hec/monitoring.md @@ -0,0 +1,142 @@ +# Hec Metrics Reference + +## Quick Reference + +| Metric | Type | Unit | Description | +|--------|------|------|-------------| +| [`blitz.output.hec.ack_confirmed`](#blitzoutputhecack-confirmed) | Counter | `{ack}` | total number of ACKs confirmed by the server | +| [`blitz.output.hec.ack_dropped`](#blitzoutputhecack-dropped) | Counter | `{ack}` | total number of batches dropped after max retries | +| [`blitz.output.hec.ack_expired`](#blitzoutputhecack-expired) | Counter | `{ack}` | total number of ACKs that expired without confirmation | +| [`blitz.output.hec.ack_pending`](#blitzoutputhecack-pending) | Gauge | `{ack}` | number of ACKs currently pending confirmation | +| [`blitz.output.hec.ack_poll_latency`](#blitzoutputhecack-poll-latency) | Histogram | `s` | latency of ACK polling requests | +| [`blitz.output.hec.ack_retried`](#blitzoutputhecack-retried) | Counter | `{ack}` | total number of batches retried due to ACK failure | +| [`blitz.output.hec.batch_size`](#blitzoutputhecbatch-size) | Histogram | `{entry}` | number of entries per HEC batch | + +--- + +## Metrics Detail + +### blitz.output.hec.ack_confirmed + +| Property | Value | +|----------|-------| +| **Type** | Counter | +| **Unit** | `{ack}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | total number of ACKs confirmed by the server | + +**Usage:** +```go +blitzOutputHecAckConfirmedCounter.Add(ctx, 1) +``` + +--- + +### blitz.output.hec.ack_dropped + +| Property | Value | +|----------|-------| +| **Type** | Counter | +| **Unit** | `{ack}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | total number of batches dropped after max retries | + +**Usage:** +```go +blitzOutputHecAckDroppedCounter.Add(ctx, 1) +``` + +--- + +### blitz.output.hec.ack_expired + +| Property | Value | +|----------|-------| +| **Type** | Counter | +| **Unit** | `{ack}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | total number of ACKs that expired without confirmation | + +**Usage:** +```go +blitzOutputHecAckExpiredCounter.Add(ctx, 1) +``` + +--- + +### blitz.output.hec.ack_pending + +| Property | Value | +|----------|-------| +| **Type** | Gauge | +| **Unit** | `{ack}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | number of ACKs currently pending confirmation | + +**Usage:** +```go +blitzOutputHecAckPendingGauge.Add(ctx, 1) +``` + +--- + +### blitz.output.hec.ack_poll_latency + +| Property | Value | +|----------|-------| +| **Type** | Histogram | +| **Unit** | `s` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | latency of ACK polling requests | + +**Usage:** +```go +blitzOutputHecAckPollLatencyHistogram.Record(ctx, 1) +``` + +--- + +### blitz.output.hec.ack_retried + +| Property | Value | +|----------|-------| +| **Type** | Counter | +| **Unit** | `{ack}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | total number of batches retried due to ACK failure | + +**Usage:** +```go +blitzOutputHecAckRetriedCounter.Add(ctx, 1) +``` + +--- + +### blitz.output.hec.batch_size + +| Property | Value | +|----------|-------| +| **Type** | Histogram | +| **Unit** | `{entry}` | +| **Meter** | `hec` | +| **Stability** | Stable | +| **Description** | number of entries per HEC batch | + +**Usage:** +```go +blitzOutputHecBatchSizeHistogram.Record(ctx, 1) +``` + +--- + + + +--- + +**Generated:** `make generate-o11y` | **Registry:** `hec/monitoring/` | **Templates:** `weaver/templates/` \ No newline at end of file diff --git a/output/hec/options.go b/output/hec/options.go index c7616ad..9edb6cb 100644 --- a/output/hec/options.go +++ b/output/hec/options.go @@ -3,6 +3,8 @@ package hec import ( "crypto/tls" "time" + + "github.com/observiq/blitz/embed" ) // Option is a functional option for configuring the HEC output @@ -26,6 +28,16 @@ type Config struct { index string enableTLS bool tlsConfig *tls.Config + tel embed.TelemetrySettings +} + +// WithTelemetry sets the OTel providers blitz routes its self-telemetry +// through. The zero value falls back to the process-global providers. +func WithTelemetry(tel embed.TelemetrySettings) Option { + return func(cfg *Config) error { + cfg.tel = tel + return nil + } } // WithHost sets the target host for HEC connections diff --git a/output/monitoring.go b/output/monitoring.go index 2ee78f0b..712d11c 100644 --- a/output/monitoring.go +++ b/output/monitoring.go @@ -7,7 +7,6 @@ package output import ( "context" "errors" - "fmt" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -20,54 +19,56 @@ const ( attrTelemetryType = "telemetry_type" ) +// Constants for enum members + // Wrapper types for metrics with required attributes -// BlitzOutputEntriesReceivedCounterType wraps the blitz.output.entries_received metric with type-safe required attributes -type BlitzOutputEntriesReceivedCounterType struct { - counter metric.Int64Counter +// blitzOutputActiveWorkersGaugeType wraps the blitz.output.active_workers metric with type-safe required attributes +type blitzOutputActiveWorkersGaugeType struct { + gauge metric.Int64Gauge } -// Add records a value for blitz.output.entries_received with required attributes -func (m BlitzOutputEntriesReceivedCounterType) Add(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.AddOption) { +// Record records a value for blitz.output.active_workers with required attributes +func (m blitzOutputActiveWorkersGaugeType) Record(ctx context.Context, value int64, outputType string, opts ...metric.RecordOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), - attribute.String(attrTelemetryType, telemetryType), )) if len(opts) == 0 { - m.counter.Add(ctx, value, attrs) + m.gauge.Record(ctx, value, attrs) } else { - allOpts := append([]metric.AddOption{attrs}, opts...) - m.counter.Add(ctx, value, allOpts...) + allOpts := append([]metric.RecordOption{attrs}, opts...) + m.gauge.Record(ctx, value, allOpts...) } } -// BlitzOutputActiveWorkersGaugeType wraps the blitz.output.active_workers metric with type-safe required attributes -type BlitzOutputActiveWorkersGaugeType struct { - gauge metric.Int64Gauge +// blitzOutputEntriesReceivedCounterType wraps the blitz.output.entries_received metric with type-safe required attributes +type blitzOutputEntriesReceivedCounterType struct { + counter metric.Int64Counter } -// Record records a value for blitz.output.active_workers with required attributes -func (m BlitzOutputActiveWorkersGaugeType) Record(ctx context.Context, value int64, outputType string, opts ...metric.RecordOption) { +// Add records a value for blitz.output.entries_received with required attributes +func (m blitzOutputEntriesReceivedCounterType) Add(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.AddOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), + attribute.String(attrTelemetryType, telemetryType), )) if len(opts) == 0 { - m.gauge.Record(ctx, value, attrs) + m.counter.Add(ctx, value, attrs) } else { - allOpts := append([]metric.RecordOption{attrs}, opts...) - m.gauge.Record(ctx, value, allOpts...) + allOpts := append([]metric.AddOption{attrs}, opts...) + m.counter.Add(ctx, value, allOpts...) } } -// BlitzOutputEntryRateCounterType wraps the blitz.output.entry_rate metric with type-safe required attributes -type BlitzOutputEntryRateCounterType struct { +// blitzOutputEntryRateCounterType wraps the blitz.output.entry_rate metric with type-safe required attributes +type blitzOutputEntryRateCounterType struct { counter metric.Float64Counter } // Add records a value for blitz.output.entry_rate with required attributes -func (m BlitzOutputEntryRateCounterType) Add(ctx context.Context, value float64, outputType string, telemetryType string, opts ...metric.AddOption) { +func (m blitzOutputEntryRateCounterType) Add(ctx context.Context, value float64, outputType string, telemetryType string, opts ...metric.AddOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), attribute.String(attrTelemetryType, telemetryType), @@ -81,13 +82,13 @@ func (m BlitzOutputEntryRateCounterType) Add(ctx context.Context, value float64, } } -// BlitzOutputRequestSizeHistogramType wraps the blitz.output.request_size metric with type-safe required attributes -type BlitzOutputRequestSizeHistogramType struct { - histogram metric.Int64Histogram +// blitzOutputRequestLatencyHistogramType wraps the blitz.output.request_latency metric with type-safe required attributes +type blitzOutputRequestLatencyHistogramType struct { + histogram metric.Float64Histogram } -// Record records a value for blitz.output.request_size with required attributes -func (m BlitzOutputRequestSizeHistogramType) Record(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.RecordOption) { +// Record records a value for blitz.output.request_latency with required attributes +func (m blitzOutputRequestLatencyHistogramType) Record(ctx context.Context, value float64, outputType string, telemetryType string, opts ...metric.RecordOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), attribute.String(attrTelemetryType, telemetryType), @@ -101,13 +102,13 @@ func (m BlitzOutputRequestSizeHistogramType) Record(ctx context.Context, value i } } -// BlitzOutputRequestLatencyHistogramType wraps the blitz.output.request_latency metric with type-safe required attributes -type BlitzOutputRequestLatencyHistogramType struct { - histogram metric.Float64Histogram +// blitzOutputRequestSizeHistogramType wraps the blitz.output.request_size metric with type-safe required attributes +type blitzOutputRequestSizeHistogramType struct { + histogram metric.Int64Histogram } -// Record records a value for blitz.output.request_latency with required attributes -func (m BlitzOutputRequestLatencyHistogramType) Record(ctx context.Context, value float64, outputType string, telemetryType string, opts ...metric.RecordOption) { +// Record records a value for blitz.output.request_size with required attributes +func (m blitzOutputRequestSizeHistogramType) Record(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.RecordOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), attribute.String(attrTelemetryType, telemetryType), @@ -121,13 +122,13 @@ func (m BlitzOutputRequestLatencyHistogramType) Record(ctx context.Context, valu } } -// BlitzOutputSendErrorsCounterType wraps the blitz.output.send_errors metric with type-safe required attributes -type BlitzOutputSendErrorsCounterType struct { +// blitzOutputSendErrorsCounterType wraps the blitz.output.send_errors metric with type-safe required attributes +type blitzOutputSendErrorsCounterType struct { counter metric.Int64Counter } // Add records a value for blitz.output.send_errors with required attributes -func (m BlitzOutputSendErrorsCounterType) Add(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.AddOption) { +func (m blitzOutputSendErrorsCounterType) Add(ctx context.Context, value int64, outputType string, telemetryType string, opts ...metric.AddOption) { attrs := metric.WithAttributeSet(attribute.NewSet( attribute.String(attrOutputType, outputType), attribute.String(attrTelemetryType, telemetryType), @@ -146,89 +147,94 @@ type ObservableCallbacks interface { ObserveBlitzOutputQueueSize(ctx context.Context, observer metric.Int64Observer) error } -var ( - outputMeter = otel.Meter("output") - - // BlitzOutputEntriesReceivedCounter tracks the total number of log entries received by the output - BlitzOutputEntriesReceivedCounter BlitzOutputEntriesReceivedCounterType - - // BlitzOutputActiveWorkersGauge tracks the number of active output worker goroutines - BlitzOutputActiveWorkersGauge BlitzOutputActiveWorkersGaugeType - - // BlitzOutputEntryRateCounter tracks the rate of log entries processed per second - BlitzOutputEntryRateCounter BlitzOutputEntryRateCounterType - - // BlitzOutputRequestSizeHistogram tracks the size of output requests in bytes - BlitzOutputRequestSizeHistogram BlitzOutputRequestSizeHistogramType - - // BlitzOutputRequestLatencyHistogram tracks the latency of output requests - BlitzOutputRequestLatencyHistogram BlitzOutputRequestLatencyHistogramType - - // BlitzOutputSendErrorsCounter tracks the total number of send errors - BlitzOutputSendErrorsCounter BlitzOutputSendErrorsCounterType -) +// Metrics holds blitz's output self-telemetry instruments. Build one +// with NewMetrics from a caller-supplied MeterProvider so metrics can be routed +// to any provider (an embedding host's, or the process global). +type Metrics struct { + outputMeter metric.Meter + + // number of active output worker goroutines + BlitzOutputActiveWorkersGauge blitzOutputActiveWorkersGaugeType + // total number of telemetry entries received by the output + BlitzOutputEntriesReceivedCounter blitzOutputEntriesReceivedCounterType + // rate of telemetry entries processed per second + BlitzOutputEntryRateCounter blitzOutputEntryRateCounterType + // latency of output requests + BlitzOutputRequestLatencyHistogram blitzOutputRequestLatencyHistogramType + // size of output requests in bytes + BlitzOutputRequestSizeHistogram blitzOutputRequestSizeHistogramType + // total number of send errors + BlitzOutputSendErrorsCounter blitzOutputSendErrorsCounterType +} -func init() { - var err, errs error +// NewMetrics builds the output instruments from mp. A nil mp falls +// back to the process-global MeterProvider, preserving standalone behavior. +func NewMetrics(mp metric.MeterProvider) (*Metrics, error) { + if mp == nil { + mp = otel.GetMeterProvider() + } + m := &Metrics{} + var errs error - blitzOutputEntriesReceivedCounterRaw, err := outputMeter.Int64Counter( - "blitz.output.entries_received", - metric.WithDescription("total number of log entries received by the output"), - metric.WithUnit("{entry}"), - ) - errs = errors.Join(errs, err) - BlitzOutputEntriesReceivedCounter = BlitzOutputEntriesReceivedCounterType{counter: blitzOutputEntriesReceivedCounterRaw} + m.outputMeter = mp.Meter("output") - blitzOutputActiveWorkersGaugeRaw, err := outputMeter.Int64Gauge( + BlitzOutputActiveWorkersGaugeRaw, err := m.outputMeter.Int64Gauge( "blitz.output.active_workers", metric.WithDescription("number of active output worker goroutines"), metric.WithUnit("{worker}"), ) errs = errors.Join(errs, err) - BlitzOutputActiveWorkersGauge = BlitzOutputActiveWorkersGaugeType{gauge: blitzOutputActiveWorkersGaugeRaw} + m.BlitzOutputActiveWorkersGauge = blitzOutputActiveWorkersGaugeType{gauge: BlitzOutputActiveWorkersGaugeRaw} - blitzOutputEntryRateCounterRaw, err := outputMeter.Float64Counter( - "blitz.output.entry_rate", - metric.WithDescription("rate of log entries processed per second"), - metric.WithUnit("{entry}/s"), + BlitzOutputEntriesReceivedCounterRaw, err := m.outputMeter.Int64Counter( + "blitz.output.entries_received", + metric.WithDescription("total number of telemetry entries received by the output"), + metric.WithUnit("{entry}"), ) errs = errors.Join(errs, err) - BlitzOutputEntryRateCounter = BlitzOutputEntryRateCounterType{counter: blitzOutputEntryRateCounterRaw} + m.BlitzOutputEntriesReceivedCounter = blitzOutputEntriesReceivedCounterType{counter: BlitzOutputEntriesReceivedCounterRaw} - blitzOutputRequestSizeHistogramRaw, err := outputMeter.Int64Histogram( - "blitz.output.request_size", - metric.WithDescription("size of output requests in bytes"), - metric.WithUnit("By"), + BlitzOutputEntryRateCounterRaw, err := m.outputMeter.Float64Counter( + "blitz.output.entry_rate", + metric.WithDescription("rate of telemetry entries processed per second"), + metric.WithUnit("{entry}/s"), ) errs = errors.Join(errs, err) - BlitzOutputRequestSizeHistogram = BlitzOutputRequestSizeHistogramType{histogram: blitzOutputRequestSizeHistogramRaw} + m.BlitzOutputEntryRateCounter = blitzOutputEntryRateCounterType{counter: BlitzOutputEntryRateCounterRaw} - blitzOutputRequestLatencyHistogramRaw, err := outputMeter.Float64Histogram( + BlitzOutputRequestLatencyHistogramRaw, err := m.outputMeter.Float64Histogram( "blitz.output.request_latency", metric.WithDescription("latency of output requests"), metric.WithUnit("s"), ) errs = errors.Join(errs, err) - BlitzOutputRequestLatencyHistogram = BlitzOutputRequestLatencyHistogramType{histogram: blitzOutputRequestLatencyHistogramRaw} + m.BlitzOutputRequestLatencyHistogram = blitzOutputRequestLatencyHistogramType{histogram: BlitzOutputRequestLatencyHistogramRaw} - blitzOutputSendErrorsCounterRaw, err := outputMeter.Int64Counter( + BlitzOutputRequestSizeHistogramRaw, err := m.outputMeter.Int64Histogram( + "blitz.output.request_size", + metric.WithDescription("size of output requests in bytes"), + metric.WithUnit("By"), + ) + errs = errors.Join(errs, err) + m.BlitzOutputRequestSizeHistogram = blitzOutputRequestSizeHistogramType{histogram: BlitzOutputRequestSizeHistogramRaw} + + BlitzOutputSendErrorsCounterRaw, err := m.outputMeter.Int64Counter( "blitz.output.send_errors", metric.WithDescription("total number of send errors"), metric.WithUnit("{error}"), ) errs = errors.Join(errs, err) - BlitzOutputSendErrorsCounter = BlitzOutputSendErrorsCounterType{counter: blitzOutputSendErrorsCounterRaw} + m.BlitzOutputSendErrorsCounter = blitzOutputSendErrorsCounterType{counter: BlitzOutputSendErrorsCounterRaw} - if errs != nil { - panic(fmt.Sprintf("Initialize output metrics: %s", errs)) - } + return m, errs } -// InitObservableMetrics initializes observable metrics with the provided callbacks -func InitObservableMetrics(callbacks ObservableCallbacks) { +// InitObservable registers the observable instruments against callbacks. Call +// it once, after NewMetrics, from the component that owns the observed state. +func (m *Metrics) InitObservable(callbacks ObservableCallbacks) error { var err, errs error - _, err = outputMeter.Int64ObservableGauge( + _, err = m.outputMeter.Int64ObservableGauge( "blitz.output.queue_size", metric.WithDescription("current number of entries in the output queue"), metric.WithUnit("{entry}"), @@ -236,7 +242,5 @@ func InitObservableMetrics(callbacks ObservableCallbacks) { ) errs = errors.Join(errs, err) - if errs != nil { - panic(fmt.Sprintf("Initialize output observable metrics: %s", errs)) - } + return errs } diff --git a/output/monitoring.md b/output/monitoring.md index 4fb27c7..ccf740b 100644 --- a/output/monitoring.md +++ b/output/monitoring.md @@ -4,28 +4,28 @@ | Metric | Type | Unit | Description | |--------|------|------|-------------| -| [`blitz.output.entries_received`](#blitzoutputentries-received) | Counter | `{entry}` | total number of log entries received by the output | | [`blitz.output.active_workers`](#blitzoutputactive-workers) | Gauge | `{worker}` | number of active output worker goroutines | -| [`blitz.output.entry_rate`](#blitzoutputentry-rate) | Counter | `{entry}/s` | rate of log entries processed per second | -| [`blitz.output.request_size`](#blitzoutputrequest-size) | Histogram | `By` | size of output requests in bytes | +| [`blitz.output.entries_received`](#blitzoutputentries-received) | Counter | `{entry}` | total number of telemetry entries received by the output | +| [`blitz.output.entry_rate`](#blitzoutputentry-rate) | Counter | `{entry}/s` | rate of telemetry entries processed per second | +| [`blitz.output.queue_size`](#blitzoutputqueue-size) | Gauge | `{entry}` | current number of entries in the output queue | | [`blitz.output.request_latency`](#blitzoutputrequest-latency) | Histogram | `s` | latency of output requests | +| [`blitz.output.request_size`](#blitzoutputrequest-size) | Histogram | `By` | size of output requests in bytes | | [`blitz.output.send_errors`](#blitzoutputsend-errors) | Counter | `{error}` | total number of send errors | -| [`blitz.output.queue_size`](#blitzoutputqueue-size) | Gauge | `{entry}` | current number of entries in the output queue | --- ## Metrics Detail -### blitz.output.entries_received +### blitz.output.active_workers | Property | Value | |----------|-------| -| **Type** | Counter | -| **Unit** | `{entry}` | +| **Type** | Gauge | +| **Unit** | `{worker}` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | total number of log entries received by the output | -| **Attributes** | `output_type` | +| **Description** | number of active output worker goroutines | +| **Attributes** |`output_type` | **Attributes:** @@ -35,31 +35,34 @@ **Usage:** ```go -output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputTypeValue) +// Type-safe wrapper with required attributes +blitzOutputActiveWorkersGauge.Add(ctx, 1, outputTypeValue) ``` --- -### blitz.output.active_workers +### blitz.output.entries_received | Property | Value | |----------|-------| -| **Type** | Gauge | -| **Unit** | `{worker}` | +| **Type** | Counter | +| **Unit** | `{entry}` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | number of active output worker goroutines | -| **Attributes** | `output_type` | +| **Description** | total number of telemetry entries received by the output | +| **Attributes** |`output_type`, `telemetry_type` | **Attributes:** | Name | Type | Required | Values | |------|------|----------|--------| | `output_type` | string | ✓ | - | +| `telemetry_type` | string | ✓ | - | **Usage:** ```go -output.BlitzOutputActiveWorkersGauge.Record(ctx, 1, outputTypeValue) +// Type-safe wrapper with required attributes +blitzOutputEntriesReceivedCounter.Add(ctx, 1, outputTypeValue, telemetryTypeValue) ``` --- @@ -68,36 +71,38 @@ output.BlitzOutputActiveWorkersGauge.Record(ctx, 1, outputTypeValue) | Property | Value | |----------|-------| -| **Type** | Float64 Counter | +| **Type** | Counter | | **Unit** | `{entry}/s` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | rate of log entries processed per second | -| **Attributes** | `output_type` | +| **Description** | rate of telemetry entries processed per second | +| **Attributes** |`output_type`, `telemetry_type` | **Attributes:** | Name | Type | Required | Values | |------|------|----------|--------| | `output_type` | string | ✓ | - | +| `telemetry_type` | string | ✓ | - | **Usage:** ```go -output.BlitzOutputEntryRateCounter.Add(ctx, 1.0, outputTypeValue) +// Type-safe wrapper with required attributes +blitzOutputEntryRateCounter.Add(ctx, 1, outputTypeValue, telemetryTypeValue) ``` --- -### blitz.output.request_size +### blitz.output.queue_size | Property | Value | |----------|-------| -| **Type** | Histogram | -| **Unit** | `By` | +| **Type** | Gauge | +| **Unit** | `{entry}` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | size of output requests in bytes | -| **Attributes** | `output_type` | +| **Description** | current number of entries in the output queue | +| **Attributes** |`output_type` | **Attributes:** @@ -105,81 +110,88 @@ output.BlitzOutputEntryRateCounter.Add(ctx, 1.0, outputTypeValue) |------|------|----------|--------| | `output_type` | string | ✓ | - | -**Usage:** -```go -output.BlitzOutputRequestSizeHistogram.Record(ctx, 1, outputTypeValue) -``` - --- ### blitz.output.request_latency | Property | Value | |----------|-------| -| **Type** | Float64 Histogram | +| **Type** | Histogram | | **Unit** | `s` | | **Meter** | `output` | | **Stability** | Stable | | **Description** | latency of output requests | -| **Attributes** | `output_type` | +| **Attributes** |`output_type`, `telemetry_type` | **Attributes:** | Name | Type | Required | Values | |------|------|----------|--------| | `output_type` | string | ✓ | - | +| `telemetry_type` | string | ✓ | - | **Usage:** ```go -output.BlitzOutputRequestLatencyHistogram.Record(ctx, 0.5, outputTypeValue) +// Type-safe wrapper with required attributes +blitzOutputRequestLatencyHistogram.Record(ctx, 1, outputTypeValue, telemetryTypeValue) ``` --- -### blitz.output.send_errors +### blitz.output.request_size | Property | Value | |----------|-------| -| **Type** | Counter | -| **Unit** | `{error}` | +| **Type** | Histogram | +| **Unit** | `By` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | total number of send errors | -| **Attributes** | `output_type` | +| **Description** | size of output requests in bytes | +| **Attributes** |`output_type`, `telemetry_type` | **Attributes:** | Name | Type | Required | Values | |------|------|----------|--------| | `output_type` | string | ✓ | - | +| `telemetry_type` | string | ✓ | - | **Usage:** ```go -output.BlitzOutputSendErrorsCounter.Add(ctx, 1, outputTypeValue) +// Type-safe wrapper with required attributes +blitzOutputRequestSizeHistogram.Record(ctx, 1, outputTypeValue, telemetryTypeValue) ``` --- -### blitz.output.queue_size +### blitz.output.send_errors | Property | Value | |----------|-------| -| **Type** | Observable Gauge | -| **Unit** | `{entry}` | +| **Type** | Counter | +| **Unit** | `{error}` | | **Meter** | `output` | | **Stability** | Stable | -| **Description** | current number of entries in the output queue | -| **Attributes** | `output_type` | +| **Description** | total number of send errors | +| **Attributes** |`output_type`, `telemetry_type` | **Attributes:** | Name | Type | Required | Values | |------|------|----------|--------| | `output_type` | string | ✓ | - | +| `telemetry_type` | string | ✓ | - | + +**Usage:** +```go +// Type-safe wrapper with required attributes +blitzOutputSendErrorsCounter.Add(ctx, 1, outputTypeValue, telemetryTypeValue) +``` --- + --- -**Generated:** `make generate-o11y` | **Registry:** `output/monitoring/` | **Templates:** `weaver/templates/` +**Generated:** `make generate-o11y` | **Registry:** `output/monitoring/` | **Templates:** `weaver/templates/` \ No newline at end of file diff --git a/output/otlp_grpc/metrics_traces.go b/output/otlp_grpc/metrics_traces.go index 53b94e7..18144a9 100644 --- a/output/otlp_grpc/metrics_traces.go +++ b/output/otlp_grpc/metrics_traces.go @@ -18,7 +18,7 @@ func (o *OTLPGrpc) WriteMetric(ctx context.Context, data output.MetricRecord) er select { case o.metricChan <- pbMetric: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "metrics") + o.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "metrics") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write metric: %w", ctx.Err()) @@ -33,7 +33,7 @@ func (o *OTLPGrpc) WriteTrace(ctx context.Context, data output.TraceRecord) erro select { case o.traceChan <- pbSpan: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "traces") + o.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "traces") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write trace: %w", ctx.Err()) diff --git a/output/otlp_grpc/otlp_grpc.go b/output/otlp_grpc/otlp_grpc.go index 56d2160..47610f0 100644 --- a/output/otlp_grpc/otlp_grpc.go +++ b/output/otlp_grpc/otlp_grpc.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/internal/workermanager" "github.com/observiq/blitz/output" "github.com/observiq/blitz/telemetry" @@ -71,6 +72,7 @@ type OTLPGrpcConfig struct { maxExportBatchSize int insecure bool tlsConfig *tls.Config + tel embed.TelemetrySettings } // WithHost sets the host for OTLP gRPC connections @@ -145,6 +147,14 @@ func WithTLSConfig(tlsConfig *tls.Config) OTLPGrpcOption { } } +// WithTelemetry sets the OTel providers blitz routes its self-telemetry through. +func WithTelemetry(tel embed.TelemetrySettings) OTLPGrpcOption { + return func(cfg *OTLPGrpcConfig) error { + cfg.tel = tel + return nil + } +} + // outputType is the output_type attribute value for OTLP gRPC metrics. const outputType = "otlp-grpc" @@ -162,6 +172,7 @@ type OTLPGrpc struct { ctx context.Context cancel context.CancelFunc workerManager *workermanager.WorkerManager + metrics *output.Metrics // Configuration batchTimeout time.Duration @@ -214,6 +225,12 @@ func New(logger *zap.Logger, opts ...OTLPGrpcOption) (*OTLPGrpc, error) { ctx, cancel := context.WithCancel(context.Background()) + m, err := output.NewMetrics(cfg.tel.MeterProvider) + if err != nil { + cancel() + return nil, fmt.Errorf("build output metrics: %w", err) + } + otlp := &OTLPGrpc{ logger: logger.Named("output-otlp-grpc"), host: cfg.host, @@ -230,6 +247,7 @@ func New(logger *zap.Logger, opts ...OTLPGrpcOption) (*OTLPGrpc, error) { requestTimeout: cfg.requestTimeout, maxQueueSize: cfg.maxQueueSize, maxExportBatchSize: cfg.maxExportBatchSize, + metrics: m, } otlp.logger.Info("Starting OTLP gRPC output", @@ -246,13 +264,16 @@ func New(logger *zap.Logger, opts ...OTLPGrpcOption) (*OTLPGrpc, error) { ) // Register observable metrics (queue_size) - output.InitObservableMetrics(otlp) + if err := otlp.metrics.InitObservable(otlp); err != nil { + cancel() + return nil, fmt.Errorf("init observable metrics: %w", err) + } // Create worker manager otlp.workerManager = workermanager.NewWorkerManager(otlp.logger, cfg.workers, otlp.otlpWorker) // Record initial active workers count - output.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(cfg.workers), outputType) + otlp.metrics.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(cfg.workers), outputType) // Start the workers otlp.workerManager.Start() @@ -328,7 +349,7 @@ func (o *OTLPGrpc) Write(ctx context.Context, data output.LogRecord) error { select { case o.dataChan <- record: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") + o.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write data: %w", ctx.Err()) @@ -345,7 +366,7 @@ func (o *OTLPGrpc) Stop(ctx context.Context) error { o.logger.Info("Stopping OTLP gRPC output") // Record zero active workers - output.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) + o.metrics.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) // Close the channels to ensure workers do not // process new data. @@ -526,15 +547,15 @@ func (o *OTLPGrpc) sendMetricBatch(client collectormetrics.MetricsServiceClient, startTime := time.Now() _, err := client.Export(ctx, request) if err != nil { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "metrics") + o.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "metrics") return fmt.Errorf("failed to export metrics: %w", err) } latency := time.Since(startTime).Seconds() requestSize := int64(proto.Size(request)) - output.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(metrics)), outputType, "metrics") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "metrics") - output.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "metrics") + o.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(metrics)), outputType, "metrics") + o.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "metrics") + o.metrics.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "metrics") return nil } @@ -557,15 +578,15 @@ func (o *OTLPGrpc) sendTraceBatch(client collectortrace.TraceServiceClient, batc startTime := time.Now() _, err := client.Export(ctx, request) if err != nil { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "traces") + o.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "traces") return fmt.Errorf("failed to export traces: %w", err) } latency := time.Since(startTime).Seconds() requestSize := int64(proto.Size(request)) - output.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(spans)), outputType, "traces") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "traces") - output.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "traces") + o.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(spans)), outputType, "traces") + o.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "traces") + o.metrics.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "traces") return nil } @@ -669,9 +690,9 @@ func (o *OTLPGrpc) sendBatch(client collectorlogs.LogsServiceClient, batch *logB // Record successful send metrics latency := time.Since(startTime).Seconds() requestSize := int64(proto.Size(request)) - output.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(logs)), outputType, "logs") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "logs") - output.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "logs") + o.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), float64(len(logs)), outputType, "logs") + o.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), requestSize, outputType, "logs") + o.metrics.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "logs") return nil } @@ -800,7 +821,7 @@ func (o *OTLPGrpc) toAnyValue(v any) *commonpb.AnyValue { // recordSendError records metrics for send errors func (o *OTLPGrpc) recordSendError(_ string, _ error) { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") + o.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") } // SupportedTelemetry returns the telemetry types this output can consume. diff --git a/output/syslog/syslog.go b/output/syslog/syslog.go index cd2f124..d610c4b 100644 --- a/output/syslog/syslog.go +++ b/output/syslog/syslog.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "github.com/observiq/blitz/output/syslog/ident" "github.com/observiq/blitz/output/tcp" @@ -61,6 +62,11 @@ type Config struct { // TCP-only TLS configuration. If nil, TLS is disabled. TLSConfig *tls.Config + + // Telemetry carries the OTel providers blitz routes its self-telemetry + // through, forwarded to the underlying TCP/UDP transport. The zero value + // falls back to the process-global providers. + Telemetry embed.TelemetrySettings } // Syslog implements output.Output by formatting records as syslog and delegating transport. @@ -107,6 +113,7 @@ func New(logger *zap.Logger, cfg Config) (*Syslog, error) { strconv.Itoa(cfg.Port), cfg.Workers, cfg.TLSConfig, + cfg.Telemetry, ) if err != nil { return nil, fmt.Errorf("create tcp transport: %w", err) @@ -117,6 +124,7 @@ func New(logger *zap.Logger, cfg Config) (*Syslog, error) { cfg.Host, strconv.Itoa(cfg.Port), cfg.Workers, + cfg.Telemetry, ) if err != nil { return nil, fmt.Errorf("create udp transport: %w", err) diff --git a/output/tcp/tcp.go b/output/tcp/tcp.go index d36a43d..05a9998 100644 --- a/output/tcp/tcp.go +++ b/output/tcp/tcp.go @@ -7,6 +7,7 @@ import ( "net" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/internal/workermanager" "github.com/observiq/blitz/output" "github.com/observiq/blitz/telemetry" @@ -45,10 +46,11 @@ type TCP struct { ctx context.Context cancel context.CancelFunc workerManager *workermanager.WorkerManager + metrics *output.Metrics } // New creates a new TCP output instance -func New(logger *zap.Logger, host, port string, workers int, tlsConfig *tls.Config) (*TCP, error) { +func New(logger *zap.Logger, host, port string, workers int, tlsConfig *tls.Config, tel embed.TelemetrySettings) (*TCP, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -62,6 +64,11 @@ func New(logger *zap.Logger, host, port string, workers int, tlsConfig *tls.Conf workers = DefaultTCPWorkers } + m, err := output.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build output metrics: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) tcp := &TCP{ @@ -73,6 +80,7 @@ func New(logger *zap.Logger, host, port string, workers int, tlsConfig *tls.Conf dataChan: make(chan string, DefaultTCPChannelSize), ctx: ctx, cancel: cancel, + metrics: m, } tcp.logger.Info("Starting TCP output", @@ -84,13 +92,15 @@ func New(logger *zap.Logger, host, port string, workers int, tlsConfig *tls.Conf ) // Register observable metrics (queue_size) - output.InitObservableMetrics(tcp) + if err := tcp.metrics.InitObservable(tcp); err != nil { + return nil, fmt.Errorf("init observable metrics: %w", err) + } // Create worker manager tcp.workerManager = workermanager.NewWorkerManager(tcp.logger, workers, tcp.tcpWorker) // Record initial active workers count - output.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) + tcp.metrics.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) // Start the workers tcp.workerManager.Start() @@ -111,7 +121,7 @@ func (t *TCP) ObserveBlitzOutputQueueSize(_ context.Context, observer metric.Int func (t *TCP) Write(ctx context.Context, data output.LogRecord) error { select { case t.dataChan <- data.Message: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") + t.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write data: %w", ctx.Err()) @@ -128,7 +138,7 @@ func (t *TCP) Stop(ctx context.Context) error { t.logger.Info("Stopping TCP output") // Record zero active workers - output.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) + t.metrics.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) // Close the channel to ensure workers do not // process new data. @@ -239,16 +249,16 @@ func (t *TCP) sendData(conn net.Conn, data string) error { // Record successful send metrics latency := time.Since(startTime).Seconds() - output.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") - output.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "logs") + t.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") + t.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") + t.metrics.BlitzOutputRequestLatencyHistogram.Record(context.Background(), latency, outputType, "logs") return nil } // recordSendError records metrics for send errors func (t *TCP) recordSendError(_ string, _ error) { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") + t.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") } // SupportedTelemetry returns the telemetry types this output can consume. diff --git a/output/tcp/tcp_bench_test.go b/output/tcp/tcp_bench_test.go index 80451d7..11af131 100644 --- a/output/tcp/tcp_bench_test.go +++ b/output/tcp/tcp_bench_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "go.uber.org/zap" ) @@ -166,7 +167,7 @@ func BenchmarkTCP_1Worker(b *testing.B) { } // Create TCP client with 1 worker - tcp, err := New(logger, host, port, 1, nil) + tcp, err := New(logger, host, port, 1, nil, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TCP client: %v", err) } @@ -201,7 +202,7 @@ func BenchmarkTCP_10Workers(b *testing.B) { } // Create TCP client with 10 workers - tcp, err := New(logger, host, port, 10, nil) + tcp, err := New(logger, host, port, 10, nil, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TCP client: %v", err) } @@ -237,7 +238,7 @@ func BenchmarkTCP_1Worker_Sequential(b *testing.B) { } // Create TCP client with 1 worker - tcp, err := New(logger, host, port, 1, nil) + tcp, err := New(logger, host, port, 1, nil, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TCP client: %v", err) } @@ -271,7 +272,7 @@ func BenchmarkTCP_10Workers_Sequential(b *testing.B) { } // Create TCP client with 10 workers - tcp, err := New(logger, host, port, 10, nil) + tcp, err := New(logger, host, port, 10, nil, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TCP client: %v", err) } @@ -360,7 +361,7 @@ func BenchmarkTCP_TLS_1Worker(b *testing.B) { } // Create TCP client with 1 worker and TLS - tcp, err := New(logger, host, port, 1, tlsConfig) + tcp, err := New(logger, host, port, 1, tlsConfig, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TLS TCP client: %v", err) } @@ -399,7 +400,7 @@ func BenchmarkTCP_TLS_10Workers(b *testing.B) { } // Create TCP client with 10 workers and TLS - tcp, err := New(logger, host, port, 10, tlsConfig) + tcp, err := New(logger, host, port, 10, tlsConfig, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TLS TCP client: %v", err) } @@ -437,7 +438,7 @@ func BenchmarkTCP_TLS_1Worker_Sequential(b *testing.B) { } // Create TCP client with 1 worker and TLS - tcp, err := New(logger, host, port, 1, tlsConfig) + tcp, err := New(logger, host, port, 1, tlsConfig, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TLS TCP client: %v", err) } @@ -473,7 +474,7 @@ func BenchmarkTCP_TLS_10Workers_Sequential(b *testing.B) { } // Create TCP client with 10 workers and TLS - tcp, err := New(logger, host, port, 10, tlsConfig) + tcp, err := New(logger, host, port, 10, tlsConfig, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create TLS TCP client: %v", err) } diff --git a/output/tcp/tcp_test.go b/output/tcp/tcp_test.go index 99c6344..1a2320c 100644 --- a/output/tcp/tcp_test.go +++ b/output/tcp/tcp_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -90,9 +91,9 @@ func TestNew(t *testing.T) { var err error if tt.name == "nil logger" { - tcp, err = New(nil, tt.host, tt.port, tt.workers, nil) + tcp, err = New(nil, tt.host, tt.port, tt.workers, nil, embed.NopTelemetry()) } else { - tcp, err = New(logger, tt.host, tt.port, tt.workers, nil) + tcp, err = New(logger, tt.host, tt.port, tt.workers, nil, embed.NopTelemetry()) } if tt.wantErr { @@ -166,7 +167,7 @@ func TestTCP_Integration(t *testing.T) { } // Create TCP client - tcp, err := New(logger, host, port, 1, nil) + tcp, err := New(logger, host, port, 1, nil, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create TCP client: %v", err) } @@ -248,7 +249,7 @@ func TestTCP_WriteAfterStop(t *testing.T) { } // Create TCP client - tcp, err := New(logger, host, port, 1, nil) + tcp, err := New(logger, host, port, 1, nil, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create TCP client: %v", err) } @@ -290,7 +291,7 @@ func TestTCP_StopTwice(t *testing.T) { } // Create TCP client - tcp, err := New(logger, host, port, 1, nil) + tcp, err := New(logger, host, port, 1, nil, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create TCP client: %v", err) } @@ -406,7 +407,7 @@ func TestTCP_IntegrationTLS(t *testing.T) { } // Create TCP client with TLS - tcp, err := New(logger, host, port, 1, clientTLSConfig) + tcp, err := New(logger, host, port, 1, clientTLSConfig, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create TLS TCP client: %v", err) } diff --git a/output/udp/udp.go b/output/udp/udp.go index 1150d49..438192c 100644 --- a/output/udp/udp.go +++ b/output/udp/udp.go @@ -6,6 +6,7 @@ import ( "net" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/internal/workermanager" "github.com/observiq/blitz/output" "github.com/observiq/blitz/telemetry" @@ -40,10 +41,11 @@ type UDP struct { ctx context.Context cancel context.CancelFunc workerManager *workermanager.WorkerManager + metrics *output.Metrics } // New creates a new UDP output instance -func New(logger *zap.Logger, host, port string, workers int) (*UDP, error) { +func New(logger *zap.Logger, host, port string, workers int, tel embed.TelemetrySettings) (*UDP, error) { if logger == nil { return nil, fmt.Errorf("logger cannot be nil") } @@ -57,6 +59,11 @@ func New(logger *zap.Logger, host, port string, workers int) (*UDP, error) { workers = DefaultUDPWorkers } + m, err := output.NewMetrics(tel.MeterProvider) + if err != nil { + return nil, fmt.Errorf("build output metrics: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) udp := &UDP{ @@ -67,6 +74,7 @@ func New(logger *zap.Logger, host, port string, workers int) (*UDP, error) { dataChan: make(chan string, DefaultUDPChannelSize), ctx: ctx, cancel: cancel, + metrics: m, } udp.logger.Info("Starting UDP output", @@ -77,13 +85,15 @@ func New(logger *zap.Logger, host, port string, workers int) (*UDP, error) { ) // Register observable metrics (queue_size) - output.InitObservableMetrics(udp) + if err := udp.metrics.InitObservable(udp); err != nil { + return nil, fmt.Errorf("init observable metrics: %w", err) + } // Create worker manager udp.workerManager = workermanager.NewWorkerManager(udp.logger, workers, udp.udpWorker) // Record initial active workers count - output.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) + udp.metrics.BlitzOutputActiveWorkersGauge.Record(context.Background(), int64(workers), outputType) // Start the workers udp.workerManager.Start() @@ -104,7 +114,7 @@ func (u *UDP) ObserveBlitzOutputQueueSize(_ context.Context, observer metric.Int func (u *UDP) Write(ctx context.Context, data output.LogRecord) error { select { case u.dataChan <- data.Message: - output.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") + u.metrics.BlitzOutputEntriesReceivedCounter.Add(ctx, 1, outputType, "logs") return nil case <-ctx.Done(): return fmt.Errorf("context cancelled while waiting to write data: %w", ctx.Err()) @@ -121,7 +131,7 @@ func (u *UDP) Stop(ctx context.Context) error { u.logger.Info("Stopping UDP output") // Record zero active workers - output.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) + u.metrics.BlitzOutputActiveWorkersGauge.Record(ctx, 0, outputType) // Close the channel to ensure workers do not // process new data. @@ -209,15 +219,15 @@ func (u *UDP) sendData(conn net.Conn, data string) error { } // Record successful send metrics - output.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") - output.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") + u.metrics.BlitzOutputEntryRateCounter.Add(context.Background(), 1.0, outputType, "logs") + u.metrics.BlitzOutputRequestSizeHistogram.Record(context.Background(), int64(bytesWritten), outputType, "logs") return nil } // recordSendError records metrics for send errors func (u *UDP) recordSendError(_ string, _ error) { - output.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") + u.metrics.BlitzOutputSendErrorsCounter.Add(context.Background(), 1, outputType, "logs") } // SupportedTelemetry returns the telemetry types this output can consume. diff --git a/output/udp/udp_bench_test.go b/output/udp/udp_bench_test.go index a98d4d9..e00bf01 100644 --- a/output/udp/udp_bench_test.go +++ b/output/udp/udp_bench_test.go @@ -6,6 +6,7 @@ import ( "sync" "testing" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "go.uber.org/zap" ) @@ -59,7 +60,7 @@ func BenchmarkUDP_1Worker(b *testing.B) { } // Create UDP client with 1 worker - udp, err := New(logger, host, port, 1) + udp, err := New(logger, host, port, 1, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create UDP client: %v", err) } @@ -94,7 +95,7 @@ func BenchmarkUDP_10Workers(b *testing.B) { } // Create UDP client with 10 workers - udp, err := New(logger, host, port, 10) + udp, err := New(logger, host, port, 10, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create UDP client: %v", err) } @@ -130,7 +131,7 @@ func BenchmarkUDP_1Worker_Sequential(b *testing.B) { } // Create UDP client with 1 worker - udp, err := New(logger, host, port, 1) + udp, err := New(logger, host, port, 1, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create UDP client: %v", err) } @@ -164,7 +165,7 @@ func BenchmarkUDP_10Workers_Sequential(b *testing.B) { } // Create UDP client with 10 workers - udp, err := New(logger, host, port, 10) + udp, err := New(logger, host, port, 10, embed.NopTelemetry()) if err != nil { b.Fatalf("Failed to create UDP client: %v", err) } diff --git a/output/udp/udp_test.go b/output/udp/udp_test.go index 035ee62..daf8f9b 100644 --- a/output/udp/udp_test.go +++ b/output/udp/udp_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/observiq/blitz/embed" "github.com/observiq/blitz/output" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -85,9 +86,9 @@ func TestNew(t *testing.T) { var err error if tt.name == "nil logger" { - udp, err = New(nil, tt.host, tt.port, tt.workers) + udp, err = New(nil, tt.host, tt.port, tt.workers, embed.NopTelemetry()) } else { - udp, err = New(logger, tt.host, tt.port, tt.workers) + udp, err = New(logger, tt.host, tt.port, tt.workers, embed.NopTelemetry()) } if tt.wantErr { @@ -161,7 +162,7 @@ func TestUDP_Integration(t *testing.T) { } // Create UDP client - udp, err := New(logger, host, port, 1) + udp, err := New(logger, host, port, 1, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create UDP client: %v", err) } @@ -240,7 +241,7 @@ func TestUDP_WriteAfterStop(t *testing.T) { } // Create UDP client - udp, err := New(logger, host, port, 1) + udp, err := New(logger, host, port, 1, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create UDP client: %v", err) } @@ -282,7 +283,7 @@ func TestUDP_StopTwice(t *testing.T) { } // Create UDP client - udp, err := New(logger, host, port, 1) + udp, err := New(logger, host, port, 1, embed.NopTelemetry()) if err != nil { t.Fatalf("Failed to create UDP client: %v", err) } diff --git a/weaver/templates/go/metric.go.j2 b/weaver/templates/go/metric.go.j2 index b18e8d1..bf25cdd 100644 --- a/weaver/templates/go/metric.go.j2 +++ b/weaver/templates/go/metric.go.j2 @@ -21,7 +21,7 @@ {%- endif -%} {%- endmacro -%} {%- macro observe_method_name(metric_name) -%} -observe{{ smart_title_case(metric_name) }} +Observe{{ smart_title_case(metric_name) }} {%- endmacro -%} {%- set registry_name = ctx[0].id.split('.')[0] if ctx and ctx | length > 0 else "unknown" -%} {%- set package_name = registry_name.split('/')[-1] if '/' in registry_name else registry_name -%} @@ -59,7 +59,6 @@ import ( "context" {%- endif %} "errors" - "fmt" "go.opentelemetry.io/otel" {%- if has_any_required_attrs.value or has_any_enum_attrs.value %} @@ -149,7 +148,7 @@ const ( {%- set method_name = "Record" if uses_record else "Add" %} {%- set option_type = "RecordOption" if uses_record else "AddOption" %} {%- set field_name = "histogram" if metric.instrument in ["histogram", "Histogram"] else ("gauge" if metric.instrument in ["gauge", "Gauge"] else "counter") %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} +{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type and metric.annotations.histogram.type[:7] == "Float64" %} {%- set value_type = "float64" if is_float64 else "int64" %} // {{ metric_type_name }} wraps the {{ metric.metric_name }} metric with type-safe required attributes @@ -215,11 +214,11 @@ func (m {{ metric_type_name }}) {{ method_name }}(ctx context.Context, value {{ {%- if has_observable_metrics.value %} -// observableCallbacks defines the callback interface for observable metrics -type observableCallbacks interface { +// ObservableCallbacks defines the callback interface for observable metrics +type ObservableCallbacks interface { {%- for metric in ctx %} {%- if metric.annotations and metric.annotations.observable %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} +{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type and metric.annotations.histogram.type[:7] == "Float64" %} {%- set observer_type = "Float64Observer" if is_float64 else "Int64Observer" %} {{ observe_method_name(metric.metric_name) }}(ctx context.Context, observer metric.{{ observer_type }}) error {%- endif %} @@ -227,45 +226,8 @@ type observableCallbacks interface { } {%- endif %} -var ( -{%- for meter_name in unique_meters.names | sort %} - {{ lower_first(smart_title_case(meter_name)) }}Meter = otel.Meter("{{ meter_name }}") -{%- endfor %} - -{% for metric in ctx %} -{%- set is_observable = metric.annotations and metric.annotations.observable %} -{%- if not is_observable %} -{%- set has_required_attrs = metric.attributes is defined and metric.attributes | selectattr("requirement_level", "equalto", "required") | list | length > 0 %} -{%- if not has_required_attrs %} -{%- set is_exported = metric.annotations and metric.annotations.exported %} -{%- if is_exported %} -{%- set metric_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} -{%- else %} -{%- set metric_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} -{%- endif %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} - // {{ metric.brief }} - {{ metric_var_name }} metric.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %} -{% endif %} -{%- endif %} -{%- endfor %} -{%- for metric in ctx %} -{%- set is_observable = metric.annotations and metric.annotations.observable %} -{%- if not is_observable %} -{%- set has_required_attrs = metric.attributes is defined and metric.attributes | selectattr("requirement_level", "equalto", "required") | list | length > 0 %} -{%- if has_required_attrs %} -{%- set is_exported = metric.annotations and metric.annotations.exported %} -{%- if is_exported %} -{%- set wrapper_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} -{%- else %} -{%- set wrapper_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} -{%- endif %} - // {{ metric.brief }} - {{ wrapper_var_name }} {{ lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) ~ "Type" }} -{% endif %} -{%- endif %} -{%- endfor %} {%- set processed_attr_members = namespace(names=[], attr_groups=[]) -%} +{%- set attr_member_lines = namespace(items=[]) -%} {%- for metric in ctx %} {%- if metric.attributes is defined %} {%- for attr in metric.attributes %} @@ -273,25 +235,18 @@ var ( {%- for member in attr.type.members %} {%- set attr_var_name = "attr" ~ smart_title_case(attr.name) ~ smart_title_case(member.id) %} {%- if attr_var_name not in processed_attr_members.names %} -{%- if attr.name not in processed_attr_members.attr_groups and processed_attr_members.attr_groups | length > 0 %} - -{% endif %} -{%- if attr.name not in processed_attr_members.attr_groups %} -{%- set processed_attr_members.attr_groups = processed_attr_members.attr_groups + [attr.name] %} - // {{ attr.name }} -{%- endif %} {%- set processed_attr_members.names = processed_attr_members.names + [attr_var_name] %} {%- set attr_key = attr.annotations['attribute.name'] if attr.annotations and 'attribute.name' in attr.annotations else attr.name %} {%- set enum_const = lower_first(smart_title_case(attr.name)) ~ smart_title_case(member.id) %} - {{ attr_var_name }} = attribute.String(attr{{ smart_title_case(attr_key) }}, string({{ enum_const }})) +{%- set attr_member_lines.items = attr_member_lines.items + ["\t" ~ attr_var_name ~ " = attribute.String(attr" ~ smart_title_case(attr_key) ~ ", string(" ~ enum_const ~ "))"] %} {%- endif %} {%- endfor %} {%- endif %} -{% endfor %} +{%- endfor %} {%- endif %} {%- endfor %} - -{%- set processed_attr_sets = namespace(names=[], attr_groups=[]) -%} +{%- set processed_attr_sets = namespace(names=[]) -%} +{%- set attr_set_lines = namespace(items=[]) -%} {%- for metric in ctx %} {%- if metric.attributes is defined %} {%- for attr in metric.attributes %} @@ -302,15 +257,8 @@ var ( {%- set attr_var_name = "attr" ~ title_case_name %} {%- set attr_set_name = "attrSet" ~ title_case_name %} {%- if attr_set_name not in processed_attr_sets.names %} -{%- if attr.name not in processed_attr_sets.attr_groups and processed_attr_sets.attr_groups | length > 0 %} - -{% endif %} -{%- if attr.name not in processed_attr_sets.attr_groups %} -{%- set processed_attr_sets.attr_groups = processed_attr_sets.attr_groups + [attr.name] %} - // {{ attr.name }} -{%- endif %} {%- set processed_attr_sets.names = processed_attr_sets.names + [attr_set_name] %} - {{ attr_set_name }} = metric.WithAttributeSet(attribute.NewSet({{ attr_var_name }})) +{%- set attr_set_lines.items = attr_set_lines.items + ["\t" ~ attr_set_name ~ " = metric.WithAttributeSet(attribute.NewSet(" ~ attr_var_name ~ "))"] %} {%- endif %} {%- endfor %} {%- endif %} @@ -318,112 +266,76 @@ var ( {%- endfor %} {%- endif %} {%- endfor %} -) -{%- set has_non_optional_non_observable_metrics = namespace(value=false) -%} -{%- for metric in ctx -%} - {%- set is_optional = metric.annotations and metric.annotations.optional -%} - {%- set is_observable = metric.annotations and metric.annotations.observable -%} - {%- if not is_optional and not is_observable -%} - {%- set has_non_optional_non_observable_metrics.value = true -%} - {%- endif -%} -{%- endfor -%} +{%- if attr_member_lines.items | length > 0 or attr_set_lines.items | length > 0 %} -{%- if has_non_optional_non_observable_metrics.value %} +// Attribute value options (provider-independent) +var ( +{%- for line in attr_member_lines.items %} +{{ line }} +{%- endfor %} +{%- if attr_set_lines.items | length > 0 %} -func init() { - var err, errs error +{% for line in attr_set_lines.items %}{{ line }} +{% endfor -%} +{%- endif %} +) +{%- endif %} +// Metrics holds blitz's {{ package_name }} self-telemetry instruments. Build one +// with NewMetrics from a caller-supplied MeterProvider so metrics can be routed +// to any provider (an embedding host's, or the process global). +type Metrics struct { +{%- for meter_name in unique_meters.names | sort %} + {{ lower_first(smart_title_case(meter_name)) }}Meter metric.Meter +{%- endfor %} {% for metric in ctx %} -{%- set is_optional = metric.annotations and metric.annotations.optional %} {%- set is_observable = metric.annotations and metric.annotations.observable %} -{%- if not is_optional and not is_observable %} +{%- if not is_observable %} {%- set has_required_attrs = metric.attributes is defined and metric.attributes | selectattr("requirement_level", "equalto", "required") | list | length > 0 %} {%- set is_exported = metric.annotations and metric.annotations.exported %} {%- if is_exported %} -{%- set metric_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} +{%- set field_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} {%- else %} -{%- set metric_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} +{%- set field_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} {%- endif %} -{%- set metric_meter_name = metric.annotations.meter if metric.annotations and metric.annotations.meter else registry_name %} -{%- set meter_var = lower_first(smart_title_case(metric_meter_name)) ~ "Meter" %} +{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type and metric.annotations.histogram.type[:7] == "Float64" %} + // {{ metric.brief }} {%- if has_required_attrs %} -{%- set raw_counter_var = metric_var_name ~ "Raw" %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} - {{ raw_counter_var }}, err := {{ meter_var }}.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %}( - "{{metric.metric_name}}", - metric.WithDescription("{{metric.brief}}"), -{%- if metric.unit %} - metric.WithUnit("{{metric.unit}}"), -{%- endif %} -{%- if metric.instrument == "histogram" and metric.annotations and metric.annotations.histogram and metric.annotations.histogram.explicit_bucket_boundaries %} - metric.WithExplicitBucketBoundaries({{ metric.annotations.histogram.explicit_bucket_boundaries | join(", ") }}), -{%- endif %} - ) - errs = errors.Join(errs, err) -{%- if is_exported %} -{%- set wrapper_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} -{%- else %} -{%- set wrapper_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} -{%- endif %} - -{%- set field_name = "histogram" if metric.instrument in ["histogram", "Histogram"] else ("gauge" if metric.instrument in ["gauge", "Gauge"] else "counter") %} - {{ wrapper_var_name }} = {{ lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) ~ "Type" }}{ {{- field_name -}}: {{ raw_counter_var }}} + {{ field_name }} {{ lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) ~ "Type" }} {%- else %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} - {{ metric_var_name }}, err = {{ meter_var }}.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %}( - "{{metric.metric_name}}", - metric.WithDescription("{{metric.brief}}"), -{%- if metric.unit %} - metric.WithUnit("{{metric.unit}}"), -{%- endif %} -{%- if metric.instrument == "histogram" and metric.annotations and metric.annotations.histogram and metric.annotations.histogram.explicit_bucket_boundaries %} - metric.WithExplicitBucketBoundaries({{ metric.annotations.histogram.explicit_bucket_boundaries | join(", ") }}), -{%- endif %} - ) - errs = errors.Join(errs, err) + {{ field_name }} metric.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %} {%- endif %} - {%- endif %} -{% endfor %} - if errs != nil { - panic(fmt.Sprintf("Initialize {{ package_name }} metrics: %s", errs)) - } +{%- endfor %} } -{%- endif %} - -{%- set has_optional_non_observable_metrics = namespace(value=false) -%} -{%- for metric in ctx -%} - {%- set is_optional = metric.annotations and metric.annotations.optional -%} - {%- set is_observable = metric.annotations and metric.annotations.observable -%} - {%- if is_optional and not is_observable -%} - {%- set has_optional_non_observable_metrics.value = true -%} - {%- endif -%} -{%- endfor -%} - -{%- if has_optional_non_observable_metrics.value %} - -// initOptionalMetrics initializes optional metrics that can be enabled when needed -func initOptionalMetrics() { - var err, errs error +// NewMetrics builds the {{ package_name }} instruments from mp. A nil mp falls +// back to the process-global MeterProvider, preserving standalone behavior. +func NewMetrics(mp metric.MeterProvider) (*Metrics, error) { + if mp == nil { + mp = otel.GetMeterProvider() + } + m := &Metrics{} + var errs error +{% for meter_name in unique_meters.names | sort %} + m.{{ lower_first(smart_title_case(meter_name)) }}Meter = mp.Meter("{{ meter_name }}") +{%- endfor %} {% for metric in ctx %} -{%- set is_optional = metric.annotations and metric.annotations.optional %} {%- set is_observable = metric.annotations and metric.annotations.observable %} -{%- if is_optional and not is_observable %} +{%- if not is_observable %} {%- set has_required_attrs = metric.attributes is defined and metric.attributes | selectattr("requirement_level", "equalto", "required") | list | length > 0 %} {%- set is_exported = metric.annotations and metric.annotations.exported %} {%- if is_exported %} -{%- set metric_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} +{%- set field_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} {%- else %} -{%- set metric_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} +{%- set field_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} {%- endif %} {%- set metric_meter_name = metric.annotations.meter if metric.annotations and metric.annotations.meter else registry_name %} {%- set meter_var = lower_first(smart_title_case(metric_meter_name)) ~ "Meter" %} -{%- if has_required_attrs %} -{%- set raw_counter_var = metric_var_name ~ "Raw" %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} - {{ raw_counter_var }}, err := {{ meter_var }}.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %}( +{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type and metric.annotations.histogram.type[:7] == "Float64" %} +{%- set field_kind = "histogram" if metric.instrument in ["histogram", "Histogram"] else ("gauge" if metric.instrument in ["gauge", "Gauge"] else "counter") %} + {{ field_name }}Raw, err := m.{{ meter_var }}.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %}( "{{metric.metric_name}}", metric.WithDescription("{{metric.brief}}"), {%- if metric.unit %} @@ -434,52 +346,30 @@ func initOptionalMetrics() { {%- endif %} ) errs = errors.Join(errs, err) -{%- if is_exported %} -{%- set wrapper_var_name = smart_title_case(metric.metric_name) ~ smart_title_case(metric.instrument) %} -{%- else %} -{%- set wrapper_var_name = lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) %} -{%- endif %} - -{%- set field_name = "histogram" if metric.instrument in ["histogram", "Histogram"] else ("gauge" if metric.instrument in ["gauge", "Gauge"] else "counter") %} - {{ wrapper_var_name }} = {{ lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) ~ "Type" }}{ {{- field_name -}}: {{ raw_counter_var }}} +{%- if has_required_attrs %} + m.{{ field_name }} = {{ lower_first(smart_title_case(metric.metric_name)) ~ smart_title_case(metric.instrument) ~ "Type" }}{ {{- field_kind -}}: {{ field_name }}Raw} {%- else %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} - {{ metric_var_name }}, err = {{ meter_var }}.{% if is_float64 %}Float64{{metric.instrument | map_text("metric_type_interface") | replace("Int64", "")}}{% else %}{{metric.instrument | map_text("metric_type_interface")}}{% endif %}( - "{{metric.metric_name}}", - metric.WithDescription("{{metric.brief}}"), -{%- if metric.unit %} - metric.WithUnit("{{metric.unit}}"), + m.{{ field_name }} = {{ field_name }}Raw {%- endif %} -{%- if metric.instrument == "histogram" and metric.annotations and metric.annotations.histogram and metric.annotations.histogram.explicit_bucket_boundaries %} - metric.WithExplicitBucketBoundaries({{ metric.annotations.histogram.explicit_bucket_boundaries | join(", ") }}), -{%- endif %} - ) - errs = errors.Join(errs, err) -{%- endif %} - -{%- endif %} -{% endfor %} - if errs != nil { - panic(fmt.Sprintf("Initialize {{ package_name }} optional metrics: %s", errs)) - } +{% endif %} +{%- endfor %} + return m, errs } -{%- endif %} - {%- if has_observable_metrics.value %} -// initObservableMetrics initializes observable metrics with the provided callbacks -func initObservableMetrics(callbacks observableCallbacks) { +// InitObservable registers the observable instruments against callbacks. Call +// it once, after NewMetrics, from the component that owns the observed state. +func (m *Metrics) InitObservable(callbacks ObservableCallbacks) error { var err, errs error - {% for metric in ctx %} {%- if metric.annotations and metric.annotations.observable %} {%- set metric_meter_name = metric.annotations.meter if metric.annotations and metric.annotations.meter else registry_name %} {%- set meter_var = lower_first(smart_title_case(metric_meter_name)) ~ "Meter" %} {%- set is_counter = metric.instrument == "counter" %} {%- set instrument_type = "ObservableCounter" if is_counter else "ObservableGauge" %} -{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type == "Float64Histogram" %} +{%- set is_float64 = metric.annotations and metric.annotations.histogram and metric.annotations.histogram.type and metric.annotations.histogram.type[:7] == "Float64" %} {%- set observer_func = "Float64" if is_float64 else "Int64" %} - _, err = {{ meter_var }}.{{ observer_func }}{{ instrument_type }}( + _, err = m.{{ meter_var }}.{{ observer_func }}{{ instrument_type }}( "{{metric.metric_name}}", metric.WithDescription("{{metric.brief}}"), {%- if metric.unit %} @@ -488,11 +378,8 @@ func initObservableMetrics(callbacks observableCallbacks) { metric.With{{ observer_func }}Callback(callbacks.{{ observe_method_name(metric.metric_name) }}), ) errs = errors.Join(errs, err) - -{%- endif %} -{% endfor %} - if errs != nil { - panic(fmt.Sprintf("Initialize {{ package_name }} observable metrics: %s", errs)) - } +{% endif %} +{%- endfor %} + return errs } {%- endif %}