From 4fdebbc119ff4d337833b622b01eaad09db6caa5 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 16 Jun 2026 13:56:12 +0000 Subject: [PATCH 01/28] Add MySQL receiver translator and register component Create the core MySQL receiver translator with functional options pattern: - translator.go with endpoint, username, passfile, TLS, localhost detection - translator_test.go with unit tests for all option functions - Register mysqlreceiver.NewFactory() in defaultcomponents - Add MySQLKey and MySQL DBI component name constants to common.go - Add go.mod dependency on mysqlreceiver --- go.mod | 4 +- go.sum | 2 + service/defaultcomponents/components.go | 2 + service/defaultcomponents/components_test.go | 1 + translator/translate/otel/common/common.go | 8 +- .../otel/receiver/mysql/translator.go | 93 +++++++++++++++++++ .../otel/receiver/mysql/translator_test.go | 83 +++++++++++++++++ 7 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 translator/translate/otel/receiver/mysql/translator.go create mode 100644 translator/translate/otel/receiver/mysql/translator_test.go diff --git a/go.mod b/go.mod index fce668d5a3..81e5202fe8 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 ) @@ -195,6 +196,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.124.1 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver v0.124.1 @@ -619,7 +621,7 @@ require ( go.opentelemetry.io/collector/component/componentstatus v0.124.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.30.0 go.opentelemetry.io/collector/config/configgrpc v0.124.0 // indirect - go.opentelemetry.io/collector/config/confignet v1.30.0 // indirect + go.opentelemetry.io/collector/config/confignet v1.30.0 go.opentelemetry.io/collector/config/configretry v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/httpprovider v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.30.0 // indirect diff --git a/go.sum b/go.sum index b535d7aebc..3ae7afa9bc 100644 --- a/go.sum +++ b/go.sum @@ -278,6 +278,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 h1:R4Ems+YMAXLtsEZwRXw8/cf1NkrzHQsc25WPdYbUaTM= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a h1:ScKUmN6HDigSB0rceZR3CmcO/CVDTxOtrk8XQSTC+Q4= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a/go.mod h1:tnnmAnxuLxWKBG7NjOI3po8R+EAX/9cG1y7TWiQTMvA= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 h1:ZLP3mZHMXzUjiXxXFVI9Z5LtU25qT13Qajz84mna/LY= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gSxi2fY+T8mNzlaU7qOEnaXTcKfhIpijDSyeCu5GOYo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 h1:p0HB9BwhUdkaI0ALIb3hYxi7oCTDsGUuGadeM3R36AM= diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 02731a24f2..efdabb5cd8 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -53,6 +53,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver" @@ -116,6 +117,7 @@ func Factories() (otelcol.Factories, error) { nopreceiver.NewFactory(), otlpreceiver.NewFactory(), prometheusreceiver.NewFactory(), + mysqlreceiver.NewFactory(), postgresqlreceiver.NewFactory(), statsdreceiver.NewFactory(), systemmetricsreceiver.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index e979358847..e14b67bf93 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -35,6 +35,7 @@ func TestComponents(t *testing.T) { "nop", "otlp", "prometheus", + "mysql", "postgresql", "statsd", "systemmetrics", diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index ad112c23c8..12cec15406 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -140,14 +140,20 @@ const ( const ( DatabaseInsightsKey = "database_insights" PostgreSQLKey = "postgresql" + MySQLKey = "mysql" - // DBI component names + // DBI component names — PostgreSQL DbiTransformFixStartTime = "dbi_fix_start_time" DbiTransformResource = "dbi_resource" DbiTransformLogs = "dbi_logs" DbiFilterExcludeMonitor = "dbi_exclude_monitor" DbiConnectorDbload = "dbi_dbload" DbiConnectorTopsql = "dbi_topsql" + + // DBI component names — MySQL + DbiTransformFixStartTimeMysql = "dbi_fix_start_time_mysql" + DbiConnectorDbloadMysql = "dbi_dbload_mysql" + DbiConnectorTopsqlMysql = "dbi_topsql_mysql" ) // OpenTelemetry pipeline processor names diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go new file mode 100644 index 0000000000..2dabba46c4 --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "strconv" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/receiver" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type Option func(*translator) + +type translator struct { + factory receiver.Factory + name string + endpoint string + username string + passfile string + caFile string + isLocalhost bool + index int + topQueryInterval time.Duration +} + +func WithName(name string) Option { return func(t *translator) { t.name = name } } +func WithEndpoint(ep string) Option { return func(t *translator) { t.endpoint = ep } } +func WithUsername(u string) Option { return func(t *translator) { t.username = u } } +func WithPassfile(p string) Option { return func(t *translator) { t.passfile = p } } +func WithCAFile(ca string) Option { return func(t *translator) { t.caFile = ca } } +func WithIsLocalhost(b bool) Option { return func(t *translator) { t.isLocalhost = b } } +func WithIndex(i int) Option { return func(t *translator) { t.index = i } } +func WithTopQueryInterval(d time.Duration) Option { + return func(t *translator) { t.topQueryInterval = d } +} + +func NewTranslator(opts ...Option) common.ComponentTranslator { + t := &translator{ + factory: mysqlreceiver.NewFactory(), + name: "metrics", + topQueryInterval: 60 * time.Second, + } + for _, opt := range opts { + opt(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.MustNewIDWithName("mysql", t.name+"_"+strconv.Itoa(t.index)) +} + +func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*mysqlreceiver.Config) + + cfg.Endpoint = t.endpoint + cfg.Username = t.username + cfg.Passfile = t.passfile + cfg.Transport = confignet.TransportTypeTCP + + if t.isLocalhost { + cfg.TLS = configtls.ClientConfig{ + Insecure: true, + } + } else { + cfg.TLS = configtls.ClientConfig{ + Config: configtls.Config{ + CAFile: t.caFile, + }, + } + } + + cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true + cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true + + cfg.QuerySampleCollection.MaxRowsPerQuery = 500 + + cfg.TopQueryCollection.CollectionInterval = t.topQueryInterval + cfg.TopQueryCollection.TopQueryCount = 200 + cfg.TopQueryCollection.MaxQuerySampleCount = 5000 + cfg.TopQueryCollection.QueryPlanCacheSize = 1000 + cfg.TopQueryCollection.QueryPlanCacheTTL = time.Hour + + return cfg, nil +} diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go new file mode 100644 index 0000000000..769d19872d --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -0,0 +1,83 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslator_ID(t *testing.T) { + tr := NewTranslator(WithName("metrics"), WithIndex(0)) + assert.Equal(t, "mysql/metrics_0", tr.ID().String()) + + tr = NewTranslator(WithName("events"), WithIndex(1)) + assert.Equal(t, "mysql/events_1", tr.ID().String()) +} + +func TestTranslator_Translate_Localhost(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithIsLocalhost(true), + WithIndex(0), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, "localhost:3306", mysqlCfg.Endpoint) + assert.Equal(t, "cw_monitor", mysqlCfg.Username) + assert.Equal(t, "/etc/.mysql_credentials", mysqlCfg.Passfile) + assert.True(t, mysqlCfg.TLS.Insecure) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) + assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.Equal(t, uint64(200), mysqlCfg.TopQueryCollection.TopQueryCount) + assert.Equal(t, uint64(5000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) + assert.Equal(t, 1000, mysqlCfg.TopQueryCollection.QueryPlanCacheSize) + assert.Equal(t, time.Hour, mysqlCfg.TopQueryCollection.QueryPlanCacheTTL) + assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) +} + +func TestTranslator_Translate_CustomInterval(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithIsLocalhost(true), + WithTopQueryInterval(30*time.Second), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + assert.Equal(t, 30*time.Second, cfg.(*mysqlreceiver.Config).TopQueryCollection.CollectionInterval) +} + +func TestTranslator_Translate_Remote(t *testing.T) { + tr := NewTranslator( + WithName("events"), + WithEndpoint("db.example.com:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithCAFile("/etc/ssl/ca.pem"), + WithIsLocalhost(false), + WithIndex(0), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, "db.example.com:3306", mysqlCfg.Endpoint) + assert.False(t, mysqlCfg.TLS.Insecure) + assert.Equal(t, "/etc/ssl/ca.pem", string(mysqlCfg.TLS.CAFile)) + assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) + assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) +} From 6c32dcd3bf9ef827f25c94d6df3a986afde4a2d7 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 16 Jun 2026 13:56:29 +0000 Subject: [PATCH 02/28] Add MySQL-specific connector and processor configurations Add YAML configs and translator wiring for MySQL DBI pipelines: - dbi_dbload_mysql.yaml: count connector for active sessions by wait/user/db/sql/host - dbi_topsql_mysql.yaml: signaltometrics connector for top query metrics - transform_dbi_fix_start_time_mysql.yaml: transform processor for MySQL timestamps - Update count, signaltometrics, and transform translators to handle MySQL component names - Add unit tests for MySQL connector/processor creation --- .../connector/count/dbi_dbload_mysql.yaml | 56 +++++++++++++++++++ .../otel/connector/count/translator.go | 8 ++- .../otel/connector/count/translator_test.go | 15 +++++ .../signaltometrics/dbi_topsql_mysql.yaml | 27 +++++++++ .../connector/signaltometrics/translator.go | 8 ++- .../signaltometrics/translator_test.go | 15 +++++ .../transform_dbi_fix_start_time_mysql.yaml | 10 ++++ .../processor/transformprocessor/translate.go | 8 ++- .../transformprocessor/translator_test.go | 15 +++++ 9 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 translator/translate/otel/connector/count/dbi_dbload_mysql.yaml create mode 100644 translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml create mode 100644 translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml diff --git a/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml new file mode 100644 index 0000000000..0e0f381c1b --- /dev/null +++ b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml @@ -0,0 +1,56 @@ +logs: + mysql.active_sessions.by_wait: + description: "Active sessions by wait event" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_user: + description: "Active sessions by user" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + mysql.active_sessions.by_db: + description: "Active sessions by database" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: db.namespace + mysql.active_sessions.by_sql: + description: "Active sessions by query digest" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + mysql.active_sessions.by_sql_wait: + description: "Active sessions by query and wait" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_host: + description: "Active sessions by host" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + mysql.active_sessions.count: + description: "Total active sessions" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: db.namespace diff --git a/translator/translate/otel/connector/count/translator.go b/translator/translate/otel/connector/count/translator.go index ad803f4386..c8f2921518 100644 --- a/translator/translate/otel/connector/count/translator.go +++ b/translator/translate/otel/connector/count/translator.go @@ -18,6 +18,9 @@ import ( //go:embed dbi_dbload.yaml var dbiDbloadConfig string +//go:embed dbi_dbload_mysql.yaml +var dbiDbloadMysqlConfig string + type translator struct { name string factory connector.Factory @@ -40,8 +43,11 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*countconnector.Config) - if t.name == common.DbiConnectorDbload { + switch t.name { + case common.DbiConnectorDbload: return common.GetYamlFileToYamlConfig(cfg, dbiDbloadConfig) + case common.DbiConnectorDbloadMysql: + return common.GetYamlFileToYamlConfig(cfg, dbiDbloadMysqlConfig) } return nil, fmt.Errorf("unsupported count connector config: %s", t.name) diff --git a/translator/translate/otel/connector/count/translator_test.go b/translator/translate/otel/connector/count/translator_test.go index 0721d3de66..c94fee1546 100644 --- a/translator/translate/otel/connector/count/translator_test.go +++ b/translator/translate/otel/connector/count/translator_test.go @@ -41,3 +41,18 @@ func TestTranslateUnsupported(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "unsupported count connector config") } + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbloadMysql) + assert.Equal(t, "count/dbi_dbload_mysql", tr.ID().String()) +} + +func TestTranslateDbloadMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbloadMysql) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + countCfg := cfg.(*countconnector.Config) + assert.Len(t, countCfg.Logs, 7) + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.count") +} diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml new file mode 100644 index 0000000000..704317a010 --- /dev/null +++ b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml @@ -0,0 +1,27 @@ +logs: + - name: mysql.count_star + include_resource_attributes: + - key: service.instance.id + description: "SQL execution count" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + - key: user.name + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.count_star"])' + - name: mysql.sum_timer_wait + include_resource_attributes: + - key: service.instance.id + description: "Total execution time in picoseconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + - key: user.name + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"])' diff --git a/translator/translate/otel/connector/signaltometrics/translator.go b/translator/translate/otel/connector/signaltometrics/translator.go index 313b2f8f71..4ca189ffa5 100644 --- a/translator/translate/otel/connector/signaltometrics/translator.go +++ b/translator/translate/otel/connector/signaltometrics/translator.go @@ -19,6 +19,9 @@ import ( //go:embed dbi_topsql.yaml var dbiTopsqlConfig string +//go:embed dbi_topsql_mysql.yaml +var dbiTopsqlMysqlConfig string + type translator struct { name string factory connector.Factory @@ -41,8 +44,11 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*signaltometricsconfig.Config) - if t.name == common.DbiConnectorTopsql { + switch t.name { + case common.DbiConnectorTopsql: return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlConfig) + case common.DbiConnectorTopsqlMysql: + return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlMysqlConfig) } return nil, fmt.Errorf("unsupported signaltometrics connector config: %s", t.name) diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index bcb37f4080..17d94134d9 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -41,3 +41,18 @@ func TestTranslateUnsupported(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "unsupported signaltometrics connector config") } + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsqlMysql) + assert.Equal(t, "signaltometrics/dbi_topsql_mysql", tr.ID().String()) +} + +func TestTranslateTopsqlMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsqlMysql) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + stmCfg := cfg.(*signaltometricsconfig.Config) + assert.Len(t, stmCfg.Logs, 2) + assert.Equal(t, "mysql.count_star", stmCfg.Logs[0].Name) + assert.Equal(t, "mysql.sum_timer_wait", stmCfg.Logs[1].Name) +} diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml new file mode 100644 index 0000000000..6c528ef72e --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml @@ -0,0 +1,10 @@ +error_mode: ignore +metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go index b354979ab2..a613f7f67f 100644 --- a/translator/translate/otel/processor/transformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -29,6 +29,9 @@ var transformEfaConfig string //go:embed transform_dbi_fix_start_time.yaml var transformDbiFixStartTimeConfig string +//go:embed transform_dbi_fix_start_time_mysql.yaml +var transformDbiFixStartTimeMysqlConfig string + //go:embed transform_identity_host.yaml var transformIdentityHostConfig string @@ -134,8 +137,11 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.name, common.PipelineNameHostDeltaMetrics) { return common.GetYamlFileToYamlConfig(cfg, transformEfaConfig) } - if t.name == common.DbiTransformFixStartTime { + switch t.name { + case common.DbiTransformFixStartTime: return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + case common.DbiTransformFixStartTimeMysql: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeMysqlConfig) } if t.name == common.Identity { if context.CurrentContext().KubernetesMode() != "" { diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index ffbb4adaf1..6f67f89aec 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -89,6 +89,21 @@ func TestDbiFixStartTimeTranslate(t *testing.T) { assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[6]) } +func TestDbiFixStartTimeMysqlTranslate(t *testing.T) { + transl := NewTranslatorWithName(common.DbiTransformFixStartTimeMysql) + assert.Equal(t, "transform/dbi_fix_start_time_mysql", transl.ID().String()) + + cfg, err := transl.Translate(nil) + require.NoError(t, err) + actualCfg := cfg.(*transformprocessor.Config) + require.Len(t, actualCfg.MetricStatements, 1) + assert.Equal(t, "datapoint", string(actualCfg.MetricStatements[0].Context)) + require.Len(t, actualCfg.MetricStatements[0].Statements, 5) + assert.Equal(t, "set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0", actualCfg.MetricStatements[0].Statements[0]) + assert.Equal(t, `replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU")`, actualCfg.MetricStatements[0].Statements[1]) + assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[4]) +} + func TestDbiResourceTranslate(t *testing.T) { stmts := []string{ `set(resource.attributes["db.system.name"], "postgresql")`, From 137120beabc470142b756b494359ce967d0417b0 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 16 Jun 2026 13:56:43 +0000 Subject: [PATCH 03/28] Add MySQL config schema definition Add mysql[] array schema to database_insights in schema.json: - endpoint, instance_name, username, password_file - tls.ca_file for TLS configuration - logs.file_path for server log ingestion --- translator/config/schema.json | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/translator/config/schema.json b/translator/config/schema.json index 3d52e38f14..1a818e49ff 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -1830,6 +1830,34 @@ "required": ["endpoint", "instance_name", "username", "password_file"], "additionalProperties": false } + }, + "mysql": { + "type": "array", + "items": { + "type": "object", + "properties": { + "endpoint": { "type": "string" }, + "instance_name": { "type": "string" }, + "username": { "type": "string" }, + "password_file": { "type": "string" }, + "tls": { + "type": "object", + "properties": { + "ca_file": { "type": "string" } + }, + "additionalProperties": false + }, + "logs": { + "type": "object", + "properties": { + "file_path": { "type": "string" } + }, + "additionalProperties": false + } + }, + "required": ["endpoint", "instance_name", "username", "password_file"], + "additionalProperties": false + } } }, "additionalProperties": false From 6b56a34a3386815f0fd9a800e6a859db6f52deaf Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 16 Jun 2026 13:57:06 +0000 Subject: [PATCH 04/28] Add MySQL pipeline translator and orchestrator wiring Extend the DBI pipeline orchestrator to support MySQL instances: - Add translator_mysql.go with dbiMysqlTranslator implementing 4 pipeline types: - metrics/dbi_mysql_N: counter metrics + DBLoad/TopSQL from connectors - logs/dbi_mysql_N: log-to-metrics via count + signaltometrics connectors - logs/dbi_mysql_rawevents_N: raw query sample/top query events - logs/dbi_mysql_serverlogs_N: MySQL server log file ingestion - Add parseDbiMysqlInstances() to parse mysql[] config array - MySQL-specific resource attributes (db.system.name = "mysql") - MySQL-specific log group paths and monitor user exclusion filter - Unit tests for pipeline wiring with single/multiple instances --- translator/translate/otel/common/common.go | 1 + .../databaseinsights/translator_mysql.go | 165 ++++++++++++++++++ .../databaseinsights/translator_test.go | 90 ++++++++++ .../databaseinsights/translators.go | 54 +++++- .../databaseinsights/translators_test.go | 78 +++++++++ .../opentelemetry/hostmetrics/translator.go | 9 +- .../hostmetrics/translator_test.go | 48 ++++- 7 files changed, 436 insertions(+), 9 deletions(-) create mode 100644 translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 12cec15406..882a8cb917 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -165,6 +165,7 @@ const ( var ( DatabaseInsightsConfigKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey) DatabaseInsightsPostgresKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, PostgreSQLKey) + DatabaseInsightsMysqlKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, MySQLKey) ) const ( diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go new file mode 100644 index 0000000000..ac91059775 --- /dev/null +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go @@ -0,0 +1,165 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package databaseinsights + +import ( + "fmt" + "strconv" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/pipeline" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/count" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/forward" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/signaltometrics" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/filelog" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/mysql" +) + +type dbiMysqlTranslator struct { + pipelineType dbiPipelineType + instanceIndex int + cfg dbiInstanceConfig +} + +var _ common.PipelineTranslator = (*dbiMysqlTranslator)(nil) + +func (t *dbiMysqlTranslator) ID() pipeline.ID { + idx := strconv.Itoa(t.instanceIndex) + switch t.pipelineType { + case dbiMetrics: + return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_mysql_"+idx) + case dbiLogToMetrics: + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_"+idx) + case dbiRawEvents: + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_rawevents_"+idx) + case dbiServerLogs: + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_serverlogs_"+idx) + } + return pipeline.NewID(pipeline.SignalMetrics) +} + +func (t *dbiMysqlTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators, error) { + if err := validateOttlSafe("username", t.cfg.username); err != nil { + return nil, err + } + if err := validateOttlSafe("instance_name", t.cfg.instanceName); err != nil { + return nil, err + } + switch t.pipelineType { + case dbiMetrics: + return t.translateMetrics() + case dbiLogToMetrics: + return t.translateLogToMetrics() + case dbiRawEvents: + return t.translateRawEvents() + case dbiServerLogs: + return t.translateServerLogs() + } + return nil, fmt.Errorf("unknown DBI MySQL pipeline type: %d", t.pipelineType) +} + +func (t *dbiMysqlTranslator) translateMetrics() (*common.ComponentTranslators, error) { + idx := strconv.Itoa(t.instanceIndex) + fwd := forward.NewTranslator(common.OpenTelemetryKey) + countConn := count.NewTranslator(common.DbiConnectorDbloadMysql) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsqlMysql) + + return &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("metrics"), countConn, s2mConn), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTimeMysql)), + Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), + Extensions: common.NewTranslatorMap[component.Config, component.ID](), + Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd, countConn, s2mConn), + }, nil +} + +func (t *dbiMysqlTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { + countConn := count.NewTranslator(common.DbiConnectorDbloadMysql) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsqlMysql) + + return &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("metrics")), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter()), + Exporters: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), + Extensions: common.NewTranslatorMap[component.Config, component.ID](), + Connectors: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), + }, nil +} + +func (t *dbiMysqlTranslator) translateRawEvents() (*common.ComponentTranslators, error) { + idx := strconv.Itoa(t.instanceIndex) + fwd := forward.NewTranslator(common.OpenTelemetryKey) + + return &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("events")), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_mysql_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), + Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), + Extensions: common.NewTranslatorMap[component.Config, component.ID](), + Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), + }, nil +} + +func (t *dbiMysqlTranslator) translateServerLogs() (*common.ComponentTranslators, error) { + idx := strconv.Itoa(t.instanceIndex) + fwd := forward.NewTranslator(common.OpenTelemetryKey) + + return &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix("mysql"), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_mysql_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), + Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), + Extensions: common.NewTranslatorMap[component.Config, component.ID](), + Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), + }, nil +} + +func (t *dbiMysqlTranslator) mysqlReceiver(name string, extraOpts ...mysql.Option) common.ComponentTranslator { + opts := []mysql.Option{ + mysql.WithName(name), + mysql.WithIndex(t.instanceIndex), + mysql.WithEndpoint(t.cfg.endpoint), + mysql.WithUsername(t.cfg.username), + mysql.WithPassfile(t.cfg.passfile), + mysql.WithCAFile(t.cfg.caFile), + mysql.WithIsLocalhost(t.cfg.isLocalhost), + } + opts = append(opts, extraOpts...) + return mysql.NewTranslator(opts...) +} + +func (t *dbiMysqlTranslator) excludeMonitorFilter() common.ComponentTranslator { + idx := strconv.Itoa(t.instanceIndex) + condition := fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_mysql_"+idx, condition) +} + +func (t *dbiMysqlTranslator) scopeTransform() common.ComponentTranslator { + idx := strconv.Itoa(t.instanceIndex) + return transformprocessor.NewTranslatorWithName("dbi_scope_mysql_"+idx, + transformprocessor.WithErrorMode("ignore"), + transformprocessor.WithScopeStatements([]string{ + `set(attributes["cloudwatch.source"], "cloudwatch-agent")`, + `set(attributes["cloudwatch.solution"], "otel-database-insights")`, + }), + ) +} + +func (t *dbiMysqlTranslator) resourceStatements() []string { + return []string{ + `set(resource.attributes["db.system.name"], "mysql")`, + fmt.Sprintf(`set(resource.attributes["db.instance.name"], "%s")`, t.cfg.instanceName), + } +} + +func (t *dbiMysqlTranslator) logStatements(destination string) []string { + return []string{ + fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/%s")`, destination), + fmt.Sprintf(`set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "%s"], "/"))`, t.cfg.instanceName), + } +} diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index 04d913311f..4090105ff6 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -103,3 +103,93 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, connectors) } + +func TestDbiMysqlTranslatorID(t *testing.T) { + tests := []struct { + pipelineType dbiPipelineType + index int + want string + }{ + {dbiMetrics, 0, "metrics/dbi_mysql_0"}, + {dbiMetrics, 1, "metrics/dbi_mysql_1"}, + {dbiLogToMetrics, 0, "logs/dbi_mysql_0"}, + {dbiRawEvents, 0, "logs/dbi_mysql_rawevents_0"}, + {dbiServerLogs, 0, "logs/dbi_mysql_serverlogs_0"}, + {dbiServerLogs, 2, "logs/dbi_mysql_serverlogs_2"}, + } + for _, tc := range tests { + t.Run(tc.want, func(t *testing.T) { + tr := &dbiMysqlTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index} + assert.Equal(t, tc.want, tr.ID().String()) + }) + } +} + +func TestDbiMysqlTranslate(t *testing.T) { + cfg := dbiInstanceConfig{ + endpoint: "localhost:3306", + username: "cw_monitor", + passfile: "/etc/.mysql_credentials", + instanceName: "my-db", + logFilePath: "/var/log/mysql/mysql.log", + isLocalhost: true, + } + + tests := []struct { + name string + pipeline dbiPipelineType + expectedID string + nRecv int + nProc int + nExp int + nConn int + }{ + {"metrics", dbiMetrics, "metrics/dbi_mysql_0", 3, 3, 1, 3}, + {"log_to_metrics", dbiLogToMetrics, "logs/dbi_mysql_0", 1, 1, 2, 2}, + {"raw_events", dbiRawEvents, "logs/dbi_mysql_rawevents_0", 1, 5, 1, 1}, + {"server_logs", dbiServerLogs, "logs/dbi_mysql_serverlogs_0", 1, 4, 1, 1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tr := &dbiMysqlTranslator{pipelineType: tc.pipeline, instanceIndex: 0, cfg: cfg} + assert.Equal(t, tc.expectedID, tr.ID().String()) + + result, err := tr.Translate(nil) + require.NoError(t, err) + assert.Equal(t, tc.nRecv, result.Receivers.Len()) + assert.Equal(t, tc.nProc, result.Processors.Len()) + assert.Equal(t, tc.nExp, result.Exporters.Len()) + assert.Equal(t, tc.nConn, result.Connectors.Len()) + }) + } +} + +func TestDbiMysqlTranslateMetrics_ComponentIDs(t *testing.T) { + tr := &dbiMysqlTranslator{ + pipelineType: dbiMetrics, + instanceIndex: 0, + cfg: dbiInstanceConfig{instanceName: "mydb"}, + } + result, err := tr.Translate(nil) + require.NoError(t, err) + + var receivers, processors, exporters, connectors []string + result.Receivers.Range(func(c common.Translator[component.Config, component.ID]) { + receivers = append(receivers, c.ID().String()) + }) + result.Processors.Range(func(c common.Translator[component.Config, component.ID]) { + processors = append(processors, c.ID().String()) + }) + result.Exporters.Range(func(c common.Translator[component.Config, component.ID]) { + exporters = append(exporters, c.ID().String()) + }) + result.Connectors.Range(func(c common.Translator[component.Config, component.ID]) { + connectors = append(connectors, c.ID().String()) + }) + + assert.ElementsMatch(t, []string{"mysql/metrics_0", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_mysql_0", "transform/dbi_resource_mysql_0", "transform/dbi_fix_start_time_mysql"}, processors) + assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, connectors) +} diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index bed6e03582..a24cc9a3fd 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -19,6 +19,10 @@ import ( // characters, preventing injection of OTTL syntax or string escape sequences. var ottlSafeRegex = regexp.MustCompile(`^[a-zA-Z0-9._@/:\-]+$`) +var ( + dbiMysqlKey = common.ConfigKey(common.OpenTelemetryKey, common.CollectKey, common.DatabaseInsightsKey, common.MySQLKey) +) + type dbiInstanceConfig struct { endpoint string username string @@ -35,8 +39,8 @@ func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap { return translators } - instances := parseDbiPostgresqlInstances(conf) - for i, cfg := range instances { + pgInstances := parseDbiPostgresqlInstances(conf) + for i, cfg := range pgInstances { translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) @@ -44,6 +48,17 @@ func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap { translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) } } + + mysqlInstances := parseDbiMysqlInstances(conf) + for i, cfg := range mysqlInstances { + translators.Set(&dbiMysqlTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiMysqlTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiMysqlTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) + if cfg.logFilePath != "" { + translators.Set(&dbiMysqlTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + } + } + return translators } @@ -89,8 +104,43 @@ func validateOttlSafe(field, value string) error { return nil } +type mysqlRawInstance struct { + Endpoint string `mapstructure:"endpoint"` + Username string `mapstructure:"username"` + PasswordFile string `mapstructure:"password_file"` + InstanceName string `mapstructure:"instance_name"` + TLS struct { + CAFile string `mapstructure:"ca_file"` + } `mapstructure:"tls"` + Logs struct { + FilePath string `mapstructure:"file_path"` + } `mapstructure:"logs"` +} + +func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { + arr, _ := conf.Get(dbiMysqlKey).([]any) + var raw []mysqlRawInstance + if err := mapstructure.Decode(arr, &raw); err != nil { + return nil + } + instances := make([]dbiInstanceConfig, 0, len(raw)) + for _, r := range raw { + instances = append(instances, dbiInstanceConfig{ + endpoint: r.Endpoint, + username: r.Username, + passfile: r.PasswordFile, + caFile: r.TLS.CAFile, + instanceName: r.InstanceName, + logFilePath: r.Logs.FilePath, + isLocalhost: isLocalhostEndpoint(r.Endpoint), + }) + } + return instances +} + func isLocalhostEndpoint(endpoint string) bool { return strings.HasPrefix(endpoint, "localhost") || strings.HasPrefix(endpoint, "127.0.0.1") || + strings.HasPrefix(endpoint, "[::1]") || strings.HasPrefix(endpoint, "::1") } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go index 0595f19c3d..92a3488313 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go @@ -108,6 +108,84 @@ func TestNewTranslators_MultiInstance(t *testing.T) { func TestIsLocalhostEndpoint(t *testing.T) { assert.True(t, isLocalhostEndpoint("localhost:5432")) assert.True(t, isLocalhostEndpoint("127.0.0.1:5432")) + assert.True(t, isLocalhostEndpoint("[::1]:3306")) + assert.True(t, isLocalhostEndpoint("::1:3306")) assert.False(t, isLocalhostEndpoint("db.example.com:5432")) assert.False(t, isLocalhostEndpoint("")) } + +func TestNewTranslators_MySQL_WithLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql.log"}, + }, + }, + }, + }, + }, + }) + assert.Equal(t, 4, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_NoLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + }, + }, + }, + }, + }, + }) + assert.Equal(t, 3, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_MultiInstance(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "db1", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql1.log"}, + }, + map[string]interface{}{ + "endpoint": "db2.example.com:3306", "username": "monitor", + "password_file": "/etc/.mysql_credentials2", "instance_name": "db2", + }, + }, + }, + }, + }, + }) + translators := NewTranslators(cfg) + assert.Equal(t, 7, translators.Len()) // 4 + 3 + + var ids []string + for _, k := range translators.Keys() { + ids = append(ids, k.String()) + } + expected := []string{ + "metrics/dbi_mysql_0", + "logs/dbi_mysql_0", + "logs/dbi_mysql_rawevents_0", + "logs/dbi_mysql_serverlogs_0", + "metrics/dbi_mysql_1", + "logs/dbi_mysql_1", + "logs/dbi_mysql_rawevents_1", + } + assert.ElementsMatch(t, expected, ids) +} diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go index 272cf00d23..9862de4720 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go @@ -38,10 +38,17 @@ func (t *hostMetricsTranslator) Translate(conf *confmap.Conf) (*common.Component var opts []hostmetrics.Option opts = append(opts, hostmetrics.WithName(common.OpenTelemetryKey)) if conf.IsSet(common.DatabaseInsightsConfigKey) { + var processNames []string + if conf.IsSet(common.DatabaseInsightsPostgresKey) { + processNames = append(processNames, "post(gres|master).*") + } + if conf.IsSet(common.DatabaseInsightsMysqlKey) { + processNames = append(processNames, "mysqld.*") + } opts = append(opts, hostmetrics.WithProcessScraper(map[string]any{ "include": map[string]any{ "match_type": "regexp", - "names": []string{"post(gres|master).*"}, + "names": processNames, }, "mute_process_all_errors": true, "metrics": map[string]any{ diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go index 4783823d18..f0f23a2a16 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go @@ -23,9 +23,10 @@ func TestHostMetricsTranslator(t *testing.T) { assert.EqualValues(t, "metrics/host_metrics", tt.ID().String()) testCases := map[string]struct { - input map[string]interface{} - wantErr error - expectProcess bool + input map[string]interface{} + wantErr error + expectProcess bool + wantProcessNames []string }{ "WithNilConf": { input: nil, @@ -56,7 +57,41 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, + }, + "WithMysqlDatabaseInsightsKey": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"mysqld.*"}, + }, + "WithBothEngines": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:5432", "username": "cwagent"}, + }, + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*", "mysqld.*"}, }, "WithBothKeys": { input: map[string]interface{}{ @@ -71,7 +106,8 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, }, } for name, tc := range testCases { @@ -109,7 +145,7 @@ func TestHostMetricsTranslator(t *testing.T) { assert.True(t, exists, "expected process scraper") include := processCfg["include"].(map[string]any) assert.Equal(t, "regexp", include["match_type"]) - assert.Equal(t, []string{"postgres.*"}, include["names"]) + assert.Equal(t, tc.wantProcessNames, include["names"]) } } }) From d7e59ded79db687ba884c3aec5caae911bfb3069 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 16 Jun 2026 13:57:22 +0000 Subject: [PATCH 05/28] Add MySQL golden YAML test fixtures Add end-to-end config translation test for MySQL DBI: - dbi_mysql_config_linux.conf: input agent config fixture - dbi_mysql_config_linux.json: parsed JSON intermediate fixture - dbi_mysql_config_linux.yaml: golden YAML output (full OTel pipeline) - .mysql_credentials: test passfile for credential loading - Register TestMySQLDbiConfig in tocwconfig_unix_test.go --- .../opentelemetry/dbi_mysql_config_linux.conf | 15 + .../opentelemetry/dbi_mysql_config_linux.json | 22 + .../opentelemetry/dbi_mysql_config_linux.yaml | 1609 +++++++++++++++++ .../opentelemetry/testdata/.mysql_credentials | 1 + translator/tocwconfig/tocwconfig_unix_test.go | 8 + 5 files changed, 1655 insertions(+) create mode 100644 translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf create mode 100644 translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json create mode 100644 translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml create mode 100644 translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf new file mode 100644 index 0000000000..9d2168ce61 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf @@ -0,0 +1,15 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json new file mode 100644 index 0000000000..5aca9f4f90 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json @@ -0,0 +1,22 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "collect": { + "database_insights": { + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "pentest-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/mysql.log" + } + } + ] + } + } + } +} diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml new file mode 100644 index 0000000000..8aa191b9dd --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -0,0 +1,1609 @@ +connectors: + count/dbi_dbload_mysql: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. + forward/opentelemetry: {} + signaltometrics/dbi_topsql_mysql: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + - key: user.name + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" +exporters: + otlphttp/logs: + auth: + authenticator: agenthealth/opentelemetry_logs + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: https://logs.us-west-2.amazonaws.com/v1/logs + max_idle_conns: 100 + metrics_endpoint: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 + otlphttp/metrics: + auth: + authenticator: agenthealth/opentelemetry_metrics + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: "" + max_idle_conns: 100 + metrics_endpoint: https://monitoring.us-west-2.amazonaws.com/v1/metrics + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 +extensions: + agenthealth/opentelemetry_logs: + additional_auth: headers_setter/logs + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + agenthealth/opentelemetry_metrics: + additional_auth: sigv4auth/monitoring + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + awscloudwatchlogsprovisioner: + additional_auth: sigv4auth/logs + imds_retries: 1 + logs_provision_failure_backoff: 30s + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 10 + headers_setter/logs: + additional_auth: awscloudwatchlogsprovisioner + headers: + - action: upsert + from_context: aws.log.group.name + key: x-aws-log-group + - action: upsert + from_context: aws.log.stream.name + key: x-aws-log-stream + sigv4auth/logs: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: logs + sigv4auth/monitoring: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: monitoring +processors: + attributestocontext/opentelemetry: + actions: + - from_resource_attribute: aws.log.group.name + key: aws.log.group.name + - from_resource_attribute: aws.log.stream.name + key: aws.log.stream.name + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s + batch/opentelemetry_metrics: + metadata_cardinality_limit: 1000 + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 15s + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} + resourcedetection/opentelemetry: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + tags: [] + consul: + address: "" + datacenter: "" + namespace: "" + resource_attributes: + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + fail_on_missing_metadata: false + max_attempts: 3 + max_backoff: 20s + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + - ^aws:autoscaling:groupName + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.id: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.account.id: + enabled: false + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.instance: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + gcp.gce.instance_group_manager.name: + enabled: true + gcp.gce.instance_group_manager.region: + enabled: true + gcp.gce.instance_group_manager.zone: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + k8snode: + auth_type: serviceAccount + context: "" + kube_config_path: "" + node_from_env_var: "" + resource_attributes: + k8s.node.name: + enabled: true + k8s.node.uid: + enabled: true + kubeadm: + auth_type: serviceAccount + context: "" + kube_config_path: "" + resource_attributes: + k8s.cluster.name: + enabled: true + k8s.cluster.uid: + enabled: true + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_idle_conns: 100 + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: {} + token: "" + override: true + system: + resource_attributes: + host.arch: + enabled: false + host.cpu.cache.l2.size: + enabled: false + host.cpu.family: + enabled: false + host.cpu.model.id: + enabled: false + host.cpu.model.name: + enabled: false + host.cpu.stepping: + enabled: false + host.cpu.vendor.id: + enabled: false + host.id: + enabled: false + host.ip: + enabled: false + host.mac: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + os.version: + enabled: false + timeout: 2s + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] + transform/dbi_logs_mysql_raw-events_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_resource_mysql_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "pentest-mysql") + metric_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "pentest-mysql") + trace_statements: [] + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + transform/host_metrics_scope: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + trace_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + transform/identity: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + trace_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + transform/logs_cleanup: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - delete_key(resource.attributes, "aws.log.group.name") + - delete_key(resource.attributes, "aws.log.stream.name") + - delete_key(resource.attributes, "aws.log.source") + metric_statements: [] + trace_statements: [] + transform/logs_routing: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], Concat(["/aws/cwagent", resource.attributes["aws.log.source"]], "/")) where resource.attributes["aws.log.group.name"] == nil and resource.attributes["aws.log.source"] != nil + - set(resource.attributes["aws.log.group.name"], "/aws/cwagent/default") where resource.attributes["aws.log.group.name"] == nil + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.id"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.name"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil + - set(resource.attributes["aws.log.stream.name"], "default") where resource.attributes["aws.log.stream.name"] == nil + metric_statements: [] + trace_statements: [] +receivers: + filelog/mysql_0: + encoding: utf-8 + exclude_older_than: 0s + fingerprint_size: 1000 + force_flush_period: 500ms + id: file_input + include: + - /var/log/mysql/mysql.log + include_file_name: true + initial_buffer_size: 16384 + max_concurrent_files: 1024 + max_log_size: 1048576 + operators: [] + poll_interval: 200ms + retry_on_failure: + enabled: false + initial_interval: 1s + max_elapsed_time: 5m0s + max_interval: 30s + start_at: end + type: file_input + hostmetrics/opentelemetry: + collection_interval: 30s + initial_delay: 1s + scrapers: + cpu: + metrics: + system.cpu.frequency: + enabled: true + system.cpu.logical.count: + enabled: true + system.cpu.physical.count: + enabled: true + system.cpu.utilization: + enabled: true + disk: null + filesystem: + metrics: + system.filesystem.utilization: + enabled: true + load: null + memory: + metrics: + system.linux.memory.available: + enabled: true + system.linux.memory.dirty: + enabled: true + system.memory.limit: + enabled: true + system.memory.page_size: + enabled: true + system.memory.utilization: + enabled: true + network: null + process: + include: + match_type: regexp + names: + - mysqld.* + metrics: + process.cpu.utilization: + enabled: true + process.memory.utilization: + enabled: true + mute_process_all_errors: true + processes: null + timeout: 0s + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: false + mysql.connection.count: + enabled: false + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: false + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: false + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: false + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: false + mysql.replica.time_behind_source: + enabled: false + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: false + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: false + mysql.connection.count: + enabled: false + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: false + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: false + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: false + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: false + mysql.replica.time_behind_source: + enabled: false + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: false + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor +service: + extensions: + - sigv4auth/monitoring + - agenthealth/opentelemetry_metrics + - sigv4auth/logs + - awscloudwatchlogsprovisioner + - headers_setter/logs + - agenthealth/opentelemetry_logs + pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 + logs/opentelemetry: + exporters: + - otlphttp/logs + processors: + - resourcedetection/opentelemetry + - transform/identity + - transform/logs_routing + - attributestocontext/opentelemetry + - transform/logs_cleanup + - batch/opentelemetry_logs + receivers: + - forward/opentelemetry + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + metrics/host_metrics: + exporters: + - forward/opentelemetry + processors: + - transform/host_metrics_scope + receivers: + - hostmetrics/opentelemetry + metrics/opentelemetry: + exporters: + - otlphttp/metrics + processors: + - resourcedetection/opentelemetry + - transform/identity + - batch/opentelemetry_metrics + receivers: + - forward/opentelemetry + telemetry: + logs: + encoding: console + level: info + output_paths: + - /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + level: None + traces: + level: None diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials new file mode 100644 index 0000000000..7627976dfa --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials @@ -0,0 +1 @@ +testpass123 diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index ee5d7022c3..906733142b 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -183,3 +183,11 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { }) assert.Empty(t, cmp.Diff(expected, actual, opt)) } + +func TestDBIMySQLConfigLinux(t *testing.T) { + resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) + expectedEnvVars := map[string]string{} + checkTranslation(t, "opentelemetry/dbi_mysql_config_linux", "linux", expectedEnvVars, "") +} From b77324e2718ee4eea0f80f0bc2bd0316e631d9b8 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 30 Jun 2026 14:07:39 +0000 Subject: [PATCH 06/28] Address PR review comments for MySQL DBI - Unify the DBI pipeline translator: merge translator_mysql.go into a single dbiTranslator that branches by engine (carried on the instance config), instead of maintaining a separate MySQL translator. - Make component naming consistent across engines using an engine suffix: count/dbi_dbload_, signaltometrics/dbi_topsql_, transform/dbi_fix_start_time_, and per-instance dbi_resource__N, dbi_scope__N, dbi_logs___N, dbi_exclude_monitor__N. PostgreSQL components now carry the _postgresql suffix for consistency. - Rename engine-specific connector/processor YAMLs for symmetry (dbi_dbload_postgresql.yaml, dbi_topsql_postgresql.yaml, transform_dbi_fix_start_time_postgresql.yaml); the connector and transform translators select the config by engine. - Add the dbi_scope transform processor (cloudwatch.source/solution) to all MySQL pipelines, matching the PostgreSQL implementation. - Use the shared named resourcedetection/opentelemetry processor. - Add the validateOttlSafe check to the unified Translate() so username and instance_name are validated for both engines. - Fix the host metrics process scraper to match the mysqld process (mysqld.*) when MySQL DBI is configured. - Remove TLS/ca_file from the MySQL schema and receiver; MySQL DBI is localhost-only and always connects with TLS disabled. - Fix the MySQL passfile test fixture to the pgpass-style format (host:port:database:username:password) and assert all seven DB Load metrics. - Regenerate golden YAMLs (mysql, postgres, combined configs). --- translator/config/schema.json | 7 - .../combined_v1_v2_ec2_config.yaml | 70 ++++---- .../combined_v1_v2_eks_config.yaml | 68 ++++---- ....conf => dbi_postgresql_config_linux.conf} | 0 ....json => dbi_postgresql_config_linux.json} | 0 ....yaml => dbi_postgresql_config_linux.yaml} | 56 +++--- .../opentelemetry/testdata/.mysql_credentials | 2 +- translator/tocwconfig/tocwconfig_unix_test.go | 2 +- translator/translate/otel/common/common.go | 7 +- ...dbload.yaml => dbi_dbload_postgresql.yaml} | 0 .../otel/connector/count/translator.go | 17 +- .../otel/connector/count/translator_test.go | 19 +- ...topsql.yaml => dbi_topsql_postgresql.yaml} | 0 .../connector/signaltometrics/translator.go | 17 +- .../signaltometrics/translator_test.go | 14 +- .../databaseinsights/translator.go | 66 ++++--- .../databaseinsights/translator_mysql.go | 165 ------------------ .../databaseinsights/translator_test.go | 56 +++--- .../databaseinsights/translators.go | 37 ++-- ...nsform_dbi_fix_start_time_postgresql.yaml} | 0 .../processor/transformprocessor/translate.go | 37 ++-- .../transformprocessor/translator_test.go | 6 +- .../otel/receiver/mysql/translator.go | 18 +- .../otel/receiver/mysql/translator_test.go | 25 --- 24 files changed, 252 insertions(+), 437 deletions(-) rename translator/tocwconfig/sampleConfig/opentelemetry/{dbi_config_linux.conf => dbi_postgresql_config_linux.conf} (100%) rename translator/tocwconfig/sampleConfig/opentelemetry/{dbi_config_linux.json => dbi_postgresql_config_linux.json} (100%) rename translator/tocwconfig/sampleConfig/opentelemetry/{dbi_config_linux.yaml => dbi_postgresql_config_linux.yaml} (97%) rename translator/translate/otel/connector/count/{dbi_dbload.yaml => dbi_dbload_postgresql.yaml} (100%) rename translator/translate/otel/connector/signaltometrics/{dbi_topsql.yaml => dbi_topsql_postgresql.yaml} (100%) delete mode 100644 translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go rename translator/translate/otel/processor/transformprocessor/{transform_dbi_fix_start_time.yaml => transform_dbi_fix_start_time_postgresql.yaml} (100%) diff --git a/translator/config/schema.json b/translator/config/schema.json index 1a818e49ff..200130c791 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -1840,13 +1840,6 @@ "instance_name": { "type": "string" }, "username": { "type": "string" }, "password_file": { "type": "string" }, - "tls": { - "type": "object", - "properties": { - "ca_file": { "type": "string" } - }, - "additionalProperties": false - }, "logs": { "type": "object", "properties": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index 7be83ef10a..93032d2bce 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -504,6 +504,14 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 1m0s + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s batch/opentelemetry_metrics: metadata_cardinality_limit: 1000 send_batch_max_size: 1000 @@ -514,14 +522,6 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - batch/opentelemetry_logs: - metadata_cardinality_limit: 1000 - metadata_keys: - - aws.log.group.name - - aws.log.stream.name - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 15s batch/xray: metadata_cardinality_limit: 1000 send_batch_max_size: 0 @@ -549,7 +549,7 @@ processors: middleware: agenthealth/statuscode refresh_tags_interval: 0s refresh_volumes_interval: 0s - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -837,7 +837,7 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -853,7 +853,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -864,7 +864,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -875,7 +875,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -891,7 +891,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1505,31 +1505,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -1564,13 +1564,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host: exporters: - awscloudwatch @@ -1578,13 +1578,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_procstat/1917393364 - telegraf_mem - - telegraf_netstat + - telegraf_procstat/1917393364 + - telegraf_processes - telegraf_cpu - telegraf_disk - telegraf_swap - - telegraf_processes + - telegraf_netstat metrics/host_metrics: exporters: - forward/opentelemetry @@ -1609,8 +1609,8 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_net - telegraf_diskio + - telegraf_net metrics/opentelemetry: exporters: - otlphttp/metrics diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 37b965b30b..352c1b4f31 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -551,6 +551,14 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 1m0s + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s batch/opentelemetry_metrics: metadata_cardinality_limit: 1000 send_batch_max_size: 1000 @@ -561,14 +569,6 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - batch/opentelemetry_logs: - metadata_cardinality_limit: 1000 - metadata_keys: - - aws.log.group.name - - aws.log.stream.name - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 15s batch/xray: metadata_cardinality_limit: 1000 send_batch_max_size: 0 @@ -624,7 +624,7 @@ processors: metrics: metric: - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -1370,7 +1370,7 @@ processors: - set(resource.attributes["k8s.workload.type"], "CronJob") where resource.attributes["k8s.workload.type"] == nil and resource.attributes["k8s.cronjob.name"] != nil - set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil - set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.workload.type"] == nil and resource.attributes["k8s.replicaset.name"] != nil - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -1386,7 +1386,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -1397,7 +1397,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -1408,7 +1408,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -1424,7 +1424,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -2347,31 +2347,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -2617,13 +2617,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host: exporters: - awscloudwatch @@ -2631,13 +2631,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_procstat/1917393364 - - telegraf_cpu - telegraf_swap + - telegraf_processes - telegraf_mem + - telegraf_procstat/1917393364 - telegraf_netstat - - telegraf_processes - telegraf_disk + - telegraf_cpu metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml similarity index 97% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml index 33db4312e1..a236a01e61 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -339,11 +339,6 @@ processors: key: aws.log.group.name - from_resource_attribute: aws.log.stream.name key: aws.log.stream.name - batch/opentelemetry_metrics: - metadata_cardinality_limit: 1000 - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 15s batch/opentelemetry_logs: metadata_cardinality_limit: 1000 metadata_keys: @@ -352,7 +347,12 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - filter/dbi_exclude_monitor_0: + batch/opentelemetry_metrics: + metadata_cardinality_limit: 1000 + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 15s + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -640,7 +640,7 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -656,7 +656,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -667,7 +667,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -678,7 +678,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -694,7 +694,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "pentest-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1098,31 +1098,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/opentelemetry: @@ -1141,13 +1141,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials index 7627976dfa..063172194e 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials +++ b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials @@ -1 +1 @@ -testpass123 +localhost:3306:*:cw_monitor:test_password diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index 906733142b..5b3a292482 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -104,7 +104,7 @@ func TestDBIConfigLinux(t *testing.T) { context.CurrentContext().SetMode(config.ModeEC2) require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.pgpass", 0600)) expectedEnvVars := map[string]string{} - checkTranslation(t, "opentelemetry/dbi_config_linux", "linux", expectedEnvVars, "") + checkTranslation(t, "opentelemetry/dbi_postgresql_config_linux", "linux", expectedEnvVars, "") } func TestJournaldLogsUnits(t *testing.T) { diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 882a8cb917..541844c1c9 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -142,18 +142,13 @@ const ( PostgreSQLKey = "postgresql" MySQLKey = "mysql" - // DBI component names — PostgreSQL + // DBI component name prefixes (shared across engines; index-suffixed per instance) DbiTransformFixStartTime = "dbi_fix_start_time" DbiTransformResource = "dbi_resource" DbiTransformLogs = "dbi_logs" DbiFilterExcludeMonitor = "dbi_exclude_monitor" DbiConnectorDbload = "dbi_dbload" DbiConnectorTopsql = "dbi_topsql" - - // DBI component names — MySQL - DbiTransformFixStartTimeMysql = "dbi_fix_start_time_mysql" - DbiConnectorDbloadMysql = "dbi_dbload_mysql" - DbiConnectorTopsqlMysql = "dbi_topsql_mysql" ) // OpenTelemetry pipeline processor names diff --git a/translator/translate/otel/connector/count/dbi_dbload.yaml b/translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/count/dbi_dbload.yaml rename to translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml diff --git a/translator/translate/otel/connector/count/translator.go b/translator/translate/otel/connector/count/translator.go index c8f2921518..1ab0e54c0e 100644 --- a/translator/translate/otel/connector/count/translator.go +++ b/translator/translate/otel/connector/count/translator.go @@ -15,7 +15,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_dbload.yaml +//go:embed dbi_dbload_postgresql.yaml var dbiDbloadConfig string //go:embed dbi_dbload_mysql.yaml @@ -23,15 +23,18 @@ var dbiDbloadMysqlConfig string type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a count connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a count connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: countconnector.NewFactory(), } } @@ -43,12 +46,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*countconnector.Config) - switch t.name { - case common.DbiConnectorDbload: + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiDbloadConfig) - case common.DbiConnectorDbloadMysql: + case common.MySQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiDbloadMysqlConfig) } - return nil, fmt.Errorf("unsupported count connector config: %s", t.name) + return nil, fmt.Errorf("unsupported count connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/count/translator_test.go b/translator/translate/otel/connector/count/translator_test.go index c94fee1546..275af49dd6 100644 --- a/translator/translate/otel/connector/count/translator_test.go +++ b/translator/translate/otel/connector/count/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) - assert.Equal(t, "count/dbi_dbload", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "count/dbi_dbload_postgresql", tr.ID().String()) } func TestTranslateDbload(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,23 +36,28 @@ func TestTranslateDbload(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_dbload_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported count connector config") + assert.Contains(t, err.Error(), "unsupported count connector engine") } func TestTranslatorID_MySQL(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbloadMysql) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) assert.Equal(t, "count/dbi_dbload_mysql", tr.ID().String()) } func TestTranslateDbloadMysql(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbloadMysql) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) countCfg := cfg.(*countconnector.Config) assert.Len(t, countCfg.Logs, 7) assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_user") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_db") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_host") assert.Contains(t, countCfg.Logs, "mysql.active_sessions.count") } diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml rename to translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml diff --git a/translator/translate/otel/connector/signaltometrics/translator.go b/translator/translate/otel/connector/signaltometrics/translator.go index 4ca189ffa5..a26fbe4870 100644 --- a/translator/translate/otel/connector/signaltometrics/translator.go +++ b/translator/translate/otel/connector/signaltometrics/translator.go @@ -16,7 +16,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_topsql.yaml +//go:embed dbi_topsql_postgresql.yaml var dbiTopsqlConfig string //go:embed dbi_topsql_mysql.yaml @@ -24,15 +24,18 @@ var dbiTopsqlMysqlConfig string type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a signaltometrics connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a signaltometrics connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: signaltometricsconnector.NewFactory(), } } @@ -44,12 +47,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*signaltometricsconfig.Config) - switch t.name { - case common.DbiConnectorTopsql: + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlConfig) - case common.DbiConnectorTopsqlMysql: + case common.MySQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlMysqlConfig) } - return nil, fmt.Errorf("unsupported signaltometrics connector config: %s", t.name) + return nil, fmt.Errorf("unsupported signaltometrics connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index 17d94134d9..5a9c5d254d 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) - assert.Equal(t, "signaltometrics/dbi_topsql", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "signaltometrics/dbi_topsql_postgresql", tr.ID().String()) } func TestTranslateTopsql(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,19 +36,19 @@ func TestTranslateTopsql(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_topsql_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported signaltometrics connector config") + assert.Contains(t, err.Error(), "unsupported signaltometrics connector engine") } func TestTranslatorID_MySQL(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsqlMysql) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) assert.Equal(t, "signaltometrics/dbi_topsql_mysql", tr.ID().String()) } func TestTranslateTopsqlMysql(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsqlMysql) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) stmCfg := cfg.(*signaltometricsconfig.Config) diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index cfd4209532..f531ca9ca5 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -20,6 +20,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/filelog" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/mysql" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/postgresql" ) @@ -32,6 +33,10 @@ const ( dbiServerLogs ) +// dbiTranslator generates DBI pipelines for a single database instance. The +// engine (carried on cfg) selects engine-specific receivers, connector configs, +// resource attributes, and log group paths. Component IDs are index-based so +// PostgreSQL and MySQL instances share one consistent naming scheme. type dbiTranslator struct { pipelineType dbiPipelineType instanceIndex int @@ -44,13 +49,13 @@ func (t *dbiTranslator) ID() pipeline.ID { idx := strconv.Itoa(t.instanceIndex) switch t.pipelineType { case dbiMetrics: - return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_"+t.cfg.engine+"_"+idx) case dbiLogToMetrics: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_"+idx) case dbiRawEvents: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_rawevents_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_rawevents_"+idx) case dbiServerLogs: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_serverlogs_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_serverlogs_"+idx) } return pipeline.NewID(pipeline.SignalMetrics) } @@ -78,12 +83,12 @@ func (t *dbiTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) { idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("metrics"), countConn, s2mConn), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime)), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("metrics"), countConn, s2mConn), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+t.cfg.engine, transformprocessor.WithDbiFixStartTime(t.cfg.engine))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd, countConn, s2mConn), @@ -91,11 +96,11 @@ func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) } func (t *dbiTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("metrics")), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("metrics")), Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter()), Exporters: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), Extensions: common.NewTranslatorMap[component.Config, component.ID](), @@ -108,8 +113,8 @@ func (t *dbiTranslator) translateRawEvents() (*common.ComponentTranslators, erro fwd := forward.NewTranslator(common.OpenTelemetryKey) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("events", postgresql.WithQuerySampleInterval(60*time.Second))), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("events")), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_"+t.cfg.engine+"_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), @@ -121,15 +126,27 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err fwd := forward.NewTranslator(common.OpenTelemetryKey) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix("postgresql"), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), + Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix(t.cfg.engine), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_"+t.cfg.engine+"_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), }, nil } -func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) common.ComponentTranslator { +// receiver builds the engine-specific receiver translator. name is "metrics" or +// "events"; for PostgreSQL events we override the query sample interval. +func (t *dbiTranslator) receiver(name string) common.ComponentTranslator { + if t.cfg.engine == common.MySQLKey { + return mysql.NewTranslator( + mysql.WithName(name), + mysql.WithIndex(t.instanceIndex), + mysql.WithEndpoint(t.cfg.endpoint), + mysql.WithUsername(t.cfg.username), + mysql.WithPassfile(t.cfg.passfile), + ) + } + opts := []postgresql.Option{ postgresql.WithName(name), postgresql.WithIndex(t.instanceIndex), @@ -139,19 +156,24 @@ func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) postgresql.WithCAFile(t.cfg.caFile), postgresql.WithIsLocalhost(t.cfg.isLocalhost), } - opts = append(opts, extraOpts...) + if name == "events" { + opts = append(opts, postgresql.WithQuerySampleInterval(60*time.Second)) + } return postgresql.NewTranslator(opts...) } func (t *dbiTranslator) excludeMonitorFilter() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - condition := fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) - return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+idx, condition) + condition := fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + if t.cfg.engine == common.PostgreSQLKey { + condition = fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) + } + return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+t.cfg.engine+"_"+idx, condition) } func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - return transformprocessor.NewTranslatorWithName("dbi_scope_"+idx, + return transformprocessor.NewTranslatorWithName("dbi_scope_"+t.cfg.engine+"_"+idx, transformprocessor.WithErrorMode("ignore"), transformprocessor.WithScopeStatements([]string{ `set(attributes["cloudwatch.source"], "cloudwatch-agent")`, @@ -162,14 +184,14 @@ func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { func (t *dbiTranslator) resourceStatements() []string { return []string{ - `set(resource.attributes["db.system.name"], "postgresql")`, + fmt.Sprintf(`set(resource.attributes["db.system.name"], "%s")`, t.cfg.engine), fmt.Sprintf(`set(resource.attributes["db.instance.name"], "%s")`, t.cfg.instanceName), } } func (t *dbiTranslator) logStatements(destination string) []string { return []string{ - fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/postgresql/%s")`, destination), + fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/%s/%s")`, t.cfg.engine, destination), fmt.Sprintf(`set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "%s"], "/"))`, t.cfg.instanceName), } } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go deleted file mode 100644 index ac91059775..0000000000 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_mysql.go +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package databaseinsights - -import ( - "fmt" - "strconv" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - "go.opentelemetry.io/collector/pipeline" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/count" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/forward" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/connector/signaltometrics" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/filelog" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/mysql" -) - -type dbiMysqlTranslator struct { - pipelineType dbiPipelineType - instanceIndex int - cfg dbiInstanceConfig -} - -var _ common.PipelineTranslator = (*dbiMysqlTranslator)(nil) - -func (t *dbiMysqlTranslator) ID() pipeline.ID { - idx := strconv.Itoa(t.instanceIndex) - switch t.pipelineType { - case dbiMetrics: - return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_mysql_"+idx) - case dbiLogToMetrics: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_"+idx) - case dbiRawEvents: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_rawevents_"+idx) - case dbiServerLogs: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_mysql_serverlogs_"+idx) - } - return pipeline.NewID(pipeline.SignalMetrics) -} - -func (t *dbiMysqlTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators, error) { - if err := validateOttlSafe("username", t.cfg.username); err != nil { - return nil, err - } - if err := validateOttlSafe("instance_name", t.cfg.instanceName); err != nil { - return nil, err - } - switch t.pipelineType { - case dbiMetrics: - return t.translateMetrics() - case dbiLogToMetrics: - return t.translateLogToMetrics() - case dbiRawEvents: - return t.translateRawEvents() - case dbiServerLogs: - return t.translateServerLogs() - } - return nil, fmt.Errorf("unknown DBI MySQL pipeline type: %d", t.pipelineType) -} - -func (t *dbiMysqlTranslator) translateMetrics() (*common.ComponentTranslators, error) { - idx := strconv.Itoa(t.instanceIndex) - fwd := forward.NewTranslator(common.OpenTelemetryKey) - countConn := count.NewTranslator(common.DbiConnectorDbloadMysql) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsqlMysql) - - return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("metrics"), countConn, s2mConn), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTimeMysql)), - Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), - Extensions: common.NewTranslatorMap[component.Config, component.ID](), - Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd, countConn, s2mConn), - }, nil -} - -func (t *dbiMysqlTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { - countConn := count.NewTranslator(common.DbiConnectorDbloadMysql) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsqlMysql) - - return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("metrics")), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter()), - Exporters: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), - Extensions: common.NewTranslatorMap[component.Config, component.ID](), - Connectors: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), - }, nil -} - -func (t *dbiMysqlTranslator) translateRawEvents() (*common.ComponentTranslators, error) { - idx := strconv.Itoa(t.instanceIndex) - fwd := forward.NewTranslator(common.OpenTelemetryKey) - - return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.mysqlReceiver("events")), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_mysql_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), - Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), - Extensions: common.NewTranslatorMap[component.Config, component.ID](), - Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), - }, nil -} - -func (t *dbiMysqlTranslator) translateServerLogs() (*common.ComponentTranslators, error) { - idx := strconv.Itoa(t.instanceIndex) - fwd := forward.NewTranslator(common.OpenTelemetryKey) - - return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix("mysql"), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_mysql_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_mysql_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), - Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), - Extensions: common.NewTranslatorMap[component.Config, component.ID](), - Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), - }, nil -} - -func (t *dbiMysqlTranslator) mysqlReceiver(name string, extraOpts ...mysql.Option) common.ComponentTranslator { - opts := []mysql.Option{ - mysql.WithName(name), - mysql.WithIndex(t.instanceIndex), - mysql.WithEndpoint(t.cfg.endpoint), - mysql.WithUsername(t.cfg.username), - mysql.WithPassfile(t.cfg.passfile), - mysql.WithCAFile(t.cfg.caFile), - mysql.WithIsLocalhost(t.cfg.isLocalhost), - } - opts = append(opts, extraOpts...) - return mysql.NewTranslator(opts...) -} - -func (t *dbiMysqlTranslator) excludeMonitorFilter() common.ComponentTranslator { - idx := strconv.Itoa(t.instanceIndex) - condition := fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) - return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_mysql_"+idx, condition) -} - -func (t *dbiMysqlTranslator) scopeTransform() common.ComponentTranslator { - idx := strconv.Itoa(t.instanceIndex) - return transformprocessor.NewTranslatorWithName("dbi_scope_mysql_"+idx, - transformprocessor.WithErrorMode("ignore"), - transformprocessor.WithScopeStatements([]string{ - `set(attributes["cloudwatch.source"], "cloudwatch-agent")`, - `set(attributes["cloudwatch.solution"], "otel-database-insights")`, - }), - ) -} - -func (t *dbiMysqlTranslator) resourceStatements() []string { - return []string{ - `set(resource.attributes["db.system.name"], "mysql")`, - fmt.Sprintf(`set(resource.attributes["db.instance.name"], "%s")`, t.cfg.instanceName), - } -} - -func (t *dbiMysqlTranslator) logStatements(destination string) []string { - return []string{ - fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/%s")`, destination), - fmt.Sprintf(`set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "%s"], "/"))`, t.cfg.instanceName), - } -} diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index 4090105ff6..98ac0daa25 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -16,20 +16,25 @@ import ( func TestDbiTranslatorID(t *testing.T) { tests := []struct { name string + engine string pipelineType dbiPipelineType index int want string }{ - {"Metrics_0", dbiMetrics, 0, "metrics/dbi_postgresql_0"}, - {"Metrics_1", dbiMetrics, 1, "metrics/dbi_postgresql_1"}, - {"LogToMetrics_0", dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, - {"RawEvents_0", dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, - {"ServerLogs_0", dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, - {"ServerLogs_2", dbiServerLogs, 2, "logs/dbi_postgresql_serverlogs_2"}, + {"PG_Metrics_0", common.PostgreSQLKey, dbiMetrics, 0, "metrics/dbi_postgresql_0"}, + {"PG_Metrics_1", common.PostgreSQLKey, dbiMetrics, 1, "metrics/dbi_postgresql_1"}, + {"PG_LogToMetrics_0", common.PostgreSQLKey, dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, + {"PG_RawEvents_0", common.PostgreSQLKey, dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, + {"PG_ServerLogs_0", common.PostgreSQLKey, dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, + {"Mysql_Metrics_0", common.MySQLKey, dbiMetrics, 0, "metrics/dbi_mysql_0"}, + {"Mysql_Metrics_1", common.MySQLKey, dbiMetrics, 1, "metrics/dbi_mysql_1"}, + {"Mysql_LogToMetrics_0", common.MySQLKey, dbiLogToMetrics, 0, "logs/dbi_mysql_0"}, + {"Mysql_RawEvents_0", common.MySQLKey, dbiRawEvents, 0, "logs/dbi_mysql_rawevents_0"}, + {"Mysql_ServerLogs_0", common.MySQLKey, dbiServerLogs, 0, "logs/dbi_mysql_serverlogs_0"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index} + tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index, cfg: dbiInstanceConfig{engine: tc.engine}} assert.Equal(t, tc.want, tr.ID().String()) }) } @@ -37,6 +42,7 @@ func TestDbiTranslatorID(t *testing.T) { func TestDbiTranslate(t *testing.T) { cfg := dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: "localhost:5432", username: "cw_monitor", passfile: "/etc/.pgpass", @@ -79,7 +85,7 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { tr := &dbiTranslator{ pipelineType: dbiMetrics, instanceIndex: 0, - cfg: dbiInstanceConfig{instanceName: "mydb"}, + cfg: dbiInstanceConfig{engine: common.PostgreSQLKey, instanceName: "mydb"}, } result, err := tr.Translate(nil) require.NoError(t, err) @@ -98,35 +104,15 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { connectors = append(connectors, c.ID().String()) }) - assert.ElementsMatch(t, []string{"postgresql/metrics_0", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, receivers) - assert.Equal(t, []string{"transform/dbi_scope_0", "transform/dbi_resource_0", "transform/dbi_fix_start_time"}, processors) + assert.ElementsMatch(t, []string{"postgresql/metrics_0", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_postgresql_0", "transform/dbi_resource_postgresql_0", "transform/dbi_fix_start_time_postgresql"}, processors) assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) - assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, connectors) -} - -func TestDbiMysqlTranslatorID(t *testing.T) { - tests := []struct { - pipelineType dbiPipelineType - index int - want string - }{ - {dbiMetrics, 0, "metrics/dbi_mysql_0"}, - {dbiMetrics, 1, "metrics/dbi_mysql_1"}, - {dbiLogToMetrics, 0, "logs/dbi_mysql_0"}, - {dbiRawEvents, 0, "logs/dbi_mysql_rawevents_0"}, - {dbiServerLogs, 0, "logs/dbi_mysql_serverlogs_0"}, - {dbiServerLogs, 2, "logs/dbi_mysql_serverlogs_2"}, - } - for _, tc := range tests { - t.Run(tc.want, func(t *testing.T) { - tr := &dbiMysqlTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index} - assert.Equal(t, tc.want, tr.ID().String()) - }) - } + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, connectors) } func TestDbiMysqlTranslate(t *testing.T) { cfg := dbiInstanceConfig{ + engine: common.MySQLKey, endpoint: "localhost:3306", username: "cw_monitor", passfile: "/etc/.mysql_credentials", @@ -152,7 +138,7 @@ func TestDbiMysqlTranslate(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tr := &dbiMysqlTranslator{pipelineType: tc.pipeline, instanceIndex: 0, cfg: cfg} + tr := &dbiTranslator{pipelineType: tc.pipeline, instanceIndex: 0, cfg: cfg} assert.Equal(t, tc.expectedID, tr.ID().String()) result, err := tr.Translate(nil) @@ -166,10 +152,10 @@ func TestDbiMysqlTranslate(t *testing.T) { } func TestDbiMysqlTranslateMetrics_ComponentIDs(t *testing.T) { - tr := &dbiMysqlTranslator{ + tr := &dbiTranslator{ pipelineType: dbiMetrics, instanceIndex: 0, - cfg: dbiInstanceConfig{instanceName: "mydb"}, + cfg: dbiInstanceConfig{engine: common.MySQLKey, instanceName: "mydb"}, } result, err := tr.Translate(nil) require.NoError(t, err) diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index a24cc9a3fd..a40e6a637e 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -24,6 +24,7 @@ var ( ) type dbiInstanceConfig struct { + engine string endpoint string username string passfile string @@ -39,25 +40,21 @@ func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap { return translators } - pgInstances := parseDbiPostgresqlInstances(conf) - for i, cfg := range pgInstances { - translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) - if cfg.logFilePath != "" { - translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + // Each engine is indexed independently; the engine name is part of every + // component ID, so PostgreSQL and MySQL instances never collide. + addInstances := func(instances []dbiInstanceConfig) { + for i, cfg := range instances { + translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) + if cfg.logFilePath != "" { + translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + } } } - mysqlInstances := parseDbiMysqlInstances(conf) - for i, cfg := range mysqlInstances { - translators.Set(&dbiMysqlTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiMysqlTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiMysqlTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) - if cfg.logFilePath != "" { - translators.Set(&dbiMysqlTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) - } - } + addInstances(parseDbiPostgresqlInstances(conf)) + addInstances(parseDbiMysqlInstances(conf)) return translators } @@ -82,6 +79,7 @@ func parseDbiPostgresqlInstances(conf *confmap.Conf) []dbiInstanceConfig { instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { instances = append(instances, dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: r.Endpoint, username: r.Username, passfile: r.PasswordFile, @@ -109,10 +107,7 @@ type mysqlRawInstance struct { Username string `mapstructure:"username"` PasswordFile string `mapstructure:"password_file"` InstanceName string `mapstructure:"instance_name"` - TLS struct { - CAFile string `mapstructure:"ca_file"` - } `mapstructure:"tls"` - Logs struct { + Logs struct { FilePath string `mapstructure:"file_path"` } `mapstructure:"logs"` } @@ -126,10 +121,10 @@ func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { instances = append(instances, dbiInstanceConfig{ + engine: common.MySQLKey, endpoint: r.Endpoint, username: r.Username, passfile: r.PasswordFile, - caFile: r.TLS.CAFile, instanceName: r.InstanceName, logFilePath: r.Logs.FilePath, isLocalhost: isLocalhostEndpoint(r.Endpoint), diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml similarity index 100% rename from translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml rename to translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go index a613f7f67f..14102fc8e1 100644 --- a/translator/translate/otel/processor/transformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -26,7 +26,7 @@ var transformJmxDropConfig string //go:embed transform_efa_config.yaml var transformEfaConfig string -//go:embed transform_dbi_fix_start_time.yaml +//go:embed transform_dbi_fix_start_time_postgresql.yaml var transformDbiFixStartTimeConfig string //go:embed transform_dbi_fix_start_time_mysql.yaml @@ -74,13 +74,22 @@ func WithScopeStatements(statements []string) Option { } } +// WithDbiFixStartTime selects the DBI fix-start-time config for the given engine +// ("postgresql" or "mysql"). The component is named by the caller (index-based). +func WithDbiFixStartTime(engine string) Option { + return func(t *translator) { + t.dbiFixStartTimeEngine = engine + } +} + type translator struct { - name string - factory processor.Factory - logStatements []string - metricStatements []string - scopeStatements []string - errorMode string + name string + factory processor.Factory + logStatements []string + metricStatements []string + scopeStatements []string + errorMode string + dbiFixStartTimeEngine string } var _ common.ComponentTranslator = (*translator)(nil) @@ -137,11 +146,15 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.name, common.PipelineNameHostDeltaMetrics) { return common.GetYamlFileToYamlConfig(cfg, transformEfaConfig) } - switch t.name { - case common.DbiTransformFixStartTime: - return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) - case common.DbiTransformFixStartTimeMysql: - return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeMysqlConfig) + // DBI fix-start-time: content is engine-specific, component name is index-based. + if t.dbiFixStartTimeEngine != "" { + switch t.dbiFixStartTimeEngine { + case common.PostgreSQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeMysqlConfig) + } + return nil, fmt.Errorf("unsupported dbi fix-start-time engine: %s", t.dbiFixStartTimeEngine) } if t.name == common.Identity { if context.CurrentContext().KubernetesMode() != "" { diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index 6f67f89aec..639060c2eb 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -76,8 +76,8 @@ func TestJmxTranslate(t *testing.T) { } func TestDbiFixStartTimeTranslate(t *testing.T) { - transl := NewTranslatorWithName(common.DbiTransformFixStartTime) - assert.Equal(t, "transform/dbi_fix_start_time", transl.ID().String()) + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.PostgreSQLKey, WithDbiFixStartTime(common.PostgreSQLKey)) + assert.Equal(t, "transform/dbi_fix_start_time_postgresql", transl.ID().String()) cfg, err := transl.Translate(nil) require.NoError(t, err) @@ -90,7 +90,7 @@ func TestDbiFixStartTimeTranslate(t *testing.T) { } func TestDbiFixStartTimeMysqlTranslate(t *testing.T) { - transl := NewTranslatorWithName(common.DbiTransformFixStartTimeMysql) + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.MySQLKey, WithDbiFixStartTime(common.MySQLKey)) assert.Equal(t, "transform/dbi_fix_start_time_mysql", transl.ID().String()) cfg, err := transl.Translate(nil) diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index 2dabba46c4..7d514df384 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -25,8 +25,6 @@ type translator struct { endpoint string username string passfile string - caFile string - isLocalhost bool index int topQueryInterval time.Duration } @@ -35,8 +33,6 @@ func WithName(name string) Option { return func(t *translator) { t.name = name func WithEndpoint(ep string) Option { return func(t *translator) { t.endpoint = ep } } func WithUsername(u string) Option { return func(t *translator) { t.username = u } } func WithPassfile(p string) Option { return func(t *translator) { t.passfile = p } } -func WithCAFile(ca string) Option { return func(t *translator) { t.caFile = ca } } -func WithIsLocalhost(b bool) Option { return func(t *translator) { t.isLocalhost = b } } func WithIndex(i int) Option { return func(t *translator) { t.index = i } } func WithTopQueryInterval(d time.Duration) Option { return func(t *translator) { t.topQueryInterval = d } @@ -66,16 +62,10 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg.Passfile = t.passfile cfg.Transport = confignet.TransportTypeTCP - if t.isLocalhost { - cfg.TLS = configtls.ClientConfig{ - Insecure: true, - } - } else { - cfg.TLS = configtls.ClientConfig{ - Config: configtls.Config{ - CAFile: t.caFile, - }, - } + // DBI only supports local MySQL instances, so the connection does not use + // TLS. Remote (TLS) connections are intentionally not supported. + cfg.TLS = configtls.ClientConfig{ + Insecure: true, } cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go index 769d19872d..587b13f02d 100644 --- a/translator/translate/otel/receiver/mysql/translator_test.go +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -25,7 +25,6 @@ func TestTranslator_Translate_Localhost(t *testing.T) { WithEndpoint("localhost:3306"), WithUsername("cw_monitor"), WithPassfile("/etc/.mysql_credentials"), - WithIsLocalhost(true), WithIndex(0), ) cfg, err := tr.Translate(nil) @@ -51,33 +50,9 @@ func TestTranslator_Translate_CustomInterval(t *testing.T) { WithEndpoint("localhost:3306"), WithUsername("cw_monitor"), WithPassfile("/etc/.mysql_credentials"), - WithIsLocalhost(true), WithTopQueryInterval(30*time.Second), ) cfg, err := tr.Translate(nil) require.NoError(t, err) assert.Equal(t, 30*time.Second, cfg.(*mysqlreceiver.Config).TopQueryCollection.CollectionInterval) } - -func TestTranslator_Translate_Remote(t *testing.T) { - tr := NewTranslator( - WithName("events"), - WithEndpoint("db.example.com:3306"), - WithUsername("cw_monitor"), - WithPassfile("/etc/.mysql_credentials"), - WithCAFile("/etc/ssl/ca.pem"), - WithIsLocalhost(false), - WithIndex(0), - ) - cfg, err := tr.Translate(nil) - require.NoError(t, err) - mysqlCfg := cfg.(*mysqlreceiver.Config) - - assert.Equal(t, "db.example.com:3306", mysqlCfg.Endpoint) - assert.False(t, mysqlCfg.TLS.Insecure) - assert.Equal(t, "/etc/ssl/ca.pem", string(mysqlCfg.TLS.CAFile)) - assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) - assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) - assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) - assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) -} From 26f8bb9815cb7154ad7cb41e0b5729bf50f96790 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 30 Jun 2026 17:04:45 +0000 Subject: [PATCH 07/28] feat(translator): Add database_insights merge rule for multi-config support The CloudWatch Console (Cinch) creates separate SSM parameters for each workload configuration (e.g. one for database-insights-postgres and one for database-insights-mysql). When the agent loads multiple config files from the .d/ directory, the config merger needs to deep-merge the database_insights section so that both postgresql and mysql arrays coexist. Without this rule, the agent treats database_insights from two separate config sources as conflicting values and drops one, causing missing log groups in CloudWatch. Register database_insights as a SectionMergeRule under the collect rule, matching the existing pattern for opentelemetry and collect. --- .../otel/pipeline/opentelemetry/merge_rule.go | 2 + .../pipeline/opentelemetry/merge_rule_test.go | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go index 94f645ee65..369c00932a 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go @@ -10,6 +10,8 @@ import ( func init() { otelRule := mergeJsonUtil.NewSectionMergeRule("opentelemetry", "/") collectRule := mergeJsonUtil.NewSectionMergeRule("collect", otelRule.Path) + dbiRule := mergeJsonUtil.NewSectionMergeRule("database_insights", collectRule.Path) + collectRule.MergeMap[dbiRule.SectionKey] = dbiRule otelRule.MergeMap[collectRule.SectionKey] = collectRule mergeJsonUtil.MergeRuleMap[otelRule.SectionKey] = otelRule } diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go index f5918d7b18..2d6e120cf9 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go @@ -32,3 +32,46 @@ func TestMergeRule(t *testing.T) { assert.Contains(t, otel, "collect") assert.Contains(t, otel["collect"].(map[string]interface{}), "host_metrics") } + +func TestMergeRule_DatabaseInsights(t *testing.T) { + // Verify that postgresql and mysql from separate configs merge correctly + // under database_insights (simulates separate SSM parameters) + source1 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:5432", + "instance_name": "pg-test", + }, + }, + }, + }, + }, + } + source2 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", + "instance_name": "mysql-test", + }, + }, + }, + }, + }, + } + + // Merge source1 into source2 (source2 acts as result) + mergeJsonUtil.MergeRuleMap["opentelemetry"].Merge(source1, source2) + + otel := source2["opentelemetry"].(map[string]interface{}) + collect := otel["collect"].(map[string]interface{}) + dbi := collect["database_insights"].(map[string]interface{}) + + assert.Contains(t, dbi, "postgresql", "postgresql should be preserved after merge") + assert.Contains(t, dbi, "mysql", "mysql should be preserved after merge") +} From 02b6d27cc23d4b4927ec529126d3ec48469ab13a Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Wed, 1 Jul 2026 18:52:40 +0000 Subject: [PATCH 08/28] Address PR review comments - Move the complete 14-metric Top SQL connector config (dbi_topsql_mysql.yaml) into this PR. The full metric set was previously committed in the MySQL workload-detection PR by mistake, which left this PR's embedded config with only 2 metrics while the golden YAML had 14 (mismatch that failed the translation unit tests). All 14 metrics match what the mysqlreceiver publishes on db.server.top_query events. - Remove the user.name attribute from all Top SQL metrics: top queries come from events_statements_summary_by_digest, aggregated by digest and schema, so the receiver does not publish user.name on top_query events (validated against the forked mysqlreceiver RecordDbServerTopQueryEvent). - Remove the duplicate dbiMysqlKey variable in favor of the shared common.DatabaseInsightsMysqlKey. - Regenerate the MySQL golden YAML. --- .../opentelemetry/dbi_mysql_config_linux.yaml | 28 ---- .../signaltometrics/dbi_topsql_mysql.yaml | 146 +++++++++++++++++- .../signaltometrics/translator_test.go | 14 +- .../databaseinsights/translators.go | 6 +- 4 files changed, 158 insertions(+), 36 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index 8aa191b9dd..3630425c1c 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -78,8 +78,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil description: SQL execution count @@ -97,8 +95,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil description: Total execution time in picoseconds @@ -116,8 +112,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil description: Total lock time in seconds @@ -135,8 +129,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil description: Rows sent to client @@ -154,8 +146,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil description: Rows examined @@ -173,8 +163,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil description: Total errors @@ -192,8 +180,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil description: Rows sorted @@ -211,8 +197,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil description: Temporary tables created @@ -230,8 +214,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil description: Temporary disk tables created @@ -249,8 +231,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil description: Queries using no index @@ -268,8 +248,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil description: Full joins without index @@ -287,8 +265,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil description: Sorts done via full scan @@ -306,8 +282,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil description: Queries using no good index @@ -325,8 +299,6 @@ connectors: optional: false - key: db.namespace optional: false - - key: user.name - optional: false conditions: - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil description: Full table scans diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml index 704317a010..bf3bb3131e 100644 --- a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml +++ b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml @@ -7,7 +7,6 @@ logs: - key: db.system.name - key: mysql.events_statements_summary_by_digest.digest - key: db.namespace - - key: user.name conditions: - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil sum: @@ -20,8 +19,151 @@ logs: - key: db.system.name - key: mysql.events_statements_summary_by_digest.digest - key: db.namespace - - key: user.name conditions: - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil sum: value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"])' + - name: mysql.sum_lock_time + include_resource_attributes: + - key: service.instance.id + description: "Total lock time in seconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"])' + - name: mysql.sum_rows_sent + include_resource_attributes: + - key: service.instance.id + description: "Rows sent to client" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"])' + - name: mysql.sum_rows_examined + include_resource_attributes: + - key: service.instance.id + description: "Rows examined" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"])' + - name: mysql.sum_errors + include_resource_attributes: + - key: service.instance.id + description: "Total errors" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"])' + - name: mysql.sum_sort_rows + include_resource_attributes: + - key: service.instance.id + description: "Rows sorted" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"])' + - name: mysql.sum_created_tmp_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"])' + - name: mysql.sum_created_tmp_disk_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary disk tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"])' + - name: mysql.sum_no_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"])' + - name: mysql.sum_select_full_join + include_resource_attributes: + - key: service.instance.id + description: "Full joins without index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"])' + - name: mysql.sum_sort_scan + include_resource_attributes: + - key: service.instance.id + description: "Sorts done via full scan" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"])' + - name: mysql.sum_no_good_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no good index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"])' + - name: mysql.sum_select_scan + include_resource_attributes: + - key: service.instance.id + description: "Full table scans" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"])' diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index 5a9c5d254d..e7a8b68fe4 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -52,7 +52,19 @@ func TestTranslateTopsqlMysql(t *testing.T) { cfg, err := tr.Translate(nil) require.NoError(t, err) stmCfg := cfg.(*signaltometricsconfig.Config) - assert.Len(t, stmCfg.Logs, 2) + assert.Len(t, stmCfg.Logs, 14) assert.Equal(t, "mysql.count_star", stmCfg.Logs[0].Name) assert.Equal(t, "mysql.sum_timer_wait", stmCfg.Logs[1].Name) + assert.Equal(t, "mysql.sum_lock_time", stmCfg.Logs[2].Name) + assert.Equal(t, "mysql.sum_rows_sent", stmCfg.Logs[3].Name) + assert.Equal(t, "mysql.sum_rows_examined", stmCfg.Logs[4].Name) + assert.Equal(t, "mysql.sum_errors", stmCfg.Logs[5].Name) + assert.Equal(t, "mysql.sum_sort_rows", stmCfg.Logs[6].Name) + assert.Equal(t, "mysql.sum_created_tmp_tables", stmCfg.Logs[7].Name) + assert.Equal(t, "mysql.sum_created_tmp_disk_tables", stmCfg.Logs[8].Name) + assert.Equal(t, "mysql.sum_no_index_used", stmCfg.Logs[9].Name) + assert.Equal(t, "mysql.sum_select_full_join", stmCfg.Logs[10].Name) + assert.Equal(t, "mysql.sum_sort_scan", stmCfg.Logs[11].Name) + assert.Equal(t, "mysql.sum_no_good_index_used", stmCfg.Logs[12].Name) + assert.Equal(t, "mysql.sum_select_scan", stmCfg.Logs[13].Name) } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index a40e6a637e..7d6c9a66a6 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -19,10 +19,6 @@ import ( // characters, preventing injection of OTTL syntax or string escape sequences. var ottlSafeRegex = regexp.MustCompile(`^[a-zA-Z0-9._@/:\-]+$`) -var ( - dbiMysqlKey = common.ConfigKey(common.OpenTelemetryKey, common.CollectKey, common.DatabaseInsightsKey, common.MySQLKey) -) - type dbiInstanceConfig struct { engine string endpoint string @@ -113,7 +109,7 @@ type mysqlRawInstance struct { } func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { - arr, _ := conf.Get(dbiMysqlKey).([]any) + arr, _ := conf.Get(common.DatabaseInsightsMysqlKey).([]any) var raw []mysqlRawInstance if err := mapstructure.Decode(arr, &raw); err != nil { return nil From 4c84ea56c82fedd9ae073574bbb40f69448791ee Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 2 Jul 2026 19:42:06 +0000 Subject: [PATCH 09/28] test: Update .mysql_credentials test fixture to .my.cnf INI format Update the test credential file to use the MySQL-native INI format (Option C) with required host, port, user, and password fields. --- .../sampleConfig/opentelemetry/testdata/.mysql_credentials | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials index 063172194e..63df2f15cc 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials +++ b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials @@ -1 +1,5 @@ -localhost:3306:*:cw_monitor:test_password +[client] +host=localhost +port=3306 +user=cw_monitor +password=test_password From 15156a62a0b3754f365817e4a422a0761fa34fdf Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 2 Jul 2026 19:46:50 +0000 Subject: [PATCH 10/28] chore: Update mysqlreceiver replace to .my.cnf INI format commit Points to amazon-contributing/opentelemetry-collector-contrib PR #577 which replaces pgpass format with .my.cnf INI format for MySQL DBI credential files. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 81e5202fe8..a273ecc85a 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 ) @@ -689,7 +689,7 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect k8s.io/kubelet v0.32.3 // indirect diff --git a/go.sum b/go.sum index 3ae7afa9bc..2c5d69e49b 100644 --- a/go.sum +++ b/go.sum @@ -278,8 +278,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 h1:R4Ems+YMAXLtsEZwRXw8/cf1NkrzHQsc25WPdYbUaTM= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a h1:ScKUmN6HDigSB0rceZR3CmcO/CVDTxOtrk8XQSTC+Q4= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260623153739-5a887b09871a/go.mod h1:tnnmAnxuLxWKBG7NjOI3po8R+EAX/9cG1y7TWiQTMvA= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730 h1:rhNSDEYjzOSuk2vVpqTdm2LLbzM/y2cT/9EVZKLu7Kk= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 h1:ZLP3mZHMXzUjiXxXFVI9Z5LtU25qT13Qajz84mna/LY= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gSxi2fY+T8mNzlaU7qOEnaXTcKfhIpijDSyeCu5GOYo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 h1:p0HB9BwhUdkaI0ALIb3hYxi7oCTDsGUuGadeM3R36AM= @@ -2425,8 +2425,8 @@ gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= From f7367a875b578f07e888810dfa03c1b4d3bf20d9 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Fri, 10 Jul 2026 15:31:07 +0000 Subject: [PATCH 11/28] fix(dbi): Count Execute command sessions in MySQL DB Load metrics The DB Load connector condition only matched processlist_command == "Query", missing prepared statement executions ("Execute") used by sysbench, ORMs, JDBC, and most real-world applications. This caused DB Load metrics to undercount active sessions. Replace the command-based filter with session.status != "idle" to match all active sessions regardless of command type. --- .../opentelemetry/dbi_mysql_config_linux.yaml | 14 +++++++------- .../otel/connector/count/dbi_dbload_mysql.yaml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index 3630425c1c..9c9eb5620d 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -9,7 +9,7 @@ connectors: - key: db.system.name - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by database mysql.active_sessions.by_host: attributes: @@ -17,7 +17,7 @@ connectors: - key: network.peer.address - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by host mysql.active_sessions.by_sql: attributes: @@ -25,7 +25,7 @@ connectors: - key: mysql.events_statements_current.digest - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by query digest mysql.active_sessions.by_sql_wait: attributes: @@ -34,7 +34,7 @@ connectors: - key: mysql.wait_type - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by query and wait mysql.active_sessions.by_user: attributes: @@ -42,7 +42,7 @@ connectors: - key: user.name - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by user mysql.active_sessions.by_wait: attributes: @@ -50,14 +50,14 @@ connectors: - key: mysql.wait_type - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Active sessions by wait event mysql.active_sessions.count: attributes: - key: db.system.name - key: db.namespace conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" description: Total active sessions metrics: metric.count: diff --git a/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml index 0e0f381c1b..5a181412f9 100644 --- a/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml +++ b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml @@ -2,7 +2,7 @@ logs: mysql.active_sessions.by_wait: description: "Active sessions by wait event" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: mysql.wait_type @@ -10,7 +10,7 @@ logs: mysql.active_sessions.by_user: description: "Active sessions by user" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: user.name @@ -18,14 +18,14 @@ logs: mysql.active_sessions.by_db: description: "Active sessions by database" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: db.namespace mysql.active_sessions.by_sql: description: "Active sessions by query digest" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: mysql.events_statements_current.digest @@ -33,7 +33,7 @@ logs: mysql.active_sessions.by_sql_wait: description: "Active sessions by query and wait" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: mysql.events_statements_current.digest @@ -42,7 +42,7 @@ logs: mysql.active_sessions.by_host: description: "Active sessions by host" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: network.peer.address @@ -50,7 +50,7 @@ logs: mysql.active_sessions.count: description: "Total active sessions" conditions: - - attributes["mysql.threads.processlist_command"] == "Query" + - attributes["mysql.session.status"] != "idle" attributes: - key: db.system.name - key: db.namespace From 7f4e6b1a7c3ea72f1844304f599e0c58092776ed Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Mon, 20 Jul 2026 20:06:30 +0000 Subject: [PATCH 12/28] feat(translator): add events_statements_summary_by_digest Top SQL metrics Add sum_rows_affected, sum_select_range_check, sum_sort_merge_passes, and sum_sort_range to dbi_topsql_mysql.yaml signaltometrics connector config. Top SQL metrics now emit 18 metrics per digest (was 14). Regenerated golden YAML test fixture. --- .../opentelemetry/dbi_mysql_config_linux.yaml | 68 +++++++++++++++++++ .../signaltometrics/dbi_topsql_mysql.yaml | 48 +++++++++++++ .../signaltometrics/translator_test.go | 6 +- 3 files changed, 121 insertions(+), 1 deletion(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index 9c9eb5620d..6a1a2d9f35 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -309,6 +309,74 @@ connectors: sum: value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" exporters: otlphttp/logs: auth: diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml index bf3bb3131e..8cba45ec24 100644 --- a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml +++ b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml @@ -167,3 +167,51 @@ logs: - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil sum: value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"])' + - name: mysql.sum_rows_affected + include_resource_attributes: + - key: service.instance.id + description: "Rows affected" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"])' + - name: mysql.sum_select_range_check + include_resource_attributes: + - key: service.instance.id + description: "Joins checking key usage per row" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"])' + - name: mysql.sum_sort_merge_passes + include_resource_attributes: + - key: service.instance.id + description: "Sort merge passes" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"])' + - name: mysql.sum_sort_range + include_resource_attributes: + - key: service.instance.id + description: "Sorts using ranges" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"])' diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index e7a8b68fe4..6515b90bba 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -52,7 +52,7 @@ func TestTranslateTopsqlMysql(t *testing.T) { cfg, err := tr.Translate(nil) require.NoError(t, err) stmCfg := cfg.(*signaltometricsconfig.Config) - assert.Len(t, stmCfg.Logs, 14) + assert.Len(t, stmCfg.Logs, 18) assert.Equal(t, "mysql.count_star", stmCfg.Logs[0].Name) assert.Equal(t, "mysql.sum_timer_wait", stmCfg.Logs[1].Name) assert.Equal(t, "mysql.sum_lock_time", stmCfg.Logs[2].Name) @@ -67,4 +67,8 @@ func TestTranslateTopsqlMysql(t *testing.T) { assert.Equal(t, "mysql.sum_sort_scan", stmCfg.Logs[11].Name) assert.Equal(t, "mysql.sum_no_good_index_used", stmCfg.Logs[12].Name) assert.Equal(t, "mysql.sum_select_scan", stmCfg.Logs[13].Name) + assert.Equal(t, "mysql.sum_rows_affected", stmCfg.Logs[14].Name) + assert.Equal(t, "mysql.sum_select_range_check", stmCfg.Logs[15].Name) + assert.Equal(t, "mysql.sum_sort_merge_passes", stmCfg.Logs[16].Name) + assert.Equal(t, "mysql.sum_sort_range", stmCfg.Logs[17].Name) } From 33f7e0027c07179a30e111cac0307e528b1ea429 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Wed, 22 Jul 2026 14:17:20 +0000 Subject: [PATCH 13/28] chore: Update mysqlreceiver replace to latest commit c7001ee3110e --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a273ecc85a..f0d27eebe9 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 ) diff --git a/go.sum b/go.sum index 2c5d69e49b..42a6891954 100644 --- a/go.sum +++ b/go.sum @@ -278,8 +278,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 h1:R4Ems+YMAXLtsEZwRXw8/cf1NkrzHQsc25WPdYbUaTM= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730 h1:rhNSDEYjzOSuk2vVpqTdm2LLbzM/y2cT/9EVZKLu7Kk= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260715195404-79c2b737d730/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e h1:Nk/ryqhal2guaLPnMhd4gqbZmQjy6OK+J1m/7snhuME= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 h1:ZLP3mZHMXzUjiXxXFVI9Z5LtU25qT13Qajz84mna/LY= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gSxi2fY+T8mNzlaU7qOEnaXTcKfhIpijDSyeCu5GOYo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 h1:p0HB9BwhUdkaI0ALIb3hYxi7oCTDsGUuGadeM3R36AM= From 5c4e9ecad1c7b8cec55f8ce290997ef85edffcc9 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 23 Jul 2026 15:05:18 +0000 Subject: [PATCH 14/28] Enable mysql.table.size and replica metrics for DBI The mysql DBI translator relied on the receiver's default metric enablement, which leaves mysql.table.size, mysql.replica.sql_delay, and mysql.replica.time_behind_source disabled. Enable them explicitly in the translator (matching the postgres translator pattern) so Database Insights collects table size and replication lag. Regenerated the dbi_mysql golden config accordingly. --- .../opentelemetry/dbi_mysql_config_linux.yaml | 12 ++++++------ .../translate/otel/receiver/mysql/translator.go | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index c6ce62231a..29bcfd0910 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -1132,9 +1132,9 @@ receivers: mysql.query.slow.count: enabled: false mysql.replica.sql_delay: - enabled: false + enabled: true mysql.replica.time_behind_source: - enabled: false + enabled: true mysql.row_locks: aggregation_strategy: sum attributes: @@ -1227,7 +1227,7 @@ receivers: - table - schema - kind - enabled: false + enabled: true mysql.table_open_cache: aggregation_strategy: sum attributes: @@ -1404,9 +1404,9 @@ receivers: mysql.query.slow.count: enabled: false mysql.replica.sql_delay: - enabled: false + enabled: true mysql.replica.time_behind_source: - enabled: false + enabled: true mysql.row_locks: aggregation_strategy: sum attributes: @@ -1499,7 +1499,7 @@ receivers: - table - schema - kind - enabled: false + enabled: true mysql.table_open_cache: aggregation_strategy: sum attributes: diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index 7d514df384..347b615a1b 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -68,6 +68,13 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { Insecure: true, } + // Enable metrics that are disabled by default in the receiver but required + // by Database Insights. mysql.replica.* metrics only emit when the monitored + // instance is a replica; they are inert on standalone/primary instances. + cfg.MetricsBuilderConfig.Metrics.MysqlTableSize.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlReplicaSQLDelay.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlReplicaTimeBehindSource.Enabled = true + cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true From 5f2550af491e62be9f86391f2b65c57efb18106c Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 23 Jul 2026 15:21:51 +0000 Subject: [PATCH 15/28] fix(dbi): parse MySQL server log lines for multiline, timestamp, and severity --- .../opentelemetry/dbi_mysql_config_linux.yaml | 46 +++++--- .../databaseinsights/translator.go | 111 +++++++++++------- .../databaseinsights/translator_test.go | 2 +- .../databaseinsights/translators_test.go | 4 +- .../otel/receiver/mysql/translator.go | 3 +- 5 files changed, 102 insertions(+), 64 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index 29bcfd0910..df327b912b 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -512,6 +512,9 @@ processors: metrics: {} spans: {} traces: {} + groupbyattrs/files: + keys: + - log.file.name resourcedetection/opentelemetry: aks: resource_attributes: @@ -826,6 +829,11 @@ processors: statements: - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] transform/dbi_resource_mysql_0: @@ -933,25 +941,30 @@ processors: receivers: filelog/mysql_0: encoding: utf-8 - exclude_older_than: 0s - fingerprint_size: 1000 - force_flush_period: 500ms - id: file_input include: - /var/log/mysql/mysql.log - include_file_name: true - initial_buffer_size: 16384 - max_concurrent_files: 1024 - max_log_size: 1048576 - operators: [] - poll_interval: 200ms - retry_on_failure: - enabled: false - initial_interval: 1s - max_elapsed_time: 5m0s - max_interval: 30s + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser start_at: end - type: file_input hostmetrics/opentelemetry: collection_interval: 30s initial_delay: 1s @@ -1576,6 +1589,7 @@ service: processors: - transform/dbi_scope_mysql_0 - resourcedetection/opentelemetry + - groupbyattrs/files - transform/dbi_resource_mysql_0 - transform/dbi_logs_mysql_server-logs_0 receivers: diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index 985e32c43c..7fa6234396 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -60,6 +60,30 @@ func buildPostgresSeverityPattern() string { return `\[\d+\]\s*(?P` + strings.Join(postgresLogSeverityLevels, "|") + `):` } +// mysqlLogSeverityLevels are the severity labels MySQL emits in the classic error log +// (per MySQL worklog #10942): Note/Warning/ERROR, plus System for force-printed events. +// Kept alongside the mapping below so the regex and mapping stay together. +var mysqlLogSeverityLevels = []string{"System", "Warning", "Note", "ERROR"} + +// mysqlLogSeverityMapping maps MySQL error-log labels to OTEL severity levels for the +// filelog severity operator. Kept here (not in the generic filelog translator) because +// these labels are MySQL-specific. System and Note are informational; there is no +// MySQL debug/fatal label in the classic error log. +var mysqlLogSeverityMapping = map[string]any{ + "info": []string{"System", "Note"}, + "warn": "Warning", + "error": "ERROR", +} + +// buildMysqlSeverityPattern builds the regex that extracts the severity from a MySQL +// error-log line of the form " [] [MY-######] ...". +// Anchoring on the numeric thread id before the bracket avoids matching the later +// error-code bracket (e.g. "[MY-010116]"). The required (?P...) named capture +// group is assembled from mysqlLogSeverityLevels. +func buildMysqlSeverityPattern() string { + return `\s\d+\s+\[(?P` + strings.Join(mysqlLogSeverityLevels, "|") + `)\]` +} + // dbiTranslator generates DBI pipelines for a single database instance. The // engine (carried on cfg) selects engine-specific receivers, connector configs, // resource attributes, and log group paths. Component IDs are index-based so @@ -168,46 +192,37 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - processors := common.NewTranslatorMap[component.Config, component.ID]( - t.scopeTransform(), - resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), - ) - - logsTransformOpts := []transformprocessor.Option{ - transformprocessor.WithLogResourceStatements(t.logStatements("server-logs")), - } - - if t.cfg.engine == common.PostgreSQLKey { - // Promote log.file.name to a resource attribute, reusing the files pipeline's - // shared groupbyattrs instance so server-log parsing/grouping is consistent - // with regular file logs. - processors.Set(groupbyattrsprocessor.NewTranslatorWithName(common.FilesKey, "log.file.name")) - // Drop the parsed attributes once promoted to the record's timestamp and - // severity fields, mirroring the files pipeline's timestamp cleanup, so they - // are not duplicated in the emitted log attributes. - logsTransformOpts = append(logsTransformOpts, transformprocessor.WithLogContextStatements([]string{ - `delete_key(attributes, "timestamp")`, - `delete_key(attributes, "severity")`, - })) - } - - processors.Set(transformprocessor.NewTranslatorWithName( - common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, - transformprocessor.WithMetricResourceStatements(t.resourceStatements()), - transformprocessor.WithLogResourceStatements(t.resourceStatements()), - )) - processors.Set(transformprocessor.NewTranslatorWithName( - common.DbiTransformLogs+"_"+t.cfg.engine+"_server-logs_"+idx, - logsTransformOpts..., - )) - - // NOTE: timestamp parsing assumes the instance logs in UTC. The %Z token only matches - // 3-letter zone abbreviations and the gotime layout parses them against a UTC location, - // so timestamps from a non-UTC instance would be silently offset. DBI therefore requires - // the instance to be configured to emit UTC timestamps in its logs. + // NOTE: timestamp parsing assumes the instance logs in UTC. Both engines' timestamp + // layouts parse against a UTC location, so timestamps from a non-UTC instance would be + // silently offset. DBI therefore requires the instance to emit UTC timestamps in its + // logs (PostgreSQL: log_timezone = 'UTC'; MySQL: log_timestamps = UTC, which is the + // default). return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.serverLogReceiver()), - Processors: processors, + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.serverLogReceiver()), + Processors: common.NewTranslatorMap[component.Config, component.ID]( + t.scopeTransform(), + resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), + // Promote log.file.name to a resource attribute, reusing the files pipeline's + // shared groupbyattrs instance so server-log parsing/grouping is consistent + // with regular file logs. + groupbyattrsprocessor.NewTranslatorWithName(common.FilesKey, "log.file.name"), + transformprocessor.NewTranslatorWithName( + common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, + transformprocessor.WithMetricResourceStatements(t.resourceStatements()), + transformprocessor.WithLogResourceStatements(t.resourceStatements()), + ), + transformprocessor.NewTranslatorWithName( + common.DbiTransformLogs+"_"+t.cfg.engine+"_server-logs_"+idx, + transformprocessor.WithLogResourceStatements(t.logStatements("server-logs")), + // Drop the parsed attributes once promoted to the record's timestamp and + // severity fields, mirroring the files pipeline's timestamp cleanup, so they + // are not duplicated in the emitted log attributes. + transformprocessor.WithLogContextStatements([]string{ + `delete_key(attributes, "timestamp")`, + `delete_key(attributes, "severity")`, + }), + ), + ), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), @@ -215,21 +230,31 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err } // serverLogReceiver builds the engine-specific filelog receiver for server logs. -// PostgreSQL logs get multiline/timestamp/severity parsing; other engines use a -// plain filelog receiver. +// Both engines get multiline grouping plus timestamp and severity parsing, but the +// timestamp layout and severity vocabulary differ by engine (PostgreSQL stderr format +// vs MySQL classic error-log format). func (t *dbiTranslator) serverLogReceiver() common.ComponentTranslator { opts := []filelog.Option{ filelog.WithNamePrefix(t.cfg.engine), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath), + filelog.WithMultilinePattern(`^\d{4}-\d{2}-\d{2}`), } - if t.cfg.engine == common.PostgreSQLKey { + switch t.cfg.engine { + case common.PostgreSQLKey: opts = append(opts, - filelog.WithMultilinePattern(`^\d{4}-\d{2}-\d{2}`), filelog.WithTimestampFormat("%Y-%m-%d %H:%M:%S.%f %Z", "UTC"), filelog.WithSeverityPattern(buildPostgresSeverityPattern()), filelog.WithSeverityMapping(postgresLogSeverityMapping), ) + case common.MySQLKey: + // MySQL 8.0+ error log timestamps are ISO 8601 UTC, e.g. + // "2026-07-20T15:27:47.123456Z" (T separator, trailing literal Z). + opts = append(opts, + filelog.WithTimestampFormat("%Y-%m-%dT%H:%M:%S.%fZ", "UTC"), + filelog.WithSeverityPattern(buildMysqlSeverityPattern()), + filelog.WithSeverityMapping(mysqlLogSeverityMapping), + ) } return filelog.NewTranslator(opts...) } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index 106dcf29c8..3aa3c345d4 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -134,7 +134,7 @@ func TestDbiMysqlTranslate(t *testing.T) { {"metrics", dbiMetrics, "metrics/dbi_mysql_0", 3, 3, 1, 3}, {"log_to_metrics", dbiLogToMetrics, "logs/dbi_mysql_0", 1, 1, 2, 2}, {"raw_events", dbiRawEvents, "logs/dbi_mysql_rawevents_0", 1, 5, 1, 1}, - {"server_logs", dbiServerLogs, "logs/dbi_mysql_serverlogs_0", 1, 4, 1, 1}, + {"server_logs", dbiServerLogs, "logs/dbi_mysql_serverlogs_0", 1, 5, 1, 1}, } for _, tc := range tests { diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go index 5db51dd003..f6afa300f6 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go @@ -157,12 +157,12 @@ func TestNewTranslators_MySQL_MultiInstance(t *testing.T) { "collect": map[string]interface{}{ "database_insights": map[string]interface{}{ "mysql": []interface{}{ - map[string]interface{}{ + map[string]interface{}{ //nolint:gosec "endpoint": "localhost:3306", "username": "cw_monitor", "password_file": "/etc/.mysql_credentials", "instance_name": "db1", "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql1.log"}, }, - map[string]interface{}{ + map[string]interface{}{ //nolint:gosec "endpoint": "db2.example.com:3306", "username": "monitor", "password_file": "/etc/.mysql_credentials2", "instance_name": "db2", }, diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index 347b615a1b..e8cfdd106f 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -69,8 +69,7 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { } // Enable metrics that are disabled by default in the receiver but required - // by Database Insights. mysql.replica.* metrics only emit when the monitored - // instance is a replica; they are inert on standalone/primary instances. + // by Database Insights. cfg.MetricsBuilderConfig.Metrics.MysqlTableSize.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlReplicaSQLDelay.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlReplicaTimeBehindSource.Enabled = true From b3163f4f9fe4812c586801267287dc9ee1abce6a Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 23 Jul 2026 19:31:22 +0000 Subject: [PATCH 16/28] Add and enable MySQL Database Insights metrics Point the mysqlreceiver replace at the DBI metric-additions build and enable the DBI-required MySQL metrics in the translator: - mysql.deadlocks - mysql.connection.count, mysql.connection.errors, mysql.max_used_connections, mysql.commands, mysql.query.count - mysql.history_list_length, mysql.active_transactions - process.threads (DBI hostmetrics process scraper) Regenerated affected golden configs. --- go.mod | 2 +- go.sum | 54 ++++++------------- .../combined_v1_v2_ec2_config.yaml | 2 + .../combined_v1_v2_eks_config.yaml | 2 + .../opentelemetry/dbi_mysql_config_linux.yaml | 34 ++++++++---- .../dbi_postgresql_config_linux.yaml | 2 + .../opentelemetry/hostmetrics/translator.go | 3 ++ .../otel/receiver/mysql/translator.go | 8 +++ 8 files changed, 58 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 1a0b381444..43005818ef 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd // Cherry-pick of upstream Query XML support for windowseventlogreceiver https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/39055 diff --git a/go.sum b/go.sum index 30bf0a82f2..ae2f501448 100644 --- a/go.sum +++ b/go.sum @@ -83,10 +83,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR8qd/Jw1Le0NZebGBUCLbtak3bJ3z1OlqZBpw= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= @@ -168,8 +164,6 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/SAP/go-hdb v1.13.5 h1:5YhG1XKJyN6JbPsdQEspzWntRhwZXcNlyga6rDc5gdA= -github.com/SAP/go-hdb v1.13.5/go.mod h1:vwLM+4Q1JDGGEPJ2PKJstzP49/KhF+mTTCetYJBuDwQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.37.2 h1:LoBbU0yJPte0cE5TZCGdlzZRmMgMtZU/XgnUKZg9Cv4= github.com/Shopify/sarama v1.37.2/go.mod h1:Nxye/E+YPru//Bpaorfhc3JsSGYwCaDDj+R4bK52U5o= @@ -280,8 +274,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd h1:AF4G2iJwaLeBivwjWMppWYbmfObc+foQws5fetLUWxk= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e h1:Nk/ryqhal2guaLPnMhd4gqbZmQjy6OK+J1m/7snhuME= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260720200020-c7001ee3110e/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48 h1:Wo9WCHext3Ha+dDzYOI9dbxmNYLbQGwMpdHVHbK7E7I= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd h1:Q0XNkIPZEiIlknoP0X0PxoKVmlKR0itxyHQjZUkFTsg= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:DdKZOg4y/ac/f/ENvYWWg6e6r+INsAkx6DqRXzps1TU= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd h1:gguRr06gAfXYB1kxUZH7VaZobUauXRX+uNWGA0hsGx4= @@ -290,8 +284,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/windowse github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/windowseventlogreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:hfCxzEKq6utMJEq/mqehk55jpp7zCoiNs1yDp7WZ+dY= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug= github.com/antchfx/xmlquery v1.4.4 h1:mxMEkdYP3pjKSftxss4nUHfjBhnMk4imGoR96FRY2dg= @@ -343,8 +337,8 @@ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.2.0/go.mod h1:UV github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 h1:b+kcDejJrXc30zU/w8Tc9klISwaO5wh+6T0sMBdDoHM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28/go.mod h1:LnI62O9GnSv6GcuLXxOYqlq0C8EmxMcgnF6m7LdYuOY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 h1:E3Y+OfzOK1+rmRo/K2G0ml8Vs+Xqk0kOnf4nS0kUtBc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59/go.mod h1:1M4PLSBUVfBI0aP+C9XI7SM6kZPCGYyI6izWz0TGprE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= @@ -493,8 +487,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= -github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -591,6 +585,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= @@ -770,8 +766,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cadvisor v0.52.1 h1:sC8SZ6jio9ds+P2dk51bgbeYeufxo55n0X3tmrpA9as= github.com/google/cadvisor v0.52.1/go.mod h1:OAhPcx1nOm5YwMh/JhpUOMKyv1YKLRtS9KgzWPndHmA= -github.com/google/flatbuffers v24.12.23+incompatible h1:ubBKR94NR4pXUCY/MUsRVzd9umNW7ht7EG9hHfS9FX8= -github.com/google/flatbuffers v24.12.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -834,9 +830,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -1131,7 +1125,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1145,8 +1138,6 @@ github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/microsoft/go-mssqldb v1.8.0 h1:7cyZ/AT7ycDsEoWPIXibd+aVKFtteUNhDGf3aobP+tw= -github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -1550,8 +1541,6 @@ github.com/signalfx/golib/v3 v3.3.53 h1:gJx0JrrHjidpo2md+YLEN+Ws3RLTaCmKYT575ZST github.com/signalfx/golib/v3 v3.3.53/go.mod h1:lgdUNXjNnZkgEhQFSbfyek4oFUG8t1APhGaitZsSBzc= github.com/signalfx/sapm-proto v0.12.0 h1:OtOe+Jm8L61Ml8K6X8a89zc8/RlaaMRElCImeGKR/Ew= github.com/signalfx/sapm-proto v0.12.0/go.mod h1:wQEki8RNCYjkv19jw5aWDcmDMTQru0ckfUbgHI69U2E= -github.com/sijms/go-ora/v2 v2.8.24 h1:TODRWjWGwJ1VlBOhbTLat+diTYe8HXq2soJeB+HMjnw= -github.com/sijms/go-ora/v2 v2.8.24/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -1563,8 +1552,8 @@ github.com/sleepinggenius2/gosmi v0.4.4 h1:xgu+Mt7CptuB10IPt3SVXBAA9tARToT4B9xGz github.com/sleepinggenius2/gosmi v0.4.4/go.mod h1:l8OniPmd3bJzw0MXP2/qh7AhP/e+bTY2CNivIhsnDT0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/snowflakedb/gosnowflake v1.13.3 h1:udARwDZ+Eb7TnihuMno1CaNVUDbJnikWC+8p4RCJQBk= -github.com/snowflakedb/gosnowflake v1.13.3/go.mod h1:NUxNYUdyPn9sRoYB/udq/fXBXuhLS3SBTPI2/OT79uc= +github.com/snowflakedb/gosnowflake v1.6.21 h1:OEn5/P+voj3P/STW+R/gGktJlEpfP127GzrxvtAJ5II= +github.com/snowflakedb/gosnowflake v1.6.21/go.mod h1:P2fE/xiD2kQXpr48OdgnazkzPsKD6aVtnHD3WP8yD9c= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -1603,8 +1592,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.36.0 h1:YpffyLuHtdp5EUsI5mT4sRw8GZhO/5ozyDT1xWGXt00= github.com/testcontainers/testcontainers-go v0.36.0/go.mod h1:yk73GVJ0KUZIHUtFna6MO7QS144qYpoY8lEEtU9Hed0= -github.com/thda/tds v0.1.7 h1:s29kbnJK0agL3ps85A/sb9XS2uxgKF5UJ6AZjbyqXX4= -github.com/thda/tds v0.1.7/go.mod h1:isLIF1oZdXfkqVMJM8RyNrsjlHPlTKnPlnsBs7ngZcM= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1682,7 +1669,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xo/tblfmt v0.0.0-20190609041254-28c54ec42ce8/go.mod h1:3U5kKQdIhwACye7ml3acccHmjGExY9WmUGU7rnDWgv0= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1696,8 +1682,6 @@ github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJB github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= @@ -1958,7 +1942,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2123,7 +2106,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2249,7 +2231,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2302,8 +2283,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -2416,7 +2397,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -2431,8 +2411,6 @@ gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index 724f041d03..4604e5646d 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1143,6 +1143,8 @@ receivers: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 51c2c2a7bf..08f5dee126 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1676,6 +1676,8 @@ receivers: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index df327b912b..7a9f84523f 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -1008,6 +1008,8 @@ receivers: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s @@ -1022,6 +1024,8 @@ receivers: enabled: true initial_delay: 1s metrics: + mysql.active_transactions: + enabled: true mysql.buffer_pool.data_pages: aggregation_strategy: sum attributes: @@ -1055,14 +1059,16 @@ receivers: aggregation_strategy: sum attributes: - command - enabled: false + enabled: true mysql.connection.count: - enabled: false + enabled: true mysql.connection.errors: aggregation_strategy: sum attributes: - error - enabled: false + enabled: true + mysql.deadlocks: + enabled: true mysql.double_writes: aggregation_strategy: sum attributes: @@ -1073,6 +1079,8 @@ receivers: attributes: - kind enabled: true + mysql.history_list_length: + enabled: true mysql.index.io.wait.count: aggregation_strategy: sum attributes: @@ -1105,7 +1113,7 @@ receivers: - operation enabled: true mysql.max_used_connections: - enabled: false + enabled: true mysql.mysqlx_connections: aggregation_strategy: sum attributes: @@ -1141,7 +1149,7 @@ receivers: mysql.query.client.count: enabled: false mysql.query.count: - enabled: false + enabled: true mysql.query.slow.count: enabled: false mysql.replica.sql_delay: @@ -1294,6 +1302,8 @@ receivers: enabled: true initial_delay: 1s metrics: + mysql.active_transactions: + enabled: true mysql.buffer_pool.data_pages: aggregation_strategy: sum attributes: @@ -1327,14 +1337,16 @@ receivers: aggregation_strategy: sum attributes: - command - enabled: false + enabled: true mysql.connection.count: - enabled: false + enabled: true mysql.connection.errors: aggregation_strategy: sum attributes: - error - enabled: false + enabled: true + mysql.deadlocks: + enabled: true mysql.double_writes: aggregation_strategy: sum attributes: @@ -1345,6 +1357,8 @@ receivers: attributes: - kind enabled: true + mysql.history_list_length: + enabled: true mysql.index.io.wait.count: aggregation_strategy: sum attributes: @@ -1377,7 +1391,7 @@ receivers: - operation enabled: true mysql.max_used_connections: - enabled: false + enabled: true mysql.mysqlx_connections: aggregation_strategy: sum attributes: @@ -1413,7 +1427,7 @@ receivers: mysql.query.client.count: enabled: false mysql.query.count: - enabled: false + enabled: true mysql.query.slow.count: enabled: false mysql.replica.sql_delay: diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml index c8fdb915e5..6a707256ac 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml @@ -870,6 +870,8 @@ receivers: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go index 793c5b9c45..b0877c4559 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go @@ -58,6 +58,9 @@ func (t *hostMetricsTranslator) Translate(conf *confmap.Conf) (*common.Component "process.memory.utilization": map[string]any{ "enabled": true, }, + "process.threads": map[string]any{ + "enabled": true, + }, }, })) } diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index e8cfdd106f..9a4e12ea93 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -73,6 +73,14 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg.MetricsBuilderConfig.Metrics.MysqlTableSize.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlReplicaSQLDelay.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlReplicaTimeBehindSource.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlDeadlocks.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlConnectionCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlConnectionErrors.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlMaxUsedConnections.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlCommands.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQueryCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlHistoryListLength.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlActiveTransactions.Enabled = true cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true From 390ae289e21a22c3916c519dcf1634cb0ea9a203 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Fri, 24 Jul 2026 18:43:52 +0000 Subject: [PATCH 17/28] Add MySQL to combined v1/v2 EC2 and EKS configs Add MySQL database_insights configuration alongside PostgreSQL in the combined sample configs to ensure both engines are represented in multi-feature configs: - Add mysql instance config to combined_v1_v2_ec2_config.json - Add mysql instance config to combined_v1_v2_eks_config.json - Regenerate golden YAML files with all MySQL components: - Receivers: mysql/metrics_0, mysql/events_0, filelog/mysql_0 - Connectors: count/dbi_dbload_mysql, signaltometrics/dbi_topsql_mysql - Processors: filter, transform, scope processors for mysql - Pipelines: logs/dbi_mysql_*, metrics/dbi_mysql_0 - Hostmetrics: process scraper now includes mysqld.* Both combined configs now demonstrate complete dual-engine DBI setups. --- .../combined_v1_v2_ec2_config.json | 11 + .../combined_v1_v2_ec2_config.yaml | 1090 +++++++++++++++- .../combined_v1_v2_eks_config.json | 11 + .../combined_v1_v2_eks_config.yaml | 1094 ++++++++++++++++- 4 files changed, 2196 insertions(+), 10 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json index 69a53e5653..64d6f92bf5 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json @@ -141,6 +141,17 @@ "file_path": "/var/lib/pgsql/data/log/postgresql-*.log" } } + ], + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "my-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/error.log" + } + } ] }, "prometheus": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index 4604e5646d..e867d2814f 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1,4 +1,73 @@ connectors: + count/dbi_dbload_mysql: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: @@ -79,6 +148,314 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} + signaltometrics/dbi_topsql_mysql: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" signaltometrics/dbi_topsql_postgresql: logs: - attributes: @@ -549,6 +926,14 @@ processors: middleware: agenthealth/statuscode refresh_tags_interval: 0s refresh_volumes_interval: 0s + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: @@ -840,6 +1225,20 @@ processors: os.version: enabled: false timeout: 2s + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false @@ -856,6 +1255,33 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] + transform/dbi_logs_mysql_raw-events_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") + metric_statements: [] + trace_statements: [] transform/dbi_logs_postgresql_raw-events_0: error_mode: ignore flatten_data: false @@ -883,6 +1309,22 @@ processors: - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] + transform/dbi_resource_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + trace_statements: [] transform/dbi_resource_postgresql_0: error_mode: ignore flatten_data: false @@ -899,6 +1341,22 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: [] transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false @@ -1062,6 +1520,32 @@ receivers: tls: insecure: true transport: udp + filelog/mysql_0: + encoding: utf-8 + include: + - /var/log/mysql/error.log + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser + start_at: end filelog/postgresql_0: encoding: utf-8 include: @@ -1138,6 +1622,7 @@ receivers: match_type: regexp names: - post(gres|master).* + - mysqld.* metrics: process.cpu.utilization: enabled: true @@ -1148,6 +1633,562 @@ receivers: mute_process_all_errors: true processes: null timeout: 0s + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor otlp/grpc_0_0_0_0_4317: protocols: grpc: @@ -1540,6 +2581,36 @@ service: - agenthealth/opentelemetry_traces - entitystore pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - groupbyattrs/files + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - count/dbi_dbload_postgresql @@ -1599,6 +2670,17 @@ service: receivers: - otlp/grpc_127_0_0_1_4327 - otlp/http_127_0_0_1_4328 + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql metrics/dbi_postgresql_0: exporters: - forward/opentelemetry @@ -1617,13 +2699,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_disk - telegraf_mem + - telegraf_netstat - telegraf_procstat/1917393364 - - telegraf_processes - telegraf_swap + - telegraf_processes - telegraf_cpu - - telegraf_netstat + - telegraf_disk metrics/host_metrics: exporters: - forward/opentelemetry @@ -1638,8 +2720,8 @@ service: - ec2tagger - awsentity/service/telegraf receivers: - - telegraf_statsd - telegraf_socket_listener + - telegraf_statsd metrics/hostDeltaMetrics: exporters: - awscloudwatch diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json index 86ab573d2e..6b6928d20f 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json @@ -144,6 +144,17 @@ "file_path": "/var/lib/pgsql/data/log/postgresql-*.log" } } + ], + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "my-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/error.log" + } + } ] }, "prometheus": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 08f5dee126..d4c6073637 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1,4 +1,73 @@ connectors: + count/dbi_dbload_mysql: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: @@ -79,6 +148,314 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} + signaltometrics/dbi_topsql_mysql: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" signaltometrics/dbi_topsql_postgresql: logs: - attributes: @@ -587,9 +964,9 @@ processors: ec2_instance_tag_keys: - AutoScalingGroupName ec2_metadata_tags: - - InstanceType - ImageId - InstanceId + - InstanceType imds_retries: 1 middleware: agenthealth/statuscode refresh_tags_interval: 0s @@ -617,6 +994,14 @@ processors: metrics: metric: - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: @@ -1273,6 +1658,20 @@ processors: - set(resource.attributes["k8s.workload.type"], "CronJob") where resource.attributes["k8s.cronjob.name"] != nil and resource.attributes["k8s.workload.type"] == nil - set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil - set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.replicaset.name"] != nil and resource.attributes["k8s.workload.type"] == nil + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false @@ -1289,6 +1688,33 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] + transform/dbi_logs_mysql_raw-events_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") + metric_statements: [] + trace_statements: [] transform/dbi_logs_postgresql_raw-events_0: error_mode: ignore flatten_data: false @@ -1316,6 +1742,22 @@ processors: - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] + transform/dbi_resource_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + trace_statements: [] transform/dbi_resource_postgresql_0: error_mode: ignore flatten_data: false @@ -1332,6 +1774,22 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: [] transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false @@ -1595,6 +2053,32 @@ receivers: tls: insecure: true transport: udp + filelog/mysql_0: + encoding: utf-8 + include: + - /var/log/mysql/error.log + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser + start_at: end filelog/postgresql_0: encoding: utf-8 include: @@ -1671,6 +2155,7 @@ receivers: match_type: regexp names: - post(gres|master).* + - mysqld.* metrics: process.cpu.utilization: enabled: true @@ -1719,6 +2204,562 @@ receivers: enabled: true k8s.pod.uptime: enabled: true + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor otlp/grpc_0_0_0_0_4317: protocols: grpc: @@ -2227,6 +3268,36 @@ service: - entitystore - server pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - groupbyattrs/files + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - count/dbi_dbload_postgresql @@ -2455,6 +3526,17 @@ service: - awsattributelimit/cw_k8s_ci_v0 receivers: - prometheus/cw_k8s_ci_v0_node_exporter + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql metrics/dbi_postgresql_0: exporters: - forward/opentelemetry @@ -2473,13 +3555,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_swap - - telegraf_processes + - telegraf_mem + - telegraf_procstat/1917393364 - telegraf_cpu - - telegraf_netstat + - telegraf_processes - telegraf_disk - - telegraf_procstat/1917393364 - - telegraf_mem + - telegraf_netstat + - telegraf_swap metrics/host_metrics: exporters: - forward/opentelemetry From 0ea2209f972181f73d07d916f43c426e0a09a90b Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 28 Jul 2026 13:23:50 +0000 Subject: [PATCH 18/28] chore: update mysqlreceiver to latest commit (bed4560687ee) with PR #617 changelog references --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 43005818ef..ffa5f8fa64 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd // Cherry-pick of upstream Query XML support for windowseventlogreceiver https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/39055 diff --git a/go.sum b/go.sum index ae2f501448..13472f85f4 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd h1:AF4G2iJwaLeBivwjWMppWYbmfObc+foQws5fetLUWxk= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48 h1:Wo9WCHext3Ha+dDzYOI9dbxmNYLbQGwMpdHVHbK7E7I= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260723202011-09a6da50bc48/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee h1:XELS9NH+v8xvMQS49SckY5HWDfqrYNzUPQT62NT4QgY= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd h1:Q0XNkIPZEiIlknoP0X0PxoKVmlKR0itxyHQjZUkFTsg= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:DdKZOg4y/ac/f/ENvYWWg6e6r+INsAkx6DqRXzps1TU= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd h1:gguRr06gAfXYB1kxUZH7VaZobUauXRX+uNWGA0hsGx4= From f550d8d68925b3ccb2c3d4a891449d570f9b233a Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Wed, 29 Jul 2026 18:22:47 +0000 Subject: [PATCH 19/28] fix(test): set MySQL credentials file permissions in combined config tests The MySQL receiver validates that passfiles have 0600 or 0400 permissions for security. Git doesn't preserve file permissions beyond the executable bit, so checked-out files default to 0644 in CI, causing test failures. Add os.Chmod(0600) calls in TestCombinedV1V2EC2Config and TestCombinedV1V2EKSConfig before config translation to ensure the .mysql_credentials testdata file has the required restrictive permissions. This matches the existing fix in TestDBIMySQLConfigLinux and ensures tests pass reliably in CI and local development environments. --- translator/tocwconfig/tocwconfig_unix_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index 1f2230c01f..edd7f291a2 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -144,6 +144,8 @@ func TestJournaldLogsUnitsAndPriority(t *testing.T) { func TestCombinedV1V2EC2Config(t *testing.T) { resetContext(t) context.CurrentContext().SetMode(config.ModeEC2) + // Fix MySQL credentials file permissions (required by receiver validation) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) checkTranslation(t, "opentelemetry/combined_v1_v2_ec2_config", "linux", nil, "") } @@ -151,6 +153,8 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetMode(config.ModeEC2) context.CurrentContext().SetKubernetesMode(config.ModeEKS) + // Fix MySQL credentials file permissions (required by receiver validation) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) // Cannot use checkTranslation here because the container_insights prometheus // receiver references /var/run/secrets/kubernetes.io/serviceaccount/token From d8e127a34d408686962b4e503fe9993d76936b82 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Fri, 31 Jul 2026 18:35:53 +0000 Subject: [PATCH 20/28] chore(dbi): tune MySQL query sample limits Adjust MySQL query sample configuration to optimize resource usage: - Increase MaxRowsPerQuery from 500 to 5000 (aligns with PostgreSQL) - Decrease MaxQuerySampleCount from 5000 to 1000 (conservative limit) This balances query plan collection with CloudWatch Logs limits. --- translator/translate/otel/receiver/mysql/translator.go | 4 ++-- translator/translate/otel/receiver/mysql/translator_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index 9a4e12ea93..e2c573c977 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -85,11 +85,11 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true - cfg.QuerySampleCollection.MaxRowsPerQuery = 500 + cfg.QuerySampleCollection.MaxRowsPerQuery = 5000 cfg.TopQueryCollection.CollectionInterval = t.topQueryInterval cfg.TopQueryCollection.TopQueryCount = 200 - cfg.TopQueryCollection.MaxQuerySampleCount = 5000 + cfg.TopQueryCollection.MaxQuerySampleCount = 1000 cfg.TopQueryCollection.QueryPlanCacheSize = 1000 cfg.TopQueryCollection.QueryPlanCacheTTL = time.Hour diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go index 587b13f02d..b6f1d7fbec 100644 --- a/translator/translate/otel/receiver/mysql/translator_test.go +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -37,9 +37,9 @@ func TestTranslator_Translate_Localhost(t *testing.T) { assert.True(t, mysqlCfg.TLS.Insecure) assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) - assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.Equal(t, uint64(5000), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) assert.Equal(t, uint64(200), mysqlCfg.TopQueryCollection.TopQueryCount) - assert.Equal(t, uint64(5000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) + assert.Equal(t, uint64(1000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) assert.Equal(t, 1000, mysqlCfg.TopQueryCollection.QueryPlanCacheSize) assert.Equal(t, time.Hour, mysqlCfg.TopQueryCollection.QueryPlanCacheTTL) assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) From 9a48cd1bfad0aa5cee3f0fd3848137ff35baed81 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 6 Aug 2026 12:02:21 +0000 Subject: [PATCH 21/28] chore(dbi): update mysqlreceiver to latest dbi-mysql branch (83b5027c386b) Update go.mod to point to the latest commit on the upstream dbi-mysql branch which includes PR #617 (InnoDB deadlocks and transaction metrics). --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ffa5f8fa64..72b0caaa8d 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd // Cherry-pick of upstream Query XML support for windowseventlogreceiver https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/39055 diff --git a/go.sum b/go.sum index 13472f85f4..75530c8fa3 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd h1:AF4G2iJwaLeBivwjWMppWYbmfObc+foQws5fetLUWxk= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee h1:XELS9NH+v8xvMQS49SckY5HWDfqrYNzUPQT62NT4QgY= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260728131824-bed4560687ee/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b h1:AHEwT0RHOaEDz0t5WrH7OkQH93I2zoFaM90qR01oLII= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd h1:Q0XNkIPZEiIlknoP0X0PxoKVmlKR0itxyHQjZUkFTsg= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260717191209-ee093fbfffbd/go.mod h1:DdKZOg4y/ac/f/ENvYWWg6e6r+INsAkx6DqRXzps1TU= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260717191209-ee093fbfffbd h1:gguRr06gAfXYB1kxUZH7VaZobUauXRX+uNWGA0hsGx4= From 10b6295567b5047c0e842288df6eff24b5c393a1 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Mon, 10 Aug 2026 14:58:45 +0000 Subject: [PATCH 22/28] fix(dbi): Address PR review comments for MySQL translator - Change QuerySampleCollection.MaxRowsPerQuery from 5000 to 500 to match PostgreSQL - Change TopQueryCollection.MaxQuerySampleCount from 1000 to 5000 to match PostgreSQL - Make TopQueryCollection.CollectionInterval configurable via WithTopQueryInterval option - Set 60s interval for 'events' receiver - Add test for custom interval configuration --- .../opentelemetry/databaseinsights/translator.go | 8 ++++++-- translator/translate/otel/receiver/mysql/translator.go | 4 ++-- .../translate/otel/receiver/mysql/translator_test.go | 10 ++++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index 7fa6234396..7686029b63 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -263,13 +263,17 @@ func (t *dbiTranslator) serverLogReceiver() common.ComponentTranslator { // "events"; for PostgreSQL events we override the query sample interval. func (t *dbiTranslator) receiver(name string) common.ComponentTranslator { if t.cfg.engine == common.MySQLKey { - return mysql.NewTranslator( + opts := []mysql.Option{ mysql.WithName(name), mysql.WithIndex(t.instanceIndex), mysql.WithEndpoint(t.cfg.endpoint), mysql.WithUsername(t.cfg.username), mysql.WithPassfile(t.cfg.passfile), - ) + } + if name == "events" { + opts = append(opts, mysql.WithTopQueryInterval(60*time.Second)) + } + return mysql.NewTranslator(opts...) } opts := []postgresql.Option{ diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index e2c573c977..9a4e12ea93 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -85,11 +85,11 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true - cfg.QuerySampleCollection.MaxRowsPerQuery = 5000 + cfg.QuerySampleCollection.MaxRowsPerQuery = 500 cfg.TopQueryCollection.CollectionInterval = t.topQueryInterval cfg.TopQueryCollection.TopQueryCount = 200 - cfg.TopQueryCollection.MaxQuerySampleCount = 1000 + cfg.TopQueryCollection.MaxQuerySampleCount = 5000 cfg.TopQueryCollection.QueryPlanCacheSize = 1000 cfg.TopQueryCollection.QueryPlanCacheTTL = time.Hour diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go index b6f1d7fbec..f0ede71bf2 100644 --- a/translator/translate/otel/receiver/mysql/translator_test.go +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -37,15 +37,15 @@ func TestTranslator_Translate_Localhost(t *testing.T) { assert.True(t, mysqlCfg.TLS.Insecure) assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) - assert.Equal(t, uint64(5000), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) assert.Equal(t, uint64(200), mysqlCfg.TopQueryCollection.TopQueryCount) - assert.Equal(t, uint64(1000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) + assert.Equal(t, uint64(5000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) assert.Equal(t, 1000, mysqlCfg.TopQueryCollection.QueryPlanCacheSize) assert.Equal(t, time.Hour, mysqlCfg.TopQueryCollection.QueryPlanCacheTTL) assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) } -func TestTranslator_Translate_CustomInterval(t *testing.T) { +func TestTranslator_Translate_CustomTopQueryInterval(t *testing.T) { tr := NewTranslator( WithEndpoint("localhost:3306"), WithUsername("cw_monitor"), @@ -54,5 +54,7 @@ func TestTranslator_Translate_CustomInterval(t *testing.T) { ) cfg, err := tr.Translate(nil) require.NoError(t, err) - assert.Equal(t, 30*time.Second, cfg.(*mysqlreceiver.Config).TopQueryCollection.CollectionInterval) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, 30*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) } From 1aa973346893d7e86517463be303ae929a04734b Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Mon, 10 Aug 2026 15:54:24 +0000 Subject: [PATCH 23/28] fix(dbi): Add error logging for MySQL config decode failures - Log decode errors to stderr for visibility when MySQL config is malformed - Return empty slice instead of nil to allow PostgreSQL instances to still be processed - Addresses PR review feedback about error handling --- .../pipeline/opentelemetry/databaseinsights/translators.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index 7d6c9a66a6..e59263372c 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -5,6 +5,7 @@ package databaseinsights import ( "fmt" + "os" "regexp" "strings" @@ -112,7 +113,9 @@ func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { arr, _ := conf.Get(common.DatabaseInsightsMysqlKey).([]any) var raw []mysqlRawInstance if err := mapstructure.Decode(arr, &raw); err != nil { - return nil + // Log error but return empty slice to allow other database instances to be processed + fmt.Fprintf(os.Stderr, "failed to decode MySQL database_insights config: %v\n", err) + return []dbiInstanceConfig{} } instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { From 879e7cbdb9013de00c71151c39a943f58dd1a65e Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 11 Aug 2026 16:08:39 +0000 Subject: [PATCH 24/28] fix: Remove extra blank line in mysql translator_test.go Run make fmt fmt-sh to fix formatting issues detected by CI. --- translator/translate/otel/receiver/mysql/translator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go index f0ede71bf2..10abf0ec57 100644 --- a/translator/translate/otel/receiver/mysql/translator_test.go +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -55,6 +55,6 @@ func TestTranslator_Translate_CustomTopQueryInterval(t *testing.T) { cfg, err := tr.Translate(nil) require.NoError(t, err) mysqlCfg := cfg.(*mysqlreceiver.Config) - + assert.Equal(t, 30*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) } From 0409a7f26fef80fda9c79e608b441b63240a7ec7 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Thu, 13 Aug 2026 13:10:50 +0000 Subject: [PATCH 25/28] fix(dbi): Address PR review comments for MySQL translator - Add error logging to PostgreSQL config decoder (similar to MySQL) - Fix receiver() function to properly handle PostgreSQL with conditional check - Remove duplicate combined_v1_v2_ec2_config_main.yaml file - Change instance name from 'pentest-mysql' to 'mysql-instance' in test configs - Keep per-instance scope processors for consistency and future flexibility --- combined_v1_v2_ec2_config_main.yaml | 1831 ----------------- .../opentelemetry/dbi_mysql_config_linux.json | 2 +- .../opentelemetry/dbi_mysql_config_linux.yaml | 8 +- .../databaseinsights/translator.go | 28 +- .../databaseinsights/translators.go | 4 +- 5 files changed, 24 insertions(+), 1849 deletions(-) delete mode 100644 combined_v1_v2_ec2_config_main.yaml diff --git a/combined_v1_v2_ec2_config_main.yaml b/combined_v1_v2_ec2_config_main.yaml deleted file mode 100644 index e244757bc6..0000000000 --- a/combined_v1_v2_ec2_config_main.yaml +++ /dev/null @@ -1,1831 +0,0 @@ -connectors: - count/dbi_dbload: - datapoints: - metric.datapoint.count: - description: The number of data points observed. - logs: - postgresql.active_sessions.by_app: - attributes: - - key: db.system.name - - key: postgresql.application_name - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by application - postgresql.active_sessions.by_db: - attributes: - - key: db.system.name - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by database - postgresql.active_sessions.by_host: - attributes: - - key: db.system.name - - key: network.peer.address - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by host - postgresql.active_sessions.by_sql: - attributes: - - key: db.system.name - - key: postgresql.query_id - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by query - postgresql.active_sessions.by_sql_wait: - attributes: - - key: db.system.name - - key: postgresql.query_id - - key: postgresql.wait_event_type - - key: postgresql.wait_event - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by query and wait - postgresql.active_sessions.by_user: - attributes: - - key: db.system.name - - key: user.name - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by user - postgresql.active_sessions.by_wait: - attributes: - - key: db.system.name - - key: postgresql.wait_event_type - - key: postgresql.wait_event - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Active sessions by wait event - postgresql.active_sessions.count: - attributes: - - key: db.system.name - - key: db.namespace - conditions: - - attributes["postgresql.state"] == "active" - description: Total active sessions - metrics: - metric.count: - description: The number of metrics observed. - spanevents: - trace.span.event.count: - description: The number of span events observed. - spans: - trace.span.count: - description: The number of spans observed. - forward/opentelemetry: {} - signaltometrics/dbi_topsql: - logs: - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - - key: postgresql.rolname - optional: false - conditions: - - attributes["postgresql.calls"] != nil - description: SQL execution count - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.calls - sum: - value: Int(attributes["postgresql.calls"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - - key: postgresql.rolname - optional: false - conditions: - - attributes["postgresql.total_exec_time"] != nil - description: Total execution time in ms - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.total_exec_time - sum: - value: Double(attributes["postgresql.total_exec_time"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - - key: postgresql.rolname - optional: false - conditions: - - attributes["postgresql.total_plan_time"] != nil - description: Total plan time in ms - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.total_plan_time - sum: - value: Double(attributes["postgresql.total_plan_time"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - - key: postgresql.rolname - optional: false - conditions: - - attributes["postgresql.rows"] != nil - description: Rows affected - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.rows - sum: - value: Int(attributes["postgresql.rows"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - conditions: - - attributes["postgresql.shared_blks_hit"] != nil - description: Shared blocks hit - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.shared_blks_hit - sum: - value: Int(attributes["postgresql.shared_blks_hit"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - conditions: - - attributes["postgresql.shared_blks_read"] != nil - description: Shared blocks read - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.shared_blks_read - sum: - value: Int(attributes["postgresql.shared_blks_read"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - conditions: - - attributes["postgresql.local_blks_hit"] != nil - description: Local blocks hit - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.local_blks_hit - sum: - value: Int(attributes["postgresql.local_blks_hit"]) - unit: "" - - attributes: - - key: db.system.name - optional: false - - key: postgresql.queryid - optional: false - - key: db.namespace - optional: false - conditions: - - attributes["postgresql.local_blks_read"] != nil - description: Local blocks read - include_resource_attributes: - - key: service.instance.id - optional: false - name: postgresql.local_blks_read - sum: - value: Int(attributes["postgresql.local_blks_read"]) - unit: "" -exporters: - awscloudwatch: - backoff_retry_base: 200ms - endpoint_override: https://monitoring-fips.us-west-2.amazonaws.com - force_flush_interval: 1m0s - max_concurrent_publishers: 10 - max_datums_per_call: 1000 - max_retry_count: 5 - max_values_per_datum: 150 - middleware: agenthealth/metrics - namespace: CWAgent - region: us-west-2 - resource_to_telemetry_conversion: - enabled: true - role_arn: metrics_role_arn_value_test - rollup_dimensions: - - - InstanceId - - InstanceType - - [] - awscloudwatchlogs/emf_logs: - certificate_file_path: "" - emf_only: true - endpoint: https://logs-fips.us-west-2.amazonaws.com - external_id: "" - imds_retries: 1 - local_mode: false - log_group_name: emf/logs/default - log_retention: 0 - log_stream_name: LOG_STREAM_NAME - max_retries: 2 - middleware: agenthealth/logs - no_verify_ssl: false - num_workers: 8 - profile: "" - proxy_address: "" - raw_log: true - region: us-west-2 - request_timeout_seconds: 30 - resource_arn: "" - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 5m0s - max_interval: 30s - multiplier: 1.5 - randomization_factor: 0.5 - role_arn: log_role_arn_value_test - sending_queue: - block_on_overflow: false - blocking: false - enabled: true - num_consumers: 1 - queue_size: 1000 - sizer: {} - wait_for_result: false - awsxray: - certificate_file_path: "" - endpoint: https://x-ray-endpoint.us-west-2.amazonaws.com - external_id: "" - imds_retries: 1 - index_all_attributes: false - local_mode: false - max_retries: 2 - middleware: agenthealth/traces - no_verify_ssl: true - num_workers: 1 - profile: "" - proxy_address: "" - region: us-west-2 - request_timeout_seconds: 30 - resource_arn: "" - role_arn: trace_role_arn_value_test - telemetry: - enabled: true - include_metadata: true - otlphttp/logs: - auth: - authenticator: agenthealth/opentelemetry_logs - compression: gzip - encoding: proto - idle_conn_timeout: 1m30s - logs_endpoint: https://logs.us-west-2.amazonaws.com/v1/logs - max_idle_conns: 100 - metrics_endpoint: "" - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 5m0s - max_interval: 30s - multiplier: 1.5 - randomization_factor: 0.5 - sending_queue: - block_on_overflow: false - blocking: false - enabled: true - num_consumers: 10 - queue_size: 1000 - sizer: {} - wait_for_result: false - timeout: 30s - traces_endpoint: "" - write_buffer_size: 524288 - otlphttp/metrics: - auth: - authenticator: agenthealth/opentelemetry_metrics - compression: gzip - encoding: proto - idle_conn_timeout: 1m30s - logs_endpoint: "" - max_idle_conns: 100 - metrics_endpoint: https://monitoring.us-west-2.amazonaws.com/v1/metrics - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 5m0s - max_interval: 30s - multiplier: 1.5 - randomization_factor: 0.5 - sending_queue: - block_on_overflow: false - blocking: false - enabled: true - num_consumers: 10 - queue_size: 1000 - sizer: {} - wait_for_result: false - timeout: 30s - traces_endpoint: "" - write_buffer_size: 524288 - otlphttp/traces: - auth: - authenticator: agenthealth/opentelemetry_traces - compression: gzip - encoding: proto - idle_conn_timeout: 1m30s - logs_endpoint: "" - max_idle_conns: 100 - metrics_endpoint: "" - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 5m0s - max_interval: 30s - multiplier: 1.5 - randomization_factor: 0.5 - sending_queue: - block_on_overflow: false - blocking: false - enabled: true - num_consumers: 10 - queue_size: 1000 - sizer: {} - wait_for_result: false - timeout: 30s - traces_endpoint: https://xray.us-west-2.amazonaws.com/v1/traces - write_buffer_size: 524288 -extensions: - agenthealth/logs: - is_usage_data_enabled: true - stats: - operations: - - PutLogEvents - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/metrics: - is_usage_data_enabled: true - stats: - operations: - - PutMetricData - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/opentelemetry_logs: - additional_auth: headers_setter/logs - is_usage_data_enabled: true - stats: - operations: - - '*' - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/opentelemetry_metrics: - additional_auth: sigv4auth/monitoring - is_usage_data_enabled: true - stats: - operations: - - '*' - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/opentelemetry_traces: - additional_auth: sigv4auth/xray - is_usage_data_enabled: true - stats: - operations: - - '*' - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/statuscode: - is_status_code_enabled: true - is_usage_data_enabled: true - stats: - usage_flags: - mode: EC2 - region_type: ACJ - agenthealth/traces: - is_usage_data_enabled: true - stats: - operations: - - PutTraceSegments - usage_flags: - mode: EC2 - region_type: ACJ - awscloudwatchlogsprovisioner: - additional_auth: sigv4auth/logs - imds_retries: 1 - logs_provision_failure_backoff: 5s - max_retries: 2 - num_workers: 8 - region: us-west-2 - request_timeout_seconds: 10 - entitystore: - mode: ec2 - region: us-west-2 - headers_setter/logs: - additional_auth: awscloudwatchlogsprovisioner - headers: - - action: upsert - from_context: aws.log.group.name - key: x-aws-log-group - - action: upsert - from_context: aws.log.stream.name - key: x-aws-log-stream - sigv4auth/logs: - assume_role: {} - imds_retries: 1 - max_retries: 2 - num_workers: 8 - region: us-west-2 - request_timeout_seconds: 30 - service: logs - sigv4auth/monitoring: - assume_role: {} - imds_retries: 1 - max_retries: 2 - num_workers: 8 - region: us-west-2 - request_timeout_seconds: 30 - service: monitoring - sigv4auth/xray: - assume_role: {} - imds_retries: 1 - max_retries: 2 - num_workers: 8 - region: us-west-2 - request_timeout_seconds: 30 - service: xray -processors: - attributestocontext/opentelemetry: - actions: - - from_resource_attribute: aws.log.group.name - key: aws.log.group.name - - from_resource_attribute: aws.log.stream.name - key: aws.log.stream.name - awsentity/resource: - entity_type: Resource - platform: ec2 - scrape_datapoint_attribute: true - awsentity/service/telegraf: - entity_type: Service - platform: ec2 - scrape_datapoint_attribute: true - batch/emf_logs: - metadata_cardinality_limit: 1000 - send_batch_max_size: 0 - send_batch_size: 8192 - timeout: 1m0s - batch/opentelemetry_logs: - metadata_cardinality_limit: 1000 - metadata_keys: - - aws.log.group.name - - aws.log.stream.name - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 30s - batch/opentelemetry_metrics: - metadata_cardinality_limit: 1000 - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 30s - batch/opentelemetry_traces: - metadata_cardinality_limit: 1000 - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 30s - batch/xray: - metadata_cardinality_limit: 1000 - send_batch_max_size: 0 - send_batch_size: 8192 - timeout: 200ms - cumulativetodelta/hostDeltaMetrics: - exclude: - match_type: strict - metrics: - - iops_in_progress - - diskio_iops_in_progress - - diskio_ebs_volume_queue_length - include: - match_type: "" - initial_value: 2 - max_staleness: 0s - ec2tagger: - ec2_instance_tag_keys: - - AutoScalingGroupName - ec2_metadata_tags: - - ImageId - - InstanceId - - InstanceType - imds_retries: 1 - middleware: agenthealth/statuscode - refresh_tags_interval: 0s - refresh_volumes_interval: 0s - filter/dbi_exclude_monitor_0: - error_mode: propagate - logs: - log_record: - - attributes["user.name"] == "cw_monitor" or attributes["postgresql.rolname"] == "cw_monitor" - metrics: {} - spans: {} - traces: {} - groupbyattrs/files: - keys: - - log.file.name - resourcedetection/opentelemetry: - aks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: false - azure: - resource_attributes: - azure.resourcegroup.name: - enabled: true - azure.vm.name: - enabled: true - azure.vm.scaleset.name: - enabled: true - azure.vm.size: - enabled: true - cloud.account.id: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - tags: [] - consul: - address: "" - datacenter: "" - namespace: "" - resource_attributes: - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - token_file: "" - detectors: - - eks - - env - - ec2 - docker: - resource_attributes: - host.name: - enabled: true - os.type: - enabled: true - ec2: - fail_on_missing_metadata: false - max_attempts: 3 - max_backoff: 20s - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.image.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - tags: - - ^kubernetes.io/cluster/.*$ - - ^aws:autoscaling:groupName - ecs: - resource_attributes: - aws.ecs.cluster.arn: - enabled: true - aws.ecs.launchtype: - enabled: true - aws.ecs.task.arn: - enabled: true - aws.ecs.task.family: - enabled: true - aws.ecs.task.id: - enabled: true - aws.ecs.task.revision: - enabled: true - aws.log.group.arns: - enabled: true - aws.log.group.names: - enabled: true - aws.log.stream.arns: - enabled: true - aws.log.stream.names: - enabled: true - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - eks: - resource_attributes: - cloud.account.id: - enabled: false - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: false - elasticbeanstalk: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - deployment.environment: - enabled: true - service.instance.id: - enabled: true - service.version: - enabled: true - gcp: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.id: - enabled: true - faas.instance: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - gcp.cloud_run.job.execution: - enabled: true - gcp.cloud_run.job.task_index: - enabled: true - gcp.gce.instance.hostname: - enabled: false - gcp.gce.instance.name: - enabled: false - gcp.gce.instance_group_manager.name: - enabled: true - gcp.gce.instance_group_manager.region: - enabled: true - gcp.gce.instance_group_manager.zone: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - k8s.cluster.name: - enabled: true - heroku: - resource_attributes: - cloud.provider: - enabled: true - heroku.app.id: - enabled: true - heroku.dyno.id: - enabled: true - heroku.release.commit: - enabled: true - heroku.release.creation_timestamp: - enabled: true - service.instance.id: - enabled: true - service.name: - enabled: true - service.version: - enabled: true - idle_conn_timeout: 1m30s - k8snode: - auth_type: serviceAccount - context: "" - kube_config_path: "" - node_from_env_var: "" - resource_attributes: - k8s.node.name: - enabled: true - k8s.node.uid: - enabled: true - kubeadm: - auth_type: serviceAccount - context: "" - kube_config_path: "" - resource_attributes: - k8s.cluster.name: - enabled: true - k8s.cluster.uid: - enabled: true - lambda: - resource_attributes: - aws.log.group.names: - enabled: true - aws.log.stream.names: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.instance: - enabled: true - faas.max_memory: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - max_idle_conns: 100 - openshift: - address: "" - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - k8s.cluster.name: - enabled: true - tls: {} - token: "" - override: true - system: - resource_attributes: - host.arch: - enabled: false - host.cpu.cache.l2.size: - enabled: false - host.cpu.family: - enabled: false - host.cpu.model.id: - enabled: false - host.cpu.model.name: - enabled: false - host.cpu.stepping: - enabled: false - host.cpu.vendor.id: - enabled: false - host.id: - enabled: false - host.ip: - enabled: false - host.mac: - enabled: false - host.name: - enabled: true - os.description: - enabled: false - os.type: - enabled: true - os.version: - enabled: false - timeout: 2s - transform/dbi_fix_start_time: - error_mode: ignore - flatten_data: false - log_statements: [] - metric_statements: - - context: datapoint - error_mode: ignore - statements: - - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 - - replace_match(datapoint.attributes["postgresql.wait_event_type"], "", "CPU") - - replace_match(datapoint.attributes["postgresql.wait_event"], "", "CPU") - - replace_match(datapoint.attributes["postgresql.application_name"], "", "unknown") - - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") - - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - - replace_match(datapoint.attributes["user.name"], "", "unknown") - trace_statements: [] - transform/dbi_logs_raw-events_0: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/postgresql/raw-events") - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) - metric_statements: [] - trace_statements: [] - transform/dbi_logs_server-logs_0: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/postgresql/server-logs") - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) - - context: log - error_mode: ignore - statements: - - delete_key(attributes, "timestamp") - - delete_key(attributes, "severity") - metric_statements: [] - trace_statements: [] - transform/dbi_resource_0: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["db.system.name"], "postgresql") - - set(resource.attributes["db.instance.name"], "my-postgres") - metric_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["db.system.name"], "postgresql") - - set(resource.attributes["db.instance.name"], "my-postgres") - trace_statements: [] - transform/dbi_scope: - error_mode: ignore - flatten_data: false - log_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") - metric_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") - trace_statements: [] - transform/host_metrics_scope: - error_mode: ignore - flatten_data: false - log_statements: [] - metric_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-host-metrics") - trace_statements: [] - transform/identity: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil - - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil - metric_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil - - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil - trace_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil - - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil - - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil - transform/logs_cleanup: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - delete_key(resource.attributes, "aws.log.group.name") - - delete_key(resource.attributes, "aws.log.stream.name") - - delete_key(resource.attributes, "aws.log.source") - metric_statements: [] - trace_statements: [] - transform/logs_routing: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["aws.log.group.name"], Concat(["/aws/cwagent", resource.attributes["aws.log.source"]], "/")) where resource.attributes["aws.log.group.name"] == nil and resource.attributes["aws.log.source"] != nil - - set(resource.attributes["aws.log.group.name"], "/aws/cwagent/default") where resource.attributes["aws.log.group.name"] == nil - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.id"] != nil and resource.attributes["log.file.name"] != nil - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.name"] != nil and resource.attributes["log.file.name"] != nil - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" - - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.id"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil - - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.name"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil - - set(resource.attributes["aws.log.stream.name"], "default") where resource.attributes["aws.log.stream.name"] == nil - metric_statements: [] - trace_statements: [] - transform/otlp_log_source: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["aws.log.source"], "otlp") where resource.attributes["aws.log.source"] == nil - metric_statements: [] - trace_statements: [] - transform/otlp_scope: - error_mode: ignore - flatten_data: false - log_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-otlp") - metric_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-otlp") - trace_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-otlp") - transform/prometheus_scope: - error_mode: ignore - flatten_data: false - log_statements: [] - metric_statements: - - context: scope - error_mode: ignore - statements: - - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") - - set(scope.attributes["cloudwatch.solution"], "otel-prometheus") - trace_statements: [] - transform/set_cluster_name: - error_mode: ignore - flatten_data: false - log_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["k8s.cluster.name"], "TestCluster") - metric_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["k8s.cluster.name"], "TestCluster") - trace_statements: - - context: resource - error_mode: ignore - statements: - - set(resource.attributes["k8s.cluster.name"], "TestCluster") -receivers: - awsxray: - endpoint: 0.0.0.0:2000 - proxy_server: - aws_endpoint: https://x-ray-endpoint.us-west-2.amazonaws.com - certificate_file_path: "" - endpoint: 0.0.0.0:1234 - imds_retries: 1 - local_mode: false - profile: "" - proxy_address: "" - region: us-west-2 - role_arn: trace_role_arn_value_test - service_name: xray - tls: - insecure: true - transport: udp - filelog/postgresql_0: - encoding: utf-8 - include: - - /var/lib/pgsql/data/log/postgresql-*.log - multiline: - line_start_pattern: ^\d{4}-\d{2}-\d{2} - operators: - - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2} \d{2}:\d{2}:\d{2}\.(\d{1,9}) \w{3}) - timestamp: - layout: 2006-1-_2 15:04:05.999999999 MST - layout_type: gotime - location: UTC - parse_from: attributes.timestamp - type: regex_parser - - on_error: send_quiet - regex: '\[\d+\]\s*(?PLOG|ERROR|WARNING|FATAL|PANIC|DEBUG\d?|INFO|NOTICE|STATEMENT):' - severity: - mapping: - debug: - - DEBUG - - DEBUG1 - - DEBUG2 - - DEBUG3 - - DEBUG4 - - DEBUG5 - error: ERROR - fatal: - - FATAL - - PANIC - info: - - LOG - - INFO - - NOTICE - - STATEMENT - warn: WARNING - parse_from: attributes.severity - type: regex_parser - start_at: end - hostmetrics/opentelemetry: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: - metrics: - system.cpu.frequency: - enabled: true - system.cpu.logical.count: - enabled: true - system.cpu.physical.count: - enabled: true - system.cpu.utilization: - enabled: true - disk: null - filesystem: - metrics: - system.filesystem.utilization: - enabled: true - load: null - memory: - metrics: - system.linux.memory.available: - enabled: true - system.linux.memory.dirty: - enabled: true - system.memory.limit: - enabled: true - system.memory.page_size: - enabled: true - system.memory.utilization: - enabled: true - network: null - process: - include: - match_type: regexp - names: - - post(gres|master).* - metrics: - process.cpu.utilization: - enabled: true - process.memory.utilization: - enabled: true - mute_process_all_errors: true - processes: null - timeout: 0s - otlp/grpc_0_0_0_0_4317: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - keepalive: - enforcement_policy: {} - server_parameters: {} - read_buffer_size: 524288 - transport: tcp - otlp/grpc_127_0_0_1_4327: - protocols: - grpc: - endpoint: 127.0.0.1:4327 - keepalive: - enforcement_policy: {} - server_parameters: {} - read_buffer_size: 524288 - transport: tcp - otlp/http_0_0_0_0_4318: - protocols: - http: - cors: {} - endpoint: 0.0.0.0:4318 - idle_timeout: 0s - logs_url_path: /v1/logs - metrics_url_path: /v1/metrics - read_header_timeout: 0s - traces_url_path: /v1/traces - write_timeout: 0s - otlp/http_127_0_0_1_4328: - protocols: - http: - cors: {} - endpoint: 127.0.0.1:4328 - idle_timeout: 0s - logs_url_path: /v1/logs - metrics_url_path: /v1/metrics - read_header_timeout: 0s - traces_url_path: /v1/traces - write_timeout: 0s - postgresql/events_0: - collection_interval: 10s - endpoint: localhost:5432 - events: - db.server.query_sample: - enabled: true - db.server.top_query: - enabled: true - initial_delay: 1s - metrics: - postgresql.backends: - enabled: false - postgresql.bgwriter.buffers.allocated: - enabled: false - postgresql.bgwriter.buffers.writes: - enabled: false - postgresql.bgwriter.checkpoint.count: - enabled: false - postgresql.bgwriter.duration: - enabled: false - postgresql.bgwriter.maxwritten: - enabled: false - postgresql.blks_hit: - enabled: false - postgresql.blks_read: - enabled: false - postgresql.blocks_read: - enabled: false - postgresql.commits: - enabled: false - postgresql.connection.max: - enabled: false - postgresql.database.count: - enabled: false - postgresql.database.locks: - enabled: false - postgresql.db_size: - enabled: false - postgresql.deadlocks: - enabled: false - postgresql.function.calls: - enabled: false - postgresql.index.scans: - enabled: false - postgresql.index.size: - enabled: false - postgresql.operations: - enabled: false - postgresql.replication.data_delay: - enabled: false - postgresql.rollbacks: - enabled: false - postgresql.rows: - enabled: false - postgresql.sequential_scans: - enabled: false - postgresql.sessions: - enabled: false - postgresql.table.count: - enabled: false - postgresql.table.size: - enabled: false - postgresql.table.vacuum.count: - enabled: false - postgresql.temp.io: - enabled: false - postgresql.temp_files: - enabled: false - postgresql.tup_deleted: - enabled: false - postgresql.tup_fetched: - enabled: false - postgresql.tup_inserted: - enabled: false - postgresql.tup_returned: - enabled: false - postgresql.tup_updated: - enabled: false - postgresql.wal.age: - enabled: false - postgresql.wal.delay: - enabled: false - postgresql.wal.lag: - enabled: false - passfile: sampleConfig/opentelemetry/testdata/.pgpass - query_sample_collection: - collection_interval: 1m0s - enabled: true - max_rows_per_query: 500 - resource_attributes: - postgresql.database.name: - enabled: true - postgresql.index.name: - enabled: true - postgresql.schema.name: - enabled: true - postgresql.table.name: - enabled: true - service.instance.id: - enabled: true - timeout: 0s - tls: - insecure: true - insecure_skip_verify: true - top_query_collection: - collection_interval: 1m0s - max_explain_each_interval: 1000 - max_rows_per_query: 5000 - query_plan_cache_size: 1000 - query_plan_cache_ttl: 1h0m0s - top_n_query: 200 - transport: tcp - username: cw_monitor - postgresql/metrics_0: - collection_interval: 10s - endpoint: localhost:5432 - events: - db.server.query_sample: - enabled: true - db.server.top_query: - enabled: true - initial_delay: 1s - metrics: - postgresql.backends: - enabled: true - postgresql.bgwriter.buffers.allocated: - enabled: true - postgresql.bgwriter.buffers.writes: - enabled: true - postgresql.bgwriter.checkpoint.count: - enabled: true - postgresql.bgwriter.duration: - enabled: true - postgresql.bgwriter.maxwritten: - enabled: true - postgresql.blks_hit: - enabled: false - postgresql.blks_read: - enabled: false - postgresql.blocks_read: - enabled: false - postgresql.commits: - enabled: true - postgresql.connection.max: - enabled: true - postgresql.database.count: - enabled: true - postgresql.database.locks: - enabled: false - postgresql.db_size: - enabled: true - postgresql.deadlocks: - enabled: false - postgresql.function.calls: - enabled: false - postgresql.index.scans: - enabled: false - postgresql.index.size: - enabled: false - postgresql.operations: - enabled: false - postgresql.replication.data_delay: - enabled: true - postgresql.rollbacks: - enabled: true - postgresql.rows: - enabled: false - postgresql.sequential_scans: - enabled: false - postgresql.sessions: - enabled: false - postgresql.table.count: - enabled: true - postgresql.table.size: - enabled: false - postgresql.table.vacuum.count: - enabled: false - postgresql.temp.io: - enabled: false - postgresql.temp_files: - enabled: false - postgresql.tup_deleted: - enabled: false - postgresql.tup_fetched: - enabled: false - postgresql.tup_inserted: - enabled: false - postgresql.tup_returned: - enabled: false - postgresql.tup_updated: - enabled: false - postgresql.wal.age: - enabled: true - postgresql.wal.delay: - enabled: false - postgresql.wal.lag: - enabled: true - passfile: sampleConfig/opentelemetry/testdata/.pgpass - query_sample_collection: - collection_interval: 1s - enabled: true - max_rows_per_query: 500 - resource_attributes: - postgresql.database.name: - enabled: true - postgresql.index.name: - enabled: true - postgresql.schema.name: - enabled: true - postgresql.table.name: - enabled: true - service.instance.id: - enabled: true - timeout: 0s - tls: - insecure: true - insecure_skip_verify: true - top_query_collection: - collection_interval: 1m0s - max_explain_each_interval: 1000 - max_rows_per_query: 5000 - query_plan_cache_size: 1000 - query_plan_cache_ttl: 1h0m0s - top_n_query: 200 - transport: tcp - username: cw_monitor - postgresql/metrics_perresource_0: - collection_interval: 1m0s - endpoint: localhost:5432 - events: - db.server.query_sample: - enabled: false - db.server.top_query: - enabled: false - initial_delay: 1s - metrics: - postgresql.backends: - enabled: false - postgresql.bgwriter.buffers.allocated: - enabled: false - postgresql.bgwriter.buffers.writes: - enabled: false - postgresql.bgwriter.checkpoint.count: - enabled: false - postgresql.bgwriter.duration: - enabled: false - postgresql.bgwriter.maxwritten: - enabled: false - postgresql.blks_hit: - enabled: false - postgresql.blks_read: - enabled: false - postgresql.blocks_read: - enabled: true - postgresql.commits: - enabled: false - postgresql.connection.max: - enabled: false - postgresql.database.count: - enabled: false - postgresql.database.locks: - enabled: false - postgresql.db_size: - enabled: false - postgresql.deadlocks: - enabled: false - postgresql.function.calls: - enabled: false - postgresql.index.scans: - enabled: true - postgresql.index.size: - enabled: true - postgresql.operations: - enabled: true - postgresql.replication.data_delay: - enabled: false - postgresql.rollbacks: - enabled: false - postgresql.rows: - enabled: true - postgresql.sequential_scans: - enabled: false - postgresql.sessions: - enabled: false - postgresql.table.count: - enabled: false - postgresql.table.size: - enabled: true - postgresql.table.vacuum.count: - enabled: true - postgresql.temp.io: - enabled: false - postgresql.temp_files: - enabled: false - postgresql.tup_deleted: - enabled: false - postgresql.tup_fetched: - enabled: false - postgresql.tup_inserted: - enabled: false - postgresql.tup_returned: - enabled: false - postgresql.tup_updated: - enabled: false - postgresql.wal.age: - enabled: false - postgresql.wal.delay: - enabled: false - postgresql.wal.lag: - enabled: false - passfile: sampleConfig/opentelemetry/testdata/.pgpass - query_sample_collection: - collection_interval: 1s - enabled: true - max_rows_per_query: 500 - resource_attributes: - postgresql.database.name: - enabled: true - postgresql.index.name: - enabled: true - postgresql.schema.name: - enabled: true - postgresql.table.name: - enabled: true - service.instance.id: - enabled: true - timeout: 0s - tls: - insecure: true - insecure_skip_verify: true - top_query_collection: - collection_interval: 1m0s - max_explain_each_interval: 1000 - max_rows_per_query: 5000 - query_plan_cache_size: 1000 - query_plan_cache_ttl: 1h0m0s - top_n_query: 200 - transport: tcp - username: cw_monitor - prometheus/opentelemetry: - config: - global: - evaluation_interval: 1m - scrape_interval: 30s - scrape_protocols: - - OpenMetricsText1.0.0 - - OpenMetricsText0.0.1 - - PrometheusText1.0.0 - - PrometheusText0.0.4 - scrape_timeout: 10s - scrape_configs: - - enable_compression: true - enable_http2: true - fallback_scrape_protocol: PrometheusText0.0.4 - follow_redirects: true - honor_timestamps: true - job_name: test_app - metrics_path: /metrics - relabel_configs: - - action: replace - regex: (.*) - replacement: $$$$1 - separator: ; - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - regex: (.*) - replacement: $$$${1} - separator: ; - source_labels: - - __address__ - target_label: host - scheme: http - scrape_interval: 30s - scrape_protocols: - - OpenMetricsText1.0.0 - - OpenMetricsText0.0.1 - - PrometheusText1.0.0 - - PrometheusText0.0.4 - scrape_timeout: 10s - static_configs: - - targets: - - localhost:9090 - track_timestamps_staleness: false - report_extra_scrape_metrics: false - start_time_metric_regex: "" - trim_metric_suffixes: false - use_start_time_metric: false - telegraf_cpu: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_disk: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_diskio: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_mem: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_net: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_netstat: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_processes: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_procstat/1917393364: - alias_name: /var/run/example1.pid - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_socket_listener: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - telegraf_statsd: - collection_interval: 10s - initial_delay: 1s - timeout: 0s - telegraf_swap: - collection_interval: 1m0s - initial_delay: 1s - timeout: 0s - udplog/emf_logs: - encoding: utf-8 - id: udp_input - listen_address: 127.0.0.1:25888 - multiline: - line_end_pattern: .^ - line_start_pattern: "" - omit_pattern: false - operators: [] - retry_on_failure: - enabled: false - initial_interval: 0s - max_elapsed_time: 0s - max_interval: 0s - type: udp_input -service: - extensions: - - agenthealth/metrics - - agenthealth/statuscode - - agenthealth/logs - - agenthealth/traces - - sigv4auth/monitoring - - agenthealth/opentelemetry_metrics - - sigv4auth/logs - - awscloudwatchlogsprovisioner - - headers_setter/logs - - agenthealth/opentelemetry_logs - - sigv4auth/xray - - agenthealth/opentelemetry_traces - - entitystore - pipelines: - logs/dbi_postgresql_0: - exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql - processors: - - filter/dbi_exclude_monitor_0 - receivers: - - postgresql/metrics_0 - logs/dbi_postgresql_rawevents_0: - exporters: - - forward/opentelemetry - processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope - - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 - receivers: - - postgresql/events_0 - logs/dbi_postgresql_serverlogs_0: - exporters: - - forward/opentelemetry - processors: - - transform/dbi_scope - - resourcedetection/opentelemetry - - groupbyattrs/files - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 - receivers: - - filelog/postgresql_0 - logs/emf_logs: - exporters: - - awscloudwatchlogs/emf_logs - processors: - - batch/emf_logs - receivers: - - udplog/emf_logs - logs/opentelemetry: - exporters: - - otlphttp/logs - processors: - - resourcedetection/opentelemetry - - transform/set_cluster_name - - transform/identity - - transform/logs_routing - - attributestocontext/opentelemetry - - transform/logs_cleanup - - batch/opentelemetry_logs - receivers: - - forward/opentelemetry - logs/otlp: - exporters: - - forward/opentelemetry - processors: - - transform/otlp_scope - - transform/otlp_log_source - receivers: - - otlp/grpc_127_0_0_1_4327 - - otlp/http_127_0_0_1_4328 - metrics/dbi_postgresql_0: - exporters: - - forward/opentelemetry - processors: - - transform/dbi_scope - - transform/dbi_resource_0 - - transform/dbi_fix_start_time - receivers: - - postgresql/metrics_0 - - postgresql/metrics_perresource_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql - metrics/host: - exporters: - - awscloudwatch - processors: - - ec2tagger - - awsentity/resource - receivers: - - telegraf_processes - - telegraf_mem - - telegraf_cpu - - telegraf_netstat - - telegraf_swap - - telegraf_disk - - telegraf_procstat/1917393364 - metrics/host_metrics: - exporters: - - forward/opentelemetry - processors: - - transform/host_metrics_scope - receivers: - - hostmetrics/opentelemetry - metrics/hostCustomMetrics: - exporters: - - awscloudwatch - processors: - - ec2tagger - - awsentity/service/telegraf - receivers: - - telegraf_statsd - - telegraf_socket_listener - metrics/hostDeltaMetrics: - exporters: - - awscloudwatch - processors: - - cumulativetodelta/hostDeltaMetrics - - ec2tagger - - awsentity/resource - receivers: - - telegraf_diskio - - telegraf_net - metrics/opentelemetry: - exporters: - - otlphttp/metrics - processors: - - resourcedetection/opentelemetry - - transform/set_cluster_name - - transform/identity - - batch/opentelemetry_metrics - receivers: - - forward/opentelemetry - metrics/otel_prometheus: - exporters: - - forward/opentelemetry - processors: - - transform/prometheus_scope - receivers: - - prometheus/opentelemetry - metrics/otlp: - exporters: - - forward/opentelemetry - processors: - - transform/otlp_scope - receivers: - - otlp/grpc_127_0_0_1_4327 - - otlp/http_127_0_0_1_4328 - traces/opentelemetry: - exporters: - - otlphttp/traces - processors: - - resourcedetection/opentelemetry - - transform/set_cluster_name - - transform/identity - - batch/opentelemetry_traces - receivers: - - forward/opentelemetry - traces/otlp: - exporters: - - forward/opentelemetry - processors: - - transform/otlp_scope - receivers: - - otlp/grpc_127_0_0_1_4327 - - otlp/http_127_0_0_1_4328 - traces/xray: - exporters: - - awsxray - processors: - - batch/xray - receivers: - - awsxray - - otlp/grpc_0_0_0_0_4317 - - otlp/http_0_0_0_0_4318 - telemetry: - logs: - encoding: console - level: debug - output_paths: - - /tmp/fake/log/hotdog.log - sampling: - enabled: true - initial: 2 - thereafter: 500 - tick: 10s - metrics: - level: None - traces: - level: None diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json index 5aca9f4f90..d9bfb800df 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json @@ -8,7 +8,7 @@ "mysql": [ { "endpoint": "localhost:3306", - "instance_name": "pentest-mysql", + "instance_name": "mysql-instance", "username": "cw_monitor", "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", "logs": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index 7a9f84523f..f302a4b3ef 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -817,7 +817,7 @@ processors: error_mode: ignore statements: - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "mysql-instance"], "/")) metric_statements: [] trace_statements: [] transform/dbi_logs_mysql_server-logs_0: @@ -828,7 +828,7 @@ processors: error_mode: ignore statements: - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") - - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "mysql-instance"], "/")) - context: log error_mode: ignore statements: @@ -844,13 +844,13 @@ processors: error_mode: ignore statements: - set(resource.attributes["db.system.name"], "mysql") - - set(resource.attributes["db.instance.name"], "pentest-mysql") + - set(resource.attributes["db.instance.name"], "mysql-instance") metric_statements: - context: resource error_mode: ignore statements: - set(resource.attributes["db.system.name"], "mysql") - - set(resource.attributes["db.instance.name"], "pentest-mysql") + - set(resource.attributes["db.instance.name"], "mysql-instance") trace_statements: [] transform/dbi_scope_mysql_0: error_mode: ignore diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index c557674ed3..7cb5c9d299 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -318,19 +318,23 @@ func (t *dbiTranslator) receiver(name string) common.ComponentTranslator { return mysql.NewTranslator(opts...) } - opts := []postgresql.Option{ - postgresql.WithName(name), - postgresql.WithIndex(t.instanceIndex), - postgresql.WithEndpoint(t.cfg.endpoint), - postgresql.WithUsername(t.cfg.username), - postgresql.WithPassfile(t.cfg.passfile), - postgresql.WithCAFile(t.cfg.caFile), - postgresql.WithIsLocalhost(t.cfg.isLocalhost), - } - if name == "events" { - opts = append(opts, postgresql.WithQuerySampleInterval(60*time.Second)) + if t.cfg.engine == common.PostgreSQLKey { + opts := []postgresql.Option{ + postgresql.WithName(name), + postgresql.WithIndex(t.instanceIndex), + postgresql.WithEndpoint(t.cfg.endpoint), + postgresql.WithUsername(t.cfg.username), + postgresql.WithPassfile(t.cfg.passfile), + postgresql.WithCAFile(t.cfg.caFile), + postgresql.WithIsLocalhost(t.cfg.isLocalhost), + } + if name == "events" { + opts = append(opts, postgresql.WithQuerySampleInterval(60*time.Second)) + } + return postgresql.NewTranslator(opts...) } - return postgresql.NewTranslator(opts...) + + return nil } // pgReceiver builds a PostgreSQL receiver with custom options (for split metrics collection). diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index e5a251038b..549cb8051e 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -74,7 +74,9 @@ func parseDbiPostgresqlInstances(conf *confmap.Conf) []dbiInstanceConfig { arr, _ := conf.Get(common.DatabaseInsightsPostgresKey).([]any) var raw []pgRawInstance if err := mapstructure.Decode(arr, &raw); err != nil { - return nil + // Log error but return empty slice to allow other database instances to be processed + fmt.Fprintf(os.Stderr, "failed to decode PostgreSQL database_insights config: %v\n", err) + return []dbiInstanceConfig{} } instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { From 3e0f9f490e66d8f07b2592b4a0830f5fb402c0e0 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Mon, 17 Aug 2026 21:52:43 +0000 Subject: [PATCH 26/28] fix(databaseinsights): Use per-instance connectors to prevent metric fan-out The DBI count and signaltometrics connectors were created with engine-scoped IDs (count/dbi_dbload_), so all instances of an engine shared a single connector whose output was consumed by every per-instance metrics pipeline. Each pipeline unconditionally set db.instance.name, cross-tagging active_sessions/top-query datapoints with every configured instance name (a cartesian product across instances) for mysql and postgresql. Make the connector IDs per-instance (append the instance index) in both translateMetrics and translateLogToMetrics so each instance's stream is isolated end to end. --- .../combined_v1_v2_ec2_config.yaml | 24 +++++++++---------- .../combined_v1_v2_eks_config.yaml | 24 +++++++++---------- .../opentelemetry/dbi_mysql_config_linux.yaml | 12 +++++----- .../dbi_postgresql_config_linux.yaml | 12 +++++----- .../databaseinsights/translator.go | 9 +++---- .../databaseinsights/translator_test.go | 8 +++---- 6 files changed, 45 insertions(+), 44 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index 1a08276d5c..c3ebb9fc88 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload_mysql: + count/dbi_dbload_mysql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -68,7 +68,7 @@ connectors: spans: trace.span.count: description: The number of spans observed. - count/dbi_dbload_postgresql: + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -148,7 +148,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql_mysql: + signaltometrics/dbi_topsql_mysql_0: logs: - attributes: - key: db.system.name @@ -456,7 +456,7 @@ connectors: sum: value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) unit: "" - signaltometrics/dbi_topsql_postgresql: + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -2696,8 +2696,8 @@ service: pipelines: logs/dbi_mysql_0: exporters: - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 processors: - filter/dbi_exclude_monitor_mysql_0 receivers: @@ -2726,8 +2726,8 @@ service: - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - filter/dbi_exclude_monitor_postgresql_0 receivers: @@ -2792,8 +2792,8 @@ service: - transform/dbi_fix_start_time_mysql receivers: - mysql/metrics_0 - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 metrics/dbi_postgresql_0: exporters: - forward/opentelemetry @@ -2805,8 +2805,8 @@ service: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host: exporters: - awscloudwatch diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 7180e056cc..58645d1786 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload_mysql: + count/dbi_dbload_mysql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -68,7 +68,7 @@ connectors: spans: trace.span.count: description: The number of spans observed. - count/dbi_dbload_postgresql: + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -148,7 +148,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql_mysql: + signaltometrics/dbi_topsql_mysql_0: logs: - attributes: - key: db.system.name @@ -456,7 +456,7 @@ connectors: sum: value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) unit: "" - signaltometrics/dbi_topsql_postgresql: + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -3392,8 +3392,8 @@ service: pipelines: logs/dbi_mysql_0: exporters: - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 processors: - filter/dbi_exclude_monitor_mysql_0 receivers: @@ -3422,8 +3422,8 @@ service: - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - filter/dbi_exclude_monitor_postgresql_0 receivers: @@ -3657,8 +3657,8 @@ service: - transform/dbi_fix_start_time_mysql receivers: - mysql/metrics_0 - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 metrics/dbi_postgresql_0: exporters: - forward/opentelemetry @@ -3669,8 +3669,8 @@ service: receivers: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host: exporters: - awscloudwatch diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index f302a4b3ef..caa3094538 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload_mysql: + count/dbi_dbload_mysql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -69,7 +69,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql_mysql: + signaltometrics/dbi_topsql_mysql_0: logs: - attributes: - key: db.system.name @@ -1580,8 +1580,8 @@ service: pipelines: logs/dbi_mysql_0: exporters: - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 processors: - filter/dbi_exclude_monitor_mysql_0 receivers: @@ -1629,8 +1629,8 @@ service: - transform/dbi_fix_start_time_mysql receivers: - mysql/metrics_0 - - count/dbi_dbload_mysql - - signaltometrics/dbi_topsql_mysql + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml index 155e7654af..225e8f956a 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload_postgresql: + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql_postgresql: + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -1225,8 +1225,8 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - filter/dbi_exclude_monitor_postgresql_0 receivers: @@ -1275,8 +1275,8 @@ service: receivers: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload_postgresql - - signaltometrics/dbi_topsql_postgresql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index 7cb5c9d299..250a02c361 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -134,8 +134,8 @@ func (t *dbiTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) { idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) var receivers []common.ComponentTranslator if t.cfg.engine == common.PostgreSQLKey { @@ -171,8 +171,9 @@ func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) } func (t *dbiTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { - countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) + idx := strconv.Itoa(t.instanceIndex) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) var receiver common.ComponentTranslator if t.cfg.engine == common.PostgreSQLKey { diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index 5c46241e72..ed5e6a8fb3 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -105,10 +105,10 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { connectors = append(connectors, c.ID().String()) }) - assert.ElementsMatch(t, []string{"postgresql/metrics_0", "postgresql/metrics_perresource_0", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, receivers) + assert.ElementsMatch(t, []string{"postgresql/metrics_0", "postgresql/metrics_perresource_0", "count/dbi_dbload_postgresql_0", "signaltometrics/dbi_topsql_postgresql_0"}, receivers) assert.Equal(t, []string{"transform/dbi_scope_postgresql_0", "transform/dbi_resource_postgresql_0", "transform/dbi_fix_start_time_postgresql"}, processors) assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) - assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, connectors) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_postgresql_0", "signaltometrics/dbi_topsql_postgresql_0"}, connectors) } func TestDbiMysqlTranslate(t *testing.T) { @@ -175,10 +175,10 @@ func TestDbiMysqlTranslateMetrics_ComponentIDs(t *testing.T) { connectors = append(connectors, c.ID().String()) }) - assert.ElementsMatch(t, []string{"mysql/metrics_0", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, receivers) + assert.ElementsMatch(t, []string{"mysql/metrics_0", "count/dbi_dbload_mysql_0", "signaltometrics/dbi_topsql_mysql_0"}, receivers) assert.Equal(t, []string{"transform/dbi_scope_mysql_0", "transform/dbi_resource_mysql_0", "transform/dbi_fix_start_time_mysql"}, processors) assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) - assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, connectors) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_mysql_0", "signaltometrics/dbi_topsql_mysql_0"}, connectors) } func TestDbiTranslateServerLogs_FilelogConfig(t *testing.T) { From 9e67c9071ce676d0a3324d90c8f87c0a57ac50e7 Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Tue, 18 Aug 2026 13:06:50 +0000 Subject: [PATCH 27/28] Add explicit MySQL filter handling in excludeMonitorFilter - Add MySQL-specific case in excludeMonitorFilter for consistency with PostgreSQL pattern - MySQL only checks user.name attribute (no separate role attribute like PostgreSQL) - Add test case TestDbiExcludeMonitorMySQL to verify MySQL filter condition - Filter condition: attributes["user.name"] == configured_username - This ensures MySQL monitoring sessions are filtered from DB Load metrics --- .../opentelemetry/databaseinsights/translator.go | 10 ++++++++-- .../processor/filterprocessor/translator_test.go | 13 +++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index 250a02c361..b743ec169a 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -355,9 +355,15 @@ func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) func (t *dbiTranslator) excludeMonitorFilter() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - condition := fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) - if t.cfg.engine == common.PostgreSQLKey { + var condition string + switch t.cfg.engine { + case common.PostgreSQLKey: condition = fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) + case common.MySQLKey: + // For MySQL, only check user.name (MySQL receiver doesn't emit a separate role attribute like PostgreSQL) + condition = fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + default: + condition = fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) } return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+t.cfg.engine+"_"+idx, condition, common.OTTLErrorModePropagate) } diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index 15347f98a4..c88c8c37a3 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -171,3 +171,16 @@ func TestDbiExcludeMonitor(t *testing.T) { require.Len(t, fCfg.Logs.LogConditions, 1) assert.Equal(t, condition, fCfg.Logs.LogConditions[0]) } + +func TestDbiExcludeMonitorMySQL(t *testing.T) { + condition := `attributes["user.name"] == "cw_monitor"` + tr := NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_mysql_0", condition, "propagate") + assert.Equal(t, "filter/dbi_exclude_monitor_mysql_0", tr.ID().String()) + + cfg, err := tr.Translate(nil) + require.NoError(t, err) + fCfg := cfg.(*filterprocessor.Config) + require.NotNil(t, fCfg.Logs) + require.Len(t, fCfg.Logs.LogConditions, 1) + assert.Equal(t, condition, fCfg.Logs.LogConditions[0]) +} From e76e0ebdf26b77066d8e57ec78ef10f64b43b53c Mon Sep 17 00:00:00 2001 From: Kirsty Larsen Date: Wed, 19 Aug 2026 15:03:29 +0000 Subject: [PATCH 28/28] Enable mysql.query.client.count and mysql.query.slow.count metrics These metrics are disabled by default in the upstream MySQL receiver but are needed for database insights. This change: - Enables both metrics in the MySQL translator - Updates golden YAML files (dbi_mysql_config_linux.yaml, combined_v1_v2_ec2_config.yaml, combined_v1_v2_eks_config.yaml) - Both metrics_0 and events_0 receivers now have these metrics enabled --- .../opentelemetry/combined_v1_v2_ec2_config.yaml | 8 ++++---- .../opentelemetry/combined_v1_v2_eks_config.yaml | 8 ++++---- .../opentelemetry/dbi_mysql_config_linux.yaml | 8 ++++---- translator/translate/otel/receiver/mysql/translator.go | 2 ++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index c3ebb9fc88..cc0e4b0dab 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1767,11 +1767,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: @@ -2045,11 +2045,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 58645d1786..1734c90ae2 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -2347,11 +2347,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: @@ -2625,11 +2625,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml index caa3094538..3f28fa4234 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -1147,11 +1147,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: @@ -1425,11 +1425,11 @@ receivers: - command enabled: true mysql.query.client.count: - enabled: false + enabled: true mysql.query.count: enabled: true mysql.query.slow.count: - enabled: false + enabled: true mysql.replica.sql_delay: enabled: true mysql.replica.time_behind_source: diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go index 9a4e12ea93..1c8a5df4ed 100644 --- a/translator/translate/otel/receiver/mysql/translator.go +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -81,6 +81,8 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg.MetricsBuilderConfig.Metrics.MysqlQueryCount.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlHistoryListLength.Enabled = true cfg.MetricsBuilderConfig.Metrics.MysqlActiveTransactions.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQueryClientCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQuerySlowCount.Enabled = true cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true