diff --git a/go.mod b/go.mod index 62c6f632c3c..31f407791c9 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260806162807-5cc6f180ecb6 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260806162807-5cc6f180ecb6 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260806162807-5cc6f180ecb6 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260806162807-5cc6f180ecb6 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260806162807-5cc6f180ecb6 // Cherry-pick of upstream Query XML support for windowseventlogreceiver https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/39055 @@ -201,6 +202,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.124.1 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver v0.124.1 @@ -284,6 +286,7 @@ require ( ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.64.3 // indirect github.com/DataDog/datadog-go/v5 v5.6.0 // indirect github.com/DataDog/go-sqllexer v0.1.3 // indirect @@ -603,7 +606,7 @@ require ( go.opentelemetry.io/collector/component/componentstatus v0.124.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.30.0 go.opentelemetry.io/collector/config/configgrpc v0.124.0 // indirect - go.opentelemetry.io/collector/config/confignet v1.30.0 // indirect + go.opentelemetry.io/collector/config/confignet v1.30.0 go.opentelemetry.io/collector/config/configretry v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/httpprovider v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.30.0 // indirect @@ -671,7 +674,7 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect k8s.io/kubelet v0.32.3 // indirect diff --git a/go.sum b/go.sum index 335421134e9..0fa8d41b15a 100644 --- a/go.sum +++ b/go.sum @@ -83,10 +83,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR8qd/Jw1Le0NZebGBUCLbtak3bJ3z1OlqZBpw= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= @@ -168,8 +164,6 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/SAP/go-hdb v1.13.5 h1:5YhG1XKJyN6JbPsdQEspzWntRhwZXcNlyga6rDc5gdA= -github.com/SAP/go-hdb v1.13.5/go.mod h1:vwLM+4Q1JDGGEPJ2PKJstzP49/KhF+mTTCetYJBuDwQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.37.2 h1:LoBbU0yJPte0cE5TZCGdlzZRmMgMtZU/XgnUKZg9Cv4= github.com/Shopify/sarama v1.37.2/go.mod h1:Nxye/E+YPru//Bpaorfhc3JsSGYwCaDDj+R4bK52U5o= @@ -280,6 +274,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260806162807-5cc6f180ecb6/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260806162807-5cc6f180ecb6 h1:UfP+JRi9iR/Wc54kvcz75yoiaDWlZzBqgvW8Ay9X97M= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260806162807-5cc6f180ecb6/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b h1:AHEwT0RHOaEDz0t5WrH7OkQH93I2zoFaM90qR01oLII= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260730161852-83b5027c386b/go.mod h1:NAyuB/xYR3FsPuegcRvVeStWW83y5oUQhekzT856urI= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260806162807-5cc6f180ecb6 h1:wmrPq+IvRyYUov4jMlqrEi+NFcv9ZfDa37hhjUrjXoo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260806162807-5cc6f180ecb6/go.mod h1:DdKZOg4y/ac/f/ENvYWWg6e6r+INsAkx6DqRXzps1TU= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260806162807-5cc6f180ecb6 h1:98qCHXkAmin7jg0Re84so9qurK+sfWDtY9jWzPErfRQ= @@ -288,8 +284,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/windowse github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/windowseventlogreceiver v0.0.0-20260806162807-5cc6f180ecb6/go.mod h1:hfCxzEKq6utMJEq/mqehk55jpp7zCoiNs1yDp7WZ+dY= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug= github.com/antchfx/xmlquery v1.4.4 h1:mxMEkdYP3pjKSftxss4nUHfjBhnMk4imGoR96FRY2dg= @@ -341,8 +337,8 @@ github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.2.0/go.mod h1:UV github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28 h1:b+kcDejJrXc30zU/w8Tc9klISwaO5wh+6T0sMBdDoHM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.28/go.mod h1:LnI62O9GnSv6GcuLXxOYqlq0C8EmxMcgnF6m7LdYuOY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59 h1:E3Y+OfzOK1+rmRo/K2G0ml8Vs+Xqk0kOnf4nS0kUtBc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.59/go.mod h1:1M4PLSBUVfBI0aP+C9XI7SM6kZPCGYyI6izWz0TGprE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28 h1:Xf2j7NdVcUKomlZ4iihOP4AZ3Fzlr8h4yKpXeP+OFPg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.28/go.mod h1:O8cDo1dW63jU7ki//kRe1z+tLGcpnD1jrouitsQddDw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.28 h1:KqIfN9kpkKkcBqBbNpNGTIrXO6ExTUvFKvXkC+YAzVo= @@ -491,8 +487,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= -github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -589,6 +585,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= @@ -768,8 +766,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cadvisor v0.52.1 h1:sC8SZ6jio9ds+P2dk51bgbeYeufxo55n0X3tmrpA9as= github.com/google/cadvisor v0.52.1/go.mod h1:OAhPcx1nOm5YwMh/JhpUOMKyv1YKLRtS9KgzWPndHmA= -github.com/google/flatbuffers v24.12.23+incompatible h1:ubBKR94NR4pXUCY/MUsRVzd9umNW7ht7EG9hHfS9FX8= -github.com/google/flatbuffers v24.12.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -832,9 +830,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -1129,7 +1125,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1143,8 +1138,6 @@ github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/microsoft/go-mssqldb v1.8.0 h1:7cyZ/AT7ycDsEoWPIXibd+aVKFtteUNhDGf3aobP+tw= -github.com/microsoft/go-mssqldb v1.8.0/go.mod h1:6znkekS3T2vp0waiMhen4GPU1BiAsrP+iXHcE7a7rFo= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -1548,8 +1541,6 @@ github.com/signalfx/golib/v3 v3.3.53 h1:gJx0JrrHjidpo2md+YLEN+Ws3RLTaCmKYT575ZST github.com/signalfx/golib/v3 v3.3.53/go.mod h1:lgdUNXjNnZkgEhQFSbfyek4oFUG8t1APhGaitZsSBzc= github.com/signalfx/sapm-proto v0.12.0 h1:OtOe+Jm8L61Ml8K6X8a89zc8/RlaaMRElCImeGKR/Ew= github.com/signalfx/sapm-proto v0.12.0/go.mod h1:wQEki8RNCYjkv19jw5aWDcmDMTQru0ckfUbgHI69U2E= -github.com/sijms/go-ora/v2 v2.8.24 h1:TODRWjWGwJ1VlBOhbTLat+diTYe8HXq2soJeB+HMjnw= -github.com/sijms/go-ora/v2 v2.8.24/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -1561,8 +1552,8 @@ github.com/sleepinggenius2/gosmi v0.4.4 h1:xgu+Mt7CptuB10IPt3SVXBAA9tARToT4B9xGz github.com/sleepinggenius2/gosmi v0.4.4/go.mod h1:l8OniPmd3bJzw0MXP2/qh7AhP/e+bTY2CNivIhsnDT0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/snowflakedb/gosnowflake v1.13.3 h1:udARwDZ+Eb7TnihuMno1CaNVUDbJnikWC+8p4RCJQBk= -github.com/snowflakedb/gosnowflake v1.13.3/go.mod h1:NUxNYUdyPn9sRoYB/udq/fXBXuhLS3SBTPI2/OT79uc= +github.com/snowflakedb/gosnowflake v1.6.21 h1:OEn5/P+voj3P/STW+R/gGktJlEpfP127GzrxvtAJ5II= +github.com/snowflakedb/gosnowflake v1.6.21/go.mod h1:P2fE/xiD2kQXpr48OdgnazkzPsKD6aVtnHD3WP8yD9c= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -1601,8 +1592,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.36.0 h1:YpffyLuHtdp5EUsI5mT4sRw8GZhO/5ozyDT1xWGXt00= github.com/testcontainers/testcontainers-go v0.36.0/go.mod h1:yk73GVJ0KUZIHUtFna6MO7QS144qYpoY8lEEtU9Hed0= -github.com/thda/tds v0.1.7 h1:s29kbnJK0agL3ps85A/sb9XS2uxgKF5UJ6AZjbyqXX4= -github.com/thda/tds v0.1.7/go.mod h1:isLIF1oZdXfkqVMJM8RyNrsjlHPlTKnPlnsBs7ngZcM= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1680,7 +1669,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xo/tblfmt v0.0.0-20190609041254-28c54ec42ce8/go.mod h1:3U5kKQdIhwACye7ml3acccHmjGExY9WmUGU7rnDWgv0= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1694,8 +1682,6 @@ github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJB github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= @@ -1956,7 +1942,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2121,7 +2106,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2247,7 +2231,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2300,8 +2283,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= -golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -2414,7 +2397,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -2429,8 +2411,8 @@ gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index a6b51b313b4..c5f324597b8 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -55,6 +55,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver" @@ -119,6 +120,7 @@ func Factories() (otelcol.Factories, error) { nopreceiver.NewFactory(), otlpreceiver.NewFactory(), prometheusreceiver.NewFactory(), + mysqlreceiver.NewFactory(), postgresqlreceiver.NewFactory(), statsdreceiver.NewFactory(), systemmetricsreceiver.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index ed45b7e22bf..6cefe44c516 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -35,6 +35,7 @@ func TestComponents(t *testing.T) { "nop", "otlp", "prometheus", + "mysql", "postgresql", "statsd", "systemmetrics", diff --git a/translator/config/schema.json b/translator/config/schema.json index 7f03e697262..17ecdc7c2c9 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -1852,6 +1852,27 @@ "required": ["endpoint", "instance_name", "username", "password_file"], "additionalProperties": false } + }, + "mysql": { + "type": "array", + "items": { + "type": "object", + "properties": { + "endpoint": { "type": "string" }, + "instance_name": { "type": "string" }, + "username": { "type": "string" }, + "password_file": { "type": "string" }, + "logs": { + "type": "object", + "properties": { + "file_path": { "type": "string" } + }, + "additionalProperties": false + } + }, + "required": ["endpoint", "instance_name", "username", "password_file"], + "additionalProperties": false + } } }, "additionalProperties": false diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json index 69a53e56534..64d6f92bf52 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.json @@ -141,6 +141,17 @@ "file_path": "/var/lib/pgsql/data/log/postgresql-*.log" } } + ], + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "my-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/error.log" + } + } ] }, "prometheus": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index e244757bc6e..cc0e4b0dabd 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1,5 +1,74 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_mysql_0: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +148,315 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_mysql_0: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -549,7 +926,15 @@ processors: middleware: agenthealth/statuscode refresh_tags_interval: 0s refresh_volumes_interval: 0s - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -840,7 +1225,21 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -856,7 +1255,34 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_mysql_raw-events_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") + metric_statements: [] + trace_statements: [] + transform/dbi_logs_postgresql_raw-events_0: error_mode: ignore flatten_data: false log_statements: @@ -867,7 +1293,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: ignore flatten_data: false log_statements: @@ -883,7 +1309,23 @@ processors: - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + trace_statements: [] + transform/dbi_resource_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -899,7 +1341,23 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope: + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: [] + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1062,6 +1520,32 @@ receivers: tls: insecure: true transport: udp + filelog/mysql_0: + encoding: utf-8 + include: + - /var/log/mysql/error.log + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser + start_at: end filelog/postgresql_0: encoding: utf-8 include: @@ -1138,14 +1622,573 @@ receivers: match_type: regexp names: - post(gres|master).* + - mysqld.* metrics: process.cpu.utilization: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor otlp/grpc_0_0_0_0_4317: protocols: grpc: @@ -1651,34 +2694,64 @@ service: - agenthealth/opentelemetry_traces - entitystore pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - groupbyattrs/files + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - groupbyattrs/files - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -1710,18 +2783,30 @@ service: receivers: - otlp/grpc_127_0_0_1_4327 - otlp/http_127_0_0_1_4328 + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 metrics/dbi_postgresql_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host: exporters: - awscloudwatch diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json index 86ab573d2e5..6b6928d20fb 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.json @@ -144,6 +144,17 @@ "file_path": "/var/lib/pgsql/data/log/postgresql-*.log" } } + ], + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "my-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/error.log" + } + } ] }, "prometheus": { diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index e03417b96c0..1734c90ae2b 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1,5 +1,74 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_mysql_0: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +148,315 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_mysql_0: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -617,7 +994,15 @@ processors: metrics: metric: - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -1273,7 +1658,21 @@ processors: - set(resource.attributes["k8s.workload.type"], "CronJob") where resource.attributes["k8s.cronjob.name"] != nil and resource.attributes["k8s.workload.type"] == nil - set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil - set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.replicaset.name"] != nil and resource.attributes["k8s.workload.type"] == nil - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -1289,7 +1688,34 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_mysql_raw-events_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-mysql"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") + metric_statements: [] + trace_statements: [] + transform/dbi_logs_postgresql_raw-events_0: error_mode: ignore flatten_data: false log_statements: @@ -1300,7 +1726,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: ignore flatten_data: false log_statements: @@ -1316,7 +1742,23 @@ processors: - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "my-mysql") + trace_statements: [] + transform/dbi_resource_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1332,7 +1774,23 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope: + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: [] + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1604,6 +2062,32 @@ receivers: tls: insecure: true transport: udp + filelog/mysql_0: + encoding: utf-8 + include: + - /var/log/mysql/error.log + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser + start_at: end filelog/postgresql_0: encoding: utf-8 include: @@ -1680,11 +2164,14 @@ receivers: match_type: regexp names: - post(gres|master).* + - mysqld.* metrics: process.cpu.utilization: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s @@ -1726,6 +2213,562 @@ receivers: enabled: true k8s.pod.uptime: enabled: true + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor otlp/grpc_0_0_0_0_4317: protocols: grpc: @@ -2347,34 +3390,64 @@ service: - entitystore - server pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - groupbyattrs/files + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - groupbyattrs/files - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -2575,18 +3648,29 @@ service: - awsattributelimit/cw_k8s_ci_v0 receivers: - prometheus/cw_k8s_ci_v0_node_exporter + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 metrics/dbi_postgresql_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host: exporters: - awscloudwatch @@ -2594,13 +3678,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_netstat - - telegraf_cpu - - telegraf_processes - - telegraf_swap - telegraf_mem - telegraf_procstat/1917393364 + - telegraf_cpu + - telegraf_processes - telegraf_disk + - telegraf_netstat + - telegraf_swap metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json new file mode 100644 index 00000000000..d9bfb800df4 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json @@ -0,0 +1,22 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "collect": { + "database_insights": { + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "mysql-instance", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/mysql.log" + } + } + ] + } + } + } +} diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml new file mode 100644 index 00000000000..3f28fa42345 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -0,0 +1,1664 @@ +connectors: + count/dbi_dbload_mysql_0: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.session.status"] != "idle" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. + forward/opentelemetry: {} + signaltometrics/dbi_topsql_mysql_0: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + description: Rows affected + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_affected + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + description: Joins checking key usage per row + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_range_check + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + description: Sort merge passes + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_merge_passes + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + description: Sorts using ranges + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_range + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"]) + unit: "" +exporters: + otlphttp/logs: + auth: + authenticator: agenthealth/opentelemetry_logs + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: https://logs.us-west-2.amazonaws.com/v1/logs + max_idle_conns: 100 + metrics_endpoint: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 + otlphttp/metrics: + auth: + authenticator: agenthealth/opentelemetry_metrics + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: "" + max_idle_conns: 100 + metrics_endpoint: https://monitoring.us-west-2.amazonaws.com/v1/metrics + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 +extensions: + agenthealth/opentelemetry_logs: + additional_auth: headers_setter/logs + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + agenthealth/opentelemetry_metrics: + additional_auth: sigv4auth/monitoring + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + awscloudwatchlogsprovisioner: + additional_auth: sigv4auth/logs + imds_retries: 1 + logs_provision_failure_backoff: 5s + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 10 + headers_setter/logs: + additional_auth: awscloudwatchlogsprovisioner + headers: + - action: upsert + from_context: aws.log.group.name + key: x-aws-log-group + - action: upsert + from_context: aws.log.stream.name + key: x-aws-log-stream + sigv4auth/logs: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: logs + sigv4auth/monitoring: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: monitoring +processors: + attributestocontext/opentelemetry: + actions: + - from_resource_attribute: aws.log.group.name + key: aws.log.group.name + - from_resource_attribute: aws.log.stream.name + key: aws.log.stream.name + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 30s + batch/opentelemetry_metrics: + metadata_cardinality_limit: 1000 + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 30s + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} + groupbyattrs/files: + keys: + - log.file.name + resourcedetection/opentelemetry: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + tags: [] + consul: + address: "" + datacenter: "" + namespace: "" + resource_attributes: + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + fail_on_missing_metadata: false + max_attempts: 3 + max_backoff: 20s + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + - ^aws:autoscaling:groupName + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.id: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.account.id: + enabled: false + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.instance: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + gcp.gce.instance_group_manager.name: + enabled: true + gcp.gce.instance_group_manager.region: + enabled: true + gcp.gce.instance_group_manager.zone: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + k8snode: + auth_type: serviceAccount + context: "" + kube_config_path: "" + node_from_env_var: "" + resource_attributes: + k8s.node.name: + enabled: true + k8s.node.uid: + enabled: true + kubeadm: + auth_type: serviceAccount + context: "" + kube_config_path: "" + resource_attributes: + k8s.cluster.name: + enabled: true + k8s.cluster.uid: + enabled: true + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_idle_conns: 100 + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: {} + token: "" + override: true + system: + resource_attributes: + host.arch: + enabled: false + host.cpu.cache.l2.size: + enabled: false + host.cpu.family: + enabled: false + host.cpu.model.id: + enabled: false + host.cpu.model.name: + enabled: false + host.cpu.stepping: + enabled: false + host.cpu.vendor.id: + enabled: false + host.id: + enabled: false + host.ip: + enabled: false + host.mac: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + os.version: + enabled: false + timeout: 2s + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] + transform/dbi_logs_mysql_raw-events_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "mysql-instance"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "mysql-instance"], "/")) + - context: log + error_mode: ignore + statements: + - delete_key(attributes, "timestamp") + - delete_key(attributes, "severity") + metric_statements: [] + trace_statements: [] + transform/dbi_resource_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "mysql-instance") + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "mysql-instance") + trace_statements: [] + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: [] + transform/host_metrics_scope: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(scope.attributes["cloudwatch.source"], "cloudwatch-agent") + - set(scope.attributes["cloudwatch.solution"], "otel-host-metrics") + trace_statements: [] + transform/identity: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + trace_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil and resource.attributes["cloud.platform"] == "aws_ec2" + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + transform/logs_cleanup: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - delete_key(resource.attributes, "aws.log.group.name") + - delete_key(resource.attributes, "aws.log.stream.name") + - delete_key(resource.attributes, "aws.log.source") + metric_statements: [] + trace_statements: [] + transform/logs_routing: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], Concat(["/aws/cwagent", resource.attributes["aws.log.source"]], "/")) where resource.attributes["aws.log.group.name"] == nil and resource.attributes["aws.log.source"] != nil + - set(resource.attributes["aws.log.group.name"], "/aws/cwagent/default") where resource.attributes["aws.log.group.name"] == nil + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.id"] != nil and resource.attributes["log.file.name"] != nil + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["log.file.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["aws.log.source"] == "files" and resource.attributes["host.name"] != nil and resource.attributes["log.file.name"] != nil + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.id"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.name"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil + - set(resource.attributes["aws.log.stream.name"], "default") where resource.attributes["aws.log.stream.name"] == nil + metric_statements: [] + trace_statements: [] +receivers: + filelog/mysql_0: + encoding: utf-8 + include: + - /var/log/mysql/mysql.log + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - regex: (?P\d{4}-\s{0,1}\d{1,2}-\s{0,1}\d{1,2}T\d{2}:\d{2}:\d{2}\.(\d{1,9})Z) + timestamp: + layout: 2006-1-_2T15:04:05.999999999Z + layout_type: gotime + location: UTC + parse_from: attributes.timestamp + type: regex_parser + - on_error: send_quiet + regex: \s\d+\s+\[(?PSystem|Warning|Note|ERROR)\] + severity: + mapping: + error: ERROR + info: + - System + - Note + warn: Warning + parse_from: attributes.severity + type: regex_parser + start_at: end + hostmetrics/opentelemetry: + collection_interval: 30s + initial_delay: 1s + scrapers: + cpu: + metrics: + system.cpu.frequency: + enabled: true + system.cpu.logical.count: + enabled: true + system.cpu.physical.count: + enabled: true + system.cpu.utilization: + enabled: true + disk: null + filesystem: + metrics: + system.filesystem.utilization: + enabled: true + load: null + memory: + metrics: + system.linux.memory.available: + enabled: true + system.linux.memory.dirty: + enabled: true + system.memory.limit: + enabled: true + system.memory.page_size: + enabled: true + system.memory.utilization: + enabled: true + network: null + process: + include: + match_type: regexp + names: + - mysqld.* + metrics: + process.cpu.utilization: + enabled: true + process.memory.utilization: + enabled: true + process.threads: + enabled: true + mute_process_all_errors: true + processes: null + timeout: 0s + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.active_transactions: + enabled: true + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.connection.count: + enabled: true + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: true + mysql.deadlocks: + enabled: true + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.history_list_length: + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: true + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: true + mysql.query.count: + enabled: true + mysql.query.slow.count: + enabled: true + mysql.replica.sql_delay: + enabled: true + mysql.replica.time_behind_source: + enabled: true + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: true + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor +service: + extensions: + - sigv4auth/monitoring + - agenthealth/opentelemetry_metrics + - sigv4auth/logs + - awscloudwatchlogsprovisioner + - headers_setter/logs + - agenthealth/opentelemetry_logs + pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - groupbyattrs/files + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 + logs/opentelemetry: + exporters: + - otlphttp/logs + processors: + - resourcedetection/opentelemetry + - transform/identity + - transform/logs_routing + - attributestocontext/opentelemetry + - transform/logs_cleanup + - batch/opentelemetry_logs + receivers: + - forward/opentelemetry + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql_0 + - signaltometrics/dbi_topsql_mysql_0 + metrics/host_metrics: + exporters: + - forward/opentelemetry + processors: + - transform/host_metrics_scope + receivers: + - hostmetrics/opentelemetry + metrics/opentelemetry: + exporters: + - otlphttp/metrics + processors: + - resourcedetection/opentelemetry + - transform/identity + - batch/opentelemetry_metrics + receivers: + - forward/opentelemetry + telemetry: + logs: + encoding: console + level: info + output_paths: + - /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + level: None + traces: + level: None diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf new file mode 100644 index 00000000000..9d2168ce61a --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf @@ -0,0 +1,15 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml similarity index 97% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml index 549b2c4d296..225e8f956aa 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql_0: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql_0: logs: - attributes: - key: db.system.name @@ -352,7 +352,7 @@ processors: send_batch_max_size: 1000 send_batch_size: 1000 timeout: 30s - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -643,7 +643,7 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -659,7 +659,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: ignore flatten_data: false log_statements: @@ -670,7 +670,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: ignore flatten_data: false log_statements: @@ -686,7 +686,7 @@ processors: - delete_key(attributes, "severity") metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -702,7 +702,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "pentest-postgres") trace_statements: [] - transform/dbi_scope: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -870,6 +870,8 @@ receivers: enabled: true process.memory.utilization: enabled: true + process.threads: + enabled: true mute_process_all_errors: true processes: null timeout: 0s @@ -1223,32 +1225,32 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - groupbyattrs/files - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/opentelemetry: @@ -1267,14 +1269,14 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - postgresql/metrics_perresource_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql_0 + - signaltometrics/dbi_topsql_postgresql_0 metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials new file mode 100644 index 00000000000..63df2f15ccd --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials @@ -0,0 +1,5 @@ +[client] +host=localhost +port=3306 +user=cw_monitor +password=test_password diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index 17e5dea224d..5addd833962 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -108,7 +108,7 @@ func TestDBIConfigLinux(t *testing.T) { context.CurrentContext().SetMode(config.ModeEC2) require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.pgpass", 0600)) expectedEnvVars := map[string]string{} - checkTranslation(t, "opentelemetry/dbi_config_linux", "linux", expectedEnvVars, "") + checkTranslation(t, "opentelemetry/dbi_postgresql_config_linux", "linux", expectedEnvVars, "") } func TestJournaldLogsUnits(t *testing.T) { @@ -144,6 +144,8 @@ func TestJournaldLogsUnitsAndPriority(t *testing.T) { func TestCombinedV1V2EC2Config(t *testing.T) { resetContext(t) context.CurrentContext().SetMode(config.ModeEC2) + // Fix MySQL credentials file permissions (required by receiver validation) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) checkTranslation(t, "opentelemetry/combined_v1_v2_ec2_config", "linux", nil, "") } @@ -151,6 +153,8 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetMode(config.ModeEC2) context.CurrentContext().SetKubernetesMode(config.ModeEKS) + // Fix MySQL credentials file permissions (required by receiver validation) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) // Cannot use checkTranslation here because the container_insights prometheus // receiver references /var/run/secrets/kubernetes.io/serviceaccount/token @@ -188,6 +192,14 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { assert.Empty(t, cmp.Diff(expected, actual, opt)) } +func TestDBIMySQLConfigLinux(t *testing.T) { + resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) + expectedEnvVars := map[string]string{} + checkTranslation(t, "opentelemetry/dbi_mysql_config_linux", "linux", expectedEnvVars, "") +} + func TestDefaultOtelConfigAzureVMTranslation(t *testing.T) { resetContext(t) context.CurrentContext().SetMode(config.ModeAzureVM) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 9fd1dae4776..6a741c0dc06 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -147,8 +147,9 @@ const FilesKey = "files" const ( DatabaseInsightsKey = "database_insights" PostgreSQLKey = "postgresql" + MySQLKey = "mysql" - // DBI component names + // DBI component name prefixes (shared across engines; index-suffixed per instance) DbiTransformFixStartTime = "dbi_fix_start_time" DbiTransformResource = "dbi_resource" DbiTransformLogs = "dbi_logs" @@ -166,6 +167,7 @@ const ( var ( DatabaseInsightsConfigKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey) DatabaseInsightsPostgresKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, PostgreSQLKey) + DatabaseInsightsMysqlKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, MySQLKey) OtelCollectLogsConfigKey = ConfigKey(OpenTelemetryKey, CollectKey, LogsKey) OtelSpanMetricsEnabledKey = ConfigKey(OpenTelemetryKey, CollectKey, OtlpKey, "span_metrics_enabled") WindowsEventsConfigKey = ConfigKey(OpenTelemetryKey, CollectKey, WindowsEventsKey) diff --git a/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml new file mode 100644 index 00000000000..5a181412f90 --- /dev/null +++ b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml @@ -0,0 +1,56 @@ +logs: + mysql.active_sessions.by_wait: + description: "Active sessions by wait event" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_user: + description: "Active sessions by user" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + mysql.active_sessions.by_db: + description: "Active sessions by database" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: db.namespace + mysql.active_sessions.by_sql: + description: "Active sessions by query digest" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + mysql.active_sessions.by_sql_wait: + description: "Active sessions by query and wait" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_host: + description: "Active sessions by host" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + mysql.active_sessions.count: + description: "Total active sessions" + conditions: + - attributes["mysql.session.status"] != "idle" + attributes: + - key: db.system.name + - key: db.namespace diff --git a/translator/translate/otel/connector/count/dbi_dbload.yaml b/translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/count/dbi_dbload.yaml rename to translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml diff --git a/translator/translate/otel/connector/count/translator.go b/translator/translate/otel/connector/count/translator.go index ad803f43865..1ab0e54c0e4 100644 --- a/translator/translate/otel/connector/count/translator.go +++ b/translator/translate/otel/connector/count/translator.go @@ -15,20 +15,26 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_dbload.yaml +//go:embed dbi_dbload_postgresql.yaml var dbiDbloadConfig string +//go:embed dbi_dbload_mysql.yaml +var dbiDbloadMysqlConfig string + type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a count connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a count connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: countconnector.NewFactory(), } } @@ -40,9 +46,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*countconnector.Config) - if t.name == common.DbiConnectorDbload { + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiDbloadConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, dbiDbloadMysqlConfig) } - return nil, fmt.Errorf("unsupported count connector config: %s", t.name) + return nil, fmt.Errorf("unsupported count connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/count/translator_test.go b/translator/translate/otel/connector/count/translator_test.go index 0721d3de662..275af49dd61 100644 --- a/translator/translate/otel/connector/count/translator_test.go +++ b/translator/translate/otel/connector/count/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) - assert.Equal(t, "count/dbi_dbload", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "count/dbi_dbload_postgresql", tr.ID().String()) } func TestTranslateDbload(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,8 +36,28 @@ func TestTranslateDbload(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_dbload_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported count connector config") + assert.Contains(t, err.Error(), "unsupported count connector engine") +} + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) + assert.Equal(t, "count/dbi_dbload_mysql", tr.ID().String()) +} + +func TestTranslateDbloadMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + countCfg := cfg.(*countconnector.Config) + assert.Len(t, countCfg.Logs, 7) + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_user") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_db") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_host") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.count") } diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml new file mode 100644 index 00000000000..8cba45ec24d --- /dev/null +++ b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml @@ -0,0 +1,217 @@ +logs: + - name: mysql.count_star + include_resource_attributes: + - key: service.instance.id + description: "SQL execution count" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.count_star"])' + - name: mysql.sum_timer_wait + include_resource_attributes: + - key: service.instance.id + description: "Total execution time in picoseconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"])' + - name: mysql.sum_lock_time + include_resource_attributes: + - key: service.instance.id + description: "Total lock time in seconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"])' + - name: mysql.sum_rows_sent + include_resource_attributes: + - key: service.instance.id + description: "Rows sent to client" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"])' + - name: mysql.sum_rows_examined + include_resource_attributes: + - key: service.instance.id + description: "Rows examined" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"])' + - name: mysql.sum_errors + include_resource_attributes: + - key: service.instance.id + description: "Total errors" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"])' + - name: mysql.sum_sort_rows + include_resource_attributes: + - key: service.instance.id + description: "Rows sorted" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"])' + - name: mysql.sum_created_tmp_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"])' + - name: mysql.sum_created_tmp_disk_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary disk tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"])' + - name: mysql.sum_no_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"])' + - name: mysql.sum_select_full_join + include_resource_attributes: + - key: service.instance.id + description: "Full joins without index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"])' + - name: mysql.sum_sort_scan + include_resource_attributes: + - key: service.instance.id + description: "Sorts done via full scan" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"])' + - name: mysql.sum_no_good_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no good index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"])' + - name: mysql.sum_select_scan + include_resource_attributes: + - key: service.instance.id + description: "Full table scans" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"])' + - name: mysql.sum_rows_affected + include_resource_attributes: + - key: service.instance.id + description: "Rows affected" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_affected"])' + - name: mysql.sum_select_range_check + include_resource_attributes: + - key: service.instance.id + description: "Joins checking key usage per row" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_range_check"])' + - name: mysql.sum_sort_merge_passes + include_resource_attributes: + - key: service.instance.id + description: "Sort merge passes" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_merge_passes"])' + - name: mysql.sum_sort_range + include_resource_attributes: + - key: service.instance.id + description: "Sorts using ranges" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_range"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_range"])' diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml rename to translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml diff --git a/translator/translate/otel/connector/signaltometrics/translator.go b/translator/translate/otel/connector/signaltometrics/translator.go index 313b2f8f71e..a26fbe4870f 100644 --- a/translator/translate/otel/connector/signaltometrics/translator.go +++ b/translator/translate/otel/connector/signaltometrics/translator.go @@ -16,20 +16,26 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_topsql.yaml +//go:embed dbi_topsql_postgresql.yaml var dbiTopsqlConfig string +//go:embed dbi_topsql_mysql.yaml +var dbiTopsqlMysqlConfig string + type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a signaltometrics connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a signaltometrics connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: signaltometricsconnector.NewFactory(), } } @@ -41,9 +47,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*signaltometricsconfig.Config) - if t.name == common.DbiConnectorTopsql { + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlMysqlConfig) } - return nil, fmt.Errorf("unsupported signaltometrics connector config: %s", t.name) + return nil, fmt.Errorf("unsupported signaltometrics connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index bcb37f40802..6515b90bba6 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) - assert.Equal(t, "signaltometrics/dbi_topsql", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "signaltometrics/dbi_topsql_postgresql", tr.ID().String()) } func TestTranslateTopsql(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,8 +36,39 @@ func TestTranslateTopsql(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_topsql_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported signaltometrics connector config") + assert.Contains(t, err.Error(), "unsupported signaltometrics connector engine") +} + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) + assert.Equal(t, "signaltometrics/dbi_topsql_mysql", tr.ID().String()) +} + +func TestTranslateTopsqlMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + stmCfg := cfg.(*signaltometricsconfig.Config) + assert.Len(t, stmCfg.Logs, 18) + assert.Equal(t, "mysql.count_star", stmCfg.Logs[0].Name) + assert.Equal(t, "mysql.sum_timer_wait", stmCfg.Logs[1].Name) + assert.Equal(t, "mysql.sum_lock_time", stmCfg.Logs[2].Name) + assert.Equal(t, "mysql.sum_rows_sent", stmCfg.Logs[3].Name) + assert.Equal(t, "mysql.sum_rows_examined", stmCfg.Logs[4].Name) + assert.Equal(t, "mysql.sum_errors", stmCfg.Logs[5].Name) + assert.Equal(t, "mysql.sum_sort_rows", stmCfg.Logs[6].Name) + assert.Equal(t, "mysql.sum_created_tmp_tables", stmCfg.Logs[7].Name) + assert.Equal(t, "mysql.sum_created_tmp_disk_tables", stmCfg.Logs[8].Name) + assert.Equal(t, "mysql.sum_no_index_used", stmCfg.Logs[9].Name) + assert.Equal(t, "mysql.sum_select_full_join", stmCfg.Logs[10].Name) + assert.Equal(t, "mysql.sum_sort_scan", stmCfg.Logs[11].Name) + assert.Equal(t, "mysql.sum_no_good_index_used", stmCfg.Logs[12].Name) + assert.Equal(t, "mysql.sum_select_scan", stmCfg.Logs[13].Name) + assert.Equal(t, "mysql.sum_rows_affected", stmCfg.Logs[14].Name) + assert.Equal(t, "mysql.sum_select_range_check", stmCfg.Logs[15].Name) + assert.Equal(t, "mysql.sum_sort_merge_passes", stmCfg.Logs[16].Name) + assert.Equal(t, "mysql.sum_sort_range", stmCfg.Logs[17].Name) } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index 3008c45728c..b743ec169a0 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -22,6 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/filelog" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/mysql" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/postgresql" ) @@ -59,6 +60,34 @@ func buildPostgresSeverityPattern() string { return `\[\d+\]\s*(?P` + strings.Join(postgresLogSeverityLevels, "|") + `):` } +// mysqlLogSeverityLevels are the severity labels MySQL emits in the classic error log +// (per MySQL worklog #10942): Note/Warning/ERROR, plus System for force-printed events. +// Kept alongside the mapping below so the regex and mapping stay together. +var mysqlLogSeverityLevels = []string{"System", "Warning", "Note", "ERROR"} + +// mysqlLogSeverityMapping maps MySQL error-log labels to OTEL severity levels for the +// filelog severity operator. Kept here (not in the generic filelog translator) because +// these labels are MySQL-specific. System and Note are informational; there is no +// MySQL debug/fatal label in the classic error log. +var mysqlLogSeverityMapping = map[string]any{ + "info": []string{"System", "Note"}, + "warn": "Warning", + "error": "ERROR", +} + +// buildMysqlSeverityPattern builds the regex that extracts the severity from a MySQL +// error-log line of the form " [] [MY-######] ...". +// Anchoring on the numeric thread id before the bracket avoids matching the later +// error-code bracket (e.g. "[MY-010116]"). The required (?P...) named capture +// group is assembled from mysqlLogSeverityLevels. +func buildMysqlSeverityPattern() string { + return `\s\d+\s+\[(?P` + strings.Join(mysqlLogSeverityLevels, "|") + `)\]` +} + +// dbiTranslator generates DBI pipelines for a single database instance. The +// engine (carried on cfg) selects engine-specific receivers, connector configs, +// resource attributes, and log group paths. Component IDs are index-based so +// PostgreSQL and MySQL instances share one consistent naming scheme. type dbiTranslator struct { pipelineType dbiPipelineType instanceIndex int @@ -71,13 +100,13 @@ func (t *dbiTranslator) ID() pipeline.ID { idx := strconv.Itoa(t.instanceIndex) switch t.pipelineType { case dbiMetrics: - return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_"+t.cfg.engine+"_"+idx) case dbiLogToMetrics: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_"+idx) case dbiRawEvents: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_rawevents_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_rawevents_"+idx) case dbiServerLogs: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_serverlogs_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_serverlogs_"+idx) } return pipeline.NewID(pipeline.SignalMetrics) } @@ -105,29 +134,36 @@ func (t *dbiTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) { idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) - - // Split postgresql metric collection into two receiver instances so per-resource - // (table/index) metrics are scraped at a configurable interval (default 60s) while - // server metrics stay at the default 10s. This caps per-resource ingestion volume - // for DBI GA pricing. The per-resource interval can be overridden via - // per_resource_collection_interval in the agent JSON config. The server instance - // keeps the events (query-sample / top-query) that feed DBLoad and TopSQL; the - // per-resource instance disables events so those are not double-counted. - serverRx := t.pgReceiver("metrics", postgresql.WithServerMetricsOnly()) - perResourceRx := t.pgReceiver("metrics_perresource", - postgresql.WithPerResourceMetricsOnly(), - postgresql.WithCollectionInterval(t.cfg.perResourceCollectionInterval), - postgresql.WithEventsDisabled(), - ) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + + var receivers []common.ComponentTranslator + if t.cfg.engine == common.PostgreSQLKey { + // Split postgresql metric collection into two receiver instances so per-resource + // (table/index) metrics are scraped at a configurable interval (default 60s) while + // server metrics stay at the default 10s. This caps per-resource ingestion volume + // for DBI GA pricing. The per-resource interval can be overridden via + // per_resource_collection_interval in the agent JSON config. The server instance + // keeps the events (query-sample / top-query) that feed DBLoad and TopSQL; the + // per-resource instance disables events so those are not double-counted. + serverRx := t.pgReceiver("metrics", postgresql.WithServerMetricsOnly()) + perResourceRx := t.pgReceiver("metrics_perresource", + postgresql.WithPerResourceMetricsOnly(), + postgresql.WithCollectionInterval(t.cfg.perResourceCollectionInterval), + postgresql.WithEventsDisabled(), + ) + receivers = []common.ComponentTranslator{serverRx, perResourceRx, countConn, s2mConn} + } else { + // MySQL uses a single metrics receiver (no split collection) + receivers = []common.ComponentTranslator{t.receiver("metrics"), countConn, s2mConn} + } return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](serverRx, perResourceRx, countConn, s2mConn), + Receivers: common.NewTranslatorMap[component.Config, component.ID](receivers...), Processors: common.NewTranslatorMap[component.Config, component.ID]( t.scopeTransform(), - transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricResourceStatements(t.resourceStatements())), - transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime)), + transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricResourceStatements(t.resourceStatements())), + transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+t.cfg.engine, transformprocessor.WithDbiFixStartTime(t.cfg.engine))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd, countConn, s2mConn), @@ -135,11 +171,19 @@ func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) } func (t *dbiTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) + idx := strconv.Itoa(t.instanceIndex) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine+"_"+idx, t.cfg.engine) + + var receiver common.ComponentTranslator + if t.cfg.engine == common.PostgreSQLKey { + receiver = t.pgReceiver("metrics", postgresql.WithServerMetricsOnly()) + } else { + receiver = t.receiver("metrics") + } return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("metrics", postgresql.WithServerMetricsOnly())), + Receivers: common.NewTranslatorMap[component.Config, component.ID](receiver), Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter()), Exporters: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), Extensions: common.NewTranslatorMap[component.Config, component.ID](), @@ -151,25 +195,33 @@ func (t *dbiTranslator) translateRawEvents() (*common.ComponentTranslators, erro idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - return &common.ComponentTranslators{ + var receiver common.ComponentTranslator + if t.cfg.engine == common.PostgreSQLKey { // The events receiver feeds a logs pipeline only, so no metrics pipeline consumes // it. Disable its metrics so the generated config reflects that instead of // listing metrics that are never collected. - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("events", + receiver = t.pgReceiver("events", postgresql.WithQuerySampleInterval(60*time.Second), postgresql.WithMetricsDisabled(), - )), + ) + } else { + // MySQL events receiver with 60s interval for top query collection + receiver = t.receiver("events") + } + + return &common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config, component.ID](receiver), Processors: common.NewTranslatorMap[component.Config, component.ID]( t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName( - common.DbiTransformResource+"_"+idx, + common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricResourceStatements(t.resourceStatements()), transformprocessor.WithLogResourceStatements(t.resourceStatements()), ), transformprocessor.NewTranslatorWithName( - common.DbiTransformLogs+"_raw-events_"+idx, + common.DbiTransformLogs+"_"+t.cfg.engine+"_raw-events_"+idx, transformprocessor.WithLogResourceStatements(t.logStatements("raw-events")), ), ), @@ -183,19 +235,13 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - // NOTE: timestamp parsing assumes the PostgreSQL instance logs in UTC. The %Z token - // only matches 3-letter zone abbreviations and the gotime layout parses them against a - // UTC location, so timestamps from a non-UTC instance would be silently offset. DBI - // therefore requires the instance to be configured with log_timezone = 'UTC'. + // NOTE: timestamp parsing assumes the instance logs in UTC. Both engines' timestamp + // layouts parse against a UTC location, so timestamps from a non-UTC instance would be + // silently offset. DBI therefore requires the instance to emit UTC timestamps in its + // logs (PostgreSQL: log_timezone = 'UTC'; MySQL: log_timestamps = UTC, which is the + // default). return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID]( - filelog.NewTranslator(filelog.WithNamePrefix("postgresql"), - filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath), - filelog.WithMultilinePattern(`^\d{4}-\d{2}-\d{2}`), - filelog.WithTimestampFormat("%Y-%m-%d %H:%M:%S.%f %Z", "UTC"), - filelog.WithSeverityPattern(buildPostgresSeverityPattern()), - filelog.WithSeverityMapping(postgresLogSeverityMapping)), - ), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.serverLogReceiver()), Processors: common.NewTranslatorMap[component.Config, component.ID]( t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), @@ -204,12 +250,12 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err // with regular file logs. groupbyattrsprocessor.NewTranslatorWithName(common.FilesKey, "log.file.name"), transformprocessor.NewTranslatorWithName( - common.DbiTransformResource+"_"+idx, + common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricResourceStatements(t.resourceStatements()), transformprocessor.WithLogResourceStatements(t.resourceStatements()), ), transformprocessor.NewTranslatorWithName( - common.DbiTransformLogs+"_server-logs_"+idx, + common.DbiTransformLogs+"_"+t.cfg.engine+"_server-logs_"+idx, transformprocessor.WithLogResourceStatements(t.logStatements("server-logs")), // Drop the parsed attributes once promoted to the record's timestamp and // severity fields, mirroring the files pipeline's timestamp cleanup, so they @@ -226,6 +272,73 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err }, nil } +// serverLogReceiver builds the engine-specific filelog receiver for server logs. +// Both engines get multiline grouping plus timestamp and severity parsing, but the +// timestamp layout and severity vocabulary differ by engine (PostgreSQL stderr format +// vs MySQL classic error-log format). +func (t *dbiTranslator) serverLogReceiver() common.ComponentTranslator { + opts := []filelog.Option{ + filelog.WithNamePrefix(t.cfg.engine), + filelog.WithIndex(t.instanceIndex), + filelog.WithFilePath(t.cfg.logFilePath), + filelog.WithMultilinePattern(`^\d{4}-\d{2}-\d{2}`), + } + switch t.cfg.engine { + case common.PostgreSQLKey: + opts = append(opts, + filelog.WithTimestampFormat("%Y-%m-%d %H:%M:%S.%f %Z", "UTC"), + filelog.WithSeverityPattern(buildPostgresSeverityPattern()), + filelog.WithSeverityMapping(postgresLogSeverityMapping), + ) + case common.MySQLKey: + // MySQL 8.0+ error log timestamps are ISO 8601 UTC, e.g. + // "2026-07-20T15:27:47.123456Z" (T separator, trailing literal Z). + opts = append(opts, + filelog.WithTimestampFormat("%Y-%m-%dT%H:%M:%S.%fZ", "UTC"), + filelog.WithSeverityPattern(buildMysqlSeverityPattern()), + filelog.WithSeverityMapping(mysqlLogSeverityMapping), + ) + } + return filelog.NewTranslator(opts...) +} + +// receiver builds the engine-specific receiver translator. name is "metrics" or +// "events"; for PostgreSQL events we override the query sample interval. +func (t *dbiTranslator) receiver(name string) common.ComponentTranslator { + if t.cfg.engine == common.MySQLKey { + opts := []mysql.Option{ + mysql.WithName(name), + mysql.WithIndex(t.instanceIndex), + mysql.WithEndpoint(t.cfg.endpoint), + mysql.WithUsername(t.cfg.username), + mysql.WithPassfile(t.cfg.passfile), + } + if name == "events" { + opts = append(opts, mysql.WithTopQueryInterval(60*time.Second)) + } + return mysql.NewTranslator(opts...) + } + + if t.cfg.engine == common.PostgreSQLKey { + opts := []postgresql.Option{ + postgresql.WithName(name), + postgresql.WithIndex(t.instanceIndex), + postgresql.WithEndpoint(t.cfg.endpoint), + postgresql.WithUsername(t.cfg.username), + postgresql.WithPassfile(t.cfg.passfile), + postgresql.WithCAFile(t.cfg.caFile), + postgresql.WithIsLocalhost(t.cfg.isLocalhost), + } + if name == "events" { + opts = append(opts, postgresql.WithQuerySampleInterval(60*time.Second)) + } + return postgresql.NewTranslator(opts...) + } + + return nil +} + +// pgReceiver builds a PostgreSQL receiver with custom options (for split metrics collection). func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) common.ComponentTranslator { opts := []postgresql.Option{ postgresql.WithName(name), @@ -242,12 +355,22 @@ func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) func (t *dbiTranslator) excludeMonitorFilter() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - condition := fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) - return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+idx, condition, common.OTTLErrorModePropagate) + var condition string + switch t.cfg.engine { + case common.PostgreSQLKey: + condition = fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) + case common.MySQLKey: + // For MySQL, only check user.name (MySQL receiver doesn't emit a separate role attribute like PostgreSQL) + condition = fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + default: + condition = fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + } + return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+t.cfg.engine+"_"+idx, condition, common.OTTLErrorModePropagate) } func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { - return transformprocessor.NewTranslatorWithName("dbi_scope", + idx := strconv.Itoa(t.instanceIndex) + return transformprocessor.NewTranslatorWithName("dbi_scope_"+t.cfg.engine+"_"+idx, transformprocessor.WithErrorMode("ignore"), transformprocessor.WithMetricScopeStatements(common.ScopeStatementsForSolution("otel-database-insights")), transformprocessor.WithLogScopeStatements(common.ScopeStatementsForSolution("otel-database-insights")), @@ -256,14 +379,14 @@ func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { func (t *dbiTranslator) resourceStatements() []string { return []string{ - `set(resource.attributes["db.system.name"], "postgresql")`, + fmt.Sprintf(`set(resource.attributes["db.system.name"], "%s")`, t.cfg.engine), fmt.Sprintf(`set(resource.attributes["db.instance.name"], "%s")`, t.cfg.instanceName), } } func (t *dbiTranslator) logStatements(destination string) []string { return []string{ - fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/postgresql/%s")`, destination), + fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/%s/%s")`, t.cfg.engine, destination), fmt.Sprintf(`set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "%s"], "/"))`, t.cfg.instanceName), } } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index bac43c5a964..ed5e6a8fb3b 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -17,20 +17,25 @@ import ( func TestDbiTranslatorID(t *testing.T) { tests := []struct { name string + engine string pipelineType dbiPipelineType index int want string }{ - {"Metrics_0", dbiMetrics, 0, "metrics/dbi_postgresql_0"}, - {"Metrics_1", dbiMetrics, 1, "metrics/dbi_postgresql_1"}, - {"LogToMetrics_0", dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, - {"RawEvents_0", dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, - {"ServerLogs_0", dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, - {"ServerLogs_2", dbiServerLogs, 2, "logs/dbi_postgresql_serverlogs_2"}, + {"PG_Metrics_0", common.PostgreSQLKey, dbiMetrics, 0, "metrics/dbi_postgresql_0"}, + {"PG_Metrics_1", common.PostgreSQLKey, dbiMetrics, 1, "metrics/dbi_postgresql_1"}, + {"PG_LogToMetrics_0", common.PostgreSQLKey, dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, + {"PG_RawEvents_0", common.PostgreSQLKey, dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, + {"PG_ServerLogs_0", common.PostgreSQLKey, dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, + {"Mysql_Metrics_0", common.MySQLKey, dbiMetrics, 0, "metrics/dbi_mysql_0"}, + {"Mysql_Metrics_1", common.MySQLKey, dbiMetrics, 1, "metrics/dbi_mysql_1"}, + {"Mysql_LogToMetrics_0", common.MySQLKey, dbiLogToMetrics, 0, "logs/dbi_mysql_0"}, + {"Mysql_RawEvents_0", common.MySQLKey, dbiRawEvents, 0, "logs/dbi_mysql_rawevents_0"}, + {"Mysql_ServerLogs_0", common.MySQLKey, dbiServerLogs, 0, "logs/dbi_mysql_serverlogs_0"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index} + tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index, cfg: dbiInstanceConfig{engine: tc.engine}} assert.Equal(t, tc.want, tr.ID().String()) }) } @@ -38,6 +43,7 @@ func TestDbiTranslatorID(t *testing.T) { func TestDbiTranslate(t *testing.T) { cfg := dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: "localhost:5432", username: "cw_monitor", passfile: "/etc/.pgpass", @@ -80,7 +86,7 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { tr := &dbiTranslator{ pipelineType: dbiMetrics, instanceIndex: 0, - cfg: dbiInstanceConfig{instanceName: "mydb"}, + cfg: dbiInstanceConfig{engine: common.PostgreSQLKey, instanceName: "mydb"}, } result, err := tr.Translate(nil) require.NoError(t, err) @@ -99,14 +105,85 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { connectors = append(connectors, c.ID().String()) }) - assert.ElementsMatch(t, []string{"postgresql/metrics_0", "postgresql/metrics_perresource_0", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, receivers) - assert.Equal(t, []string{"transform/dbi_scope", "transform/dbi_resource_0", "transform/dbi_fix_start_time"}, processors) + assert.ElementsMatch(t, []string{"postgresql/metrics_0", "postgresql/metrics_perresource_0", "count/dbi_dbload_postgresql_0", "signaltometrics/dbi_topsql_postgresql_0"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_postgresql_0", "transform/dbi_resource_postgresql_0", "transform/dbi_fix_start_time_postgresql"}, processors) assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) - assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, connectors) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_postgresql_0", "signaltometrics/dbi_topsql_postgresql_0"}, connectors) +} + +func TestDbiMysqlTranslate(t *testing.T) { + cfg := dbiInstanceConfig{ //nolint:gosec + engine: common.MySQLKey, + endpoint: "localhost:3306", + username: "cw_monitor", + passfile: "/etc/.mysql_credentials", + instanceName: "my-db", + logFilePath: "/var/log/mysql/mysql.log", + isLocalhost: true, + } + + tests := []struct { + name string + pipeline dbiPipelineType + expectedID string + nRecv int + nProc int + nExp int + nConn int + }{ + {"metrics", dbiMetrics, "metrics/dbi_mysql_0", 3, 3, 1, 3}, + {"log_to_metrics", dbiLogToMetrics, "logs/dbi_mysql_0", 1, 1, 2, 2}, + {"raw_events", dbiRawEvents, "logs/dbi_mysql_rawevents_0", 1, 5, 1, 1}, + {"server_logs", dbiServerLogs, "logs/dbi_mysql_serverlogs_0", 1, 5, 1, 1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tr := &dbiTranslator{pipelineType: tc.pipeline, instanceIndex: 0, cfg: cfg} + assert.Equal(t, tc.expectedID, tr.ID().String()) + + result, err := tr.Translate(nil) + require.NoError(t, err) + assert.Equal(t, tc.nRecv, result.Receivers.Len()) + assert.Equal(t, tc.nProc, result.Processors.Len()) + assert.Equal(t, tc.nExp, result.Exporters.Len()) + assert.Equal(t, tc.nConn, result.Connectors.Len()) + }) + } +} + +func TestDbiMysqlTranslateMetrics_ComponentIDs(t *testing.T) { + tr := &dbiTranslator{ + pipelineType: dbiMetrics, + instanceIndex: 0, + cfg: dbiInstanceConfig{engine: common.MySQLKey, instanceName: "mydb"}, + } + result, err := tr.Translate(nil) + require.NoError(t, err) + + var receivers, processors, exporters, connectors []string + result.Receivers.Range(func(c common.Translator[component.Config, component.ID]) { + receivers = append(receivers, c.ID().String()) + }) + result.Processors.Range(func(c common.Translator[component.Config, component.ID]) { + processors = append(processors, c.ID().String()) + }) + result.Exporters.Range(func(c common.Translator[component.Config, component.ID]) { + exporters = append(exporters, c.ID().String()) + }) + result.Connectors.Range(func(c common.Translator[component.Config, component.ID]) { + connectors = append(connectors, c.ID().String()) + }) + + assert.ElementsMatch(t, []string{"mysql/metrics_0", "count/dbi_dbload_mysql_0", "signaltometrics/dbi_topsql_mysql_0"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_mysql_0", "transform/dbi_resource_mysql_0", "transform/dbi_fix_start_time_mysql"}, processors) + assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_mysql_0", "signaltometrics/dbi_topsql_mysql_0"}, connectors) } func TestDbiTranslateServerLogs_FilelogConfig(t *testing.T) { cfg := dbiInstanceConfig{ //nolint:gosec + engine: common.PostgreSQLKey, endpoint: "localhost:5432", username: "cw_monitor", passfile: "/etc/.pgpass", diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index a3148d8e285..549cb8051e6 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -5,6 +5,7 @@ package databaseinsights import ( "fmt" + "os" "regexp" "strings" "time" @@ -21,6 +22,7 @@ import ( var ottlSafeRegex = regexp.MustCompile(`^[a-zA-Z0-9._@/:\-]+$`) type dbiInstanceConfig struct { + engine string endpoint string username string passfile string @@ -37,15 +39,22 @@ func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap { return translators } - instances := parseDbiPostgresqlInstances(conf) - for i, cfg := range instances { - translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) - if cfg.logFilePath != "" { - translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + // Each engine is indexed independently; the engine name is part of every + // component ID, so PostgreSQL and MySQL instances never collide. + addInstances := func(instances []dbiInstanceConfig) { + for i, cfg := range instances { + translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) + if cfg.logFilePath != "" { + translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + } } } + + addInstances(parseDbiPostgresqlInstances(conf)) + addInstances(parseDbiMysqlInstances(conf)) + return translators } @@ -65,7 +74,9 @@ func parseDbiPostgresqlInstances(conf *confmap.Conf) []dbiInstanceConfig { arr, _ := conf.Get(common.DatabaseInsightsPostgresKey).([]any) var raw []pgRawInstance if err := mapstructure.Decode(arr, &raw); err != nil { - return nil + // Log error but return empty slice to allow other database instances to be processed + fmt.Fprintf(os.Stderr, "failed to decode PostgreSQL database_insights config: %v\n", err) + return []dbiInstanceConfig{} } instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { @@ -74,6 +85,7 @@ func parseDbiPostgresqlInstances(conf *confmap.Conf) []dbiInstanceConfig { perResourceInterval = time.Duration(r.PerResourceCollectionInterval) * time.Second } instances = append(instances, dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: r.Endpoint, username: r.Username, passfile: r.PasswordFile, @@ -97,8 +109,43 @@ func validateOttlSafe(field, value string) error { return nil } +type mysqlRawInstance struct { + Endpoint string `mapstructure:"endpoint"` + Username string `mapstructure:"username"` + PasswordFile string `mapstructure:"password_file"` + InstanceName string `mapstructure:"instance_name"` + Logs struct { + FilePath string `mapstructure:"file_path"` + } `mapstructure:"logs"` +} + +func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { + arr, _ := conf.Get(common.DatabaseInsightsMysqlKey).([]any) + var raw []mysqlRawInstance + if err := mapstructure.Decode(arr, &raw); err != nil { + // Log error but return empty slice to allow other database instances to be processed + fmt.Fprintf(os.Stderr, "failed to decode MySQL database_insights config: %v\n", err) + return []dbiInstanceConfig{} + } + instances := make([]dbiInstanceConfig, 0, len(raw)) + for _, r := range raw { + instances = append(instances, dbiInstanceConfig{ + engine: common.MySQLKey, + endpoint: r.Endpoint, + username: r.Username, + passfile: r.PasswordFile, + instanceName: r.InstanceName, + logFilePath: r.Logs.FilePath, + isLocalhost: isLocalhostEndpoint(r.Endpoint), + perResourceCollectionInterval: 0, // MySQL doesn't have per-resource metrics + }) + } + return instances +} + func isLocalhostEndpoint(endpoint string) bool { return strings.HasPrefix(endpoint, "localhost") || strings.HasPrefix(endpoint, "127.0.0.1") || - strings.HasPrefix(endpoint, "[::1]") + strings.HasPrefix(endpoint, "[::1]") || + strings.HasPrefix(endpoint, "::1") } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go index 004ad40182d..81828af3271 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go @@ -110,11 +110,88 @@ func TestNewTranslators_MultiInstance(t *testing.T) { func TestIsLocalhostEndpoint(t *testing.T) { assert.True(t, isLocalhostEndpoint("localhost:5432")) assert.True(t, isLocalhostEndpoint("127.0.0.1:5432")) - assert.True(t, isLocalhostEndpoint("[::1]:5432")) + assert.True(t, isLocalhostEndpoint("[::1]:3306")) + assert.True(t, isLocalhostEndpoint("::1:3306")) assert.False(t, isLocalhostEndpoint("db.example.com:5432")) assert.False(t, isLocalhostEndpoint("")) } +func TestNewTranslators_MySQL_WithLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ //nolint:gosec + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql.log"}, + }, + }, + }, + }, + }, + }) + assert.Equal(t, 4, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_NoLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ //nolint:gosec + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + }, + }, + }, + }, + }, + }) + assert.Equal(t, 3, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_MultiInstance(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ //nolint:gosec + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "db1", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql1.log"}, + }, + map[string]interface{}{ //nolint:gosec + "endpoint": "db2.example.com:3306", "username": "monitor", + "password_file": "/etc/.mysql_credentials2", "instance_name": "db2", + }, + }, + }, + }, + }, + }) + translators := NewTranslators(cfg) + assert.Equal(t, 7, translators.Len()) // 4 + 3 + + var ids []string + for _, k := range translators.Keys() { + ids = append(ids, k.String()) + } + expected := []string{ + "metrics/dbi_mysql_0", + "logs/dbi_mysql_0", + "logs/dbi_mysql_rawevents_0", + "logs/dbi_mysql_serverlogs_0", + "metrics/dbi_mysql_1", + "logs/dbi_mysql_1", + "logs/dbi_mysql_rawevents_1", + } + assert.ElementsMatch(t, expected, ids) +} + func TestParseDbiPostgresqlInstances_PerResourceCollectionInterval(t *testing.T) { tests := []struct { name string diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go index 60ccf01e8d9..b0877c4559d 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go @@ -38,10 +38,17 @@ func (t *hostMetricsTranslator) Translate(conf *confmap.Conf) (*common.Component var opts []hostmetrics.Option opts = append(opts, hostmetrics.WithName(common.OpenTelemetryKey)) if conf.IsSet(common.DatabaseInsightsConfigKey) { + var processNames []string + if conf.IsSet(common.DatabaseInsightsPostgresKey) { + processNames = append(processNames, "post(gres|master).*") + } + if conf.IsSet(common.DatabaseInsightsMysqlKey) { + processNames = append(processNames, "mysqld.*") + } opts = append(opts, hostmetrics.WithProcessScraper(map[string]any{ "include": map[string]any{ "match_type": "regexp", - "names": []string{"post(gres|master).*"}, + "names": processNames, }, "mute_process_all_errors": true, "metrics": map[string]any{ @@ -51,6 +58,9 @@ func (t *hostMetricsTranslator) Translate(conf *confmap.Conf) (*common.Component "process.memory.utilization": map[string]any{ "enabled": true, }, + "process.threads": map[string]any{ + "enabled": true, + }, }, })) } diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go index 75adf559383..f0f23a2a161 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go @@ -23,9 +23,10 @@ func TestHostMetricsTranslator(t *testing.T) { assert.EqualValues(t, "metrics/host_metrics", tt.ID().String()) testCases := map[string]struct { - input map[string]interface{} - wantErr error - expectProcess bool + input map[string]interface{} + wantErr error + expectProcess bool + wantProcessNames []string }{ "WithNilConf": { input: nil, @@ -56,7 +57,41 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, + }, + "WithMysqlDatabaseInsightsKey": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"mysqld.*"}, + }, + "WithBothEngines": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:5432", "username": "cwagent"}, + }, + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*", "mysqld.*"}, }, "WithBothKeys": { input: map[string]interface{}{ @@ -71,7 +106,8 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, }, } for name, tc := range testCases { @@ -109,7 +145,7 @@ func TestHostMetricsTranslator(t *testing.T) { assert.True(t, exists, "expected process scraper") include := processCfg["include"].(map[string]any) assert.Equal(t, "regexp", include["match_type"]) - assert.Equal(t, []string{"post(gres|master).*"}, include["names"]) + assert.Equal(t, tc.wantProcessNames, include["names"]) } } }) diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go index 94f645ee651..369c00932a0 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go @@ -10,6 +10,8 @@ import ( func init() { otelRule := mergeJsonUtil.NewSectionMergeRule("opentelemetry", "/") collectRule := mergeJsonUtil.NewSectionMergeRule("collect", otelRule.Path) + dbiRule := mergeJsonUtil.NewSectionMergeRule("database_insights", collectRule.Path) + collectRule.MergeMap[dbiRule.SectionKey] = dbiRule otelRule.MergeMap[collectRule.SectionKey] = collectRule mergeJsonUtil.MergeRuleMap[otelRule.SectionKey] = otelRule } diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go index f5918d7b181..2d6e120cf95 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go @@ -32,3 +32,46 @@ func TestMergeRule(t *testing.T) { assert.Contains(t, otel, "collect") assert.Contains(t, otel["collect"].(map[string]interface{}), "host_metrics") } + +func TestMergeRule_DatabaseInsights(t *testing.T) { + // Verify that postgresql and mysql from separate configs merge correctly + // under database_insights (simulates separate SSM parameters) + source1 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:5432", + "instance_name": "pg-test", + }, + }, + }, + }, + }, + } + source2 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", + "instance_name": "mysql-test", + }, + }, + }, + }, + }, + } + + // Merge source1 into source2 (source2 acts as result) + mergeJsonUtil.MergeRuleMap["opentelemetry"].Merge(source1, source2) + + otel := source2["opentelemetry"].(map[string]interface{}) + collect := otel["collect"].(map[string]interface{}) + dbi := collect["database_insights"].(map[string]interface{}) + + assert.Contains(t, dbi, "postgresql", "postgresql should be preserved after merge") + assert.Contains(t, dbi, "mysql", "mysql should be preserved after merge") +} diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index 15347f98a4e..c88c8c37a3e 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -171,3 +171,16 @@ func TestDbiExcludeMonitor(t *testing.T) { require.Len(t, fCfg.Logs.LogConditions, 1) assert.Equal(t, condition, fCfg.Logs.LogConditions[0]) } + +func TestDbiExcludeMonitorMySQL(t *testing.T) { + condition := `attributes["user.name"] == "cw_monitor"` + tr := NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_mysql_0", condition, "propagate") + assert.Equal(t, "filter/dbi_exclude_monitor_mysql_0", tr.ID().String()) + + cfg, err := tr.Translate(nil) + require.NoError(t, err) + fCfg := cfg.(*filterprocessor.Config) + require.NotNil(t, fCfg.Logs) + require.Len(t, fCfg.Logs.LogConditions, 1) + assert.Equal(t, condition, fCfg.Logs.LogConditions[0]) +} diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml new file mode 100644 index 00000000000..6c528ef72e5 --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml @@ -0,0 +1,10 @@ +error_mode: ignore +metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml similarity index 100% rename from translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml rename to translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml diff --git a/translator/translate/otel/processor/transformprocessor/translator.go b/translator/translate/otel/processor/transformprocessor/translator.go index 41d039a534c..1f1b12cc1e3 100644 --- a/translator/translate/otel/processor/transformprocessor/translator.go +++ b/translator/translate/otel/processor/transformprocessor/translator.go @@ -29,9 +29,12 @@ var transformJmxDropConfig string //go:embed transform_efa_config.yaml var transformEfaConfig string -//go:embed transform_dbi_fix_start_time.yaml +//go:embed transform_dbi_fix_start_time_postgresql.yaml var transformDbiFixStartTimeConfig string +//go:embed transform_dbi_fix_start_time_mysql.yaml +var transformDbiFixStartTimeMysqlConfig string + //go:embed transform_identity_ec2.yaml var transformIdentityEC2Config string @@ -108,6 +111,14 @@ func WithMetricScopeStatements(statements []string) Option { } } +// WithDbiFixStartTime selects the DBI fix-start-time config for the given engine +// ("postgresql" or "mysql"). The component is named by the caller (index-based). +func WithDbiFixStartTime(engine string) Option { + return func(t *translator) { + t.dbiFixStartTimeEngine = engine + } +} + type translator struct { name string factory processor.Factory @@ -119,6 +130,7 @@ type translator struct { logScopeStatements []string metricScopeStatements []string errorMode string + dbiFixStartTimeEngine string } var _ common.ComponentTranslator = (*translator)(nil) @@ -196,8 +208,15 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.name, common.PipelineNameHostDeltaMetrics) { return common.GetYamlFileToYamlConfig(cfg, transformEfaConfig) } - if t.name == common.DbiTransformFixStartTime { - return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + // DBI fix-start-time: content is engine-specific, component name is index-based. + if t.dbiFixStartTimeEngine != "" { + switch t.dbiFixStartTimeEngine { + case common.PostgreSQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeMysqlConfig) + } + return nil, fmt.Errorf("unsupported dbi fix-start-time engine: %s", t.dbiFixStartTimeEngine) } if t.name == common.Identity { if context.CurrentContext().KubernetesMode() != "" { diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index 340b3a94d39..1641777ec4c 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -81,8 +81,8 @@ func TestJmxTranslate(t *testing.T) { } func TestDbiFixStartTimeTranslate(t *testing.T) { - transl := NewTranslatorWithName(common.DbiTransformFixStartTime) - assert.Equal(t, "transform/dbi_fix_start_time", transl.ID().String()) + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.PostgreSQLKey, WithDbiFixStartTime(common.PostgreSQLKey)) + assert.Equal(t, "transform/dbi_fix_start_time_postgresql", transl.ID().String()) cfg, err := transl.Translate(nil) require.NoError(t, err) @@ -94,6 +94,21 @@ func TestDbiFixStartTimeTranslate(t *testing.T) { assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[6]) } +func TestDbiFixStartTimeMysqlTranslate(t *testing.T) { + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.MySQLKey, WithDbiFixStartTime(common.MySQLKey)) + assert.Equal(t, "transform/dbi_fix_start_time_mysql", transl.ID().String()) + + cfg, err := transl.Translate(nil) + require.NoError(t, err) + actualCfg := cfg.(*transformprocessor.Config) + require.Len(t, actualCfg.MetricStatements, 1) + assert.Equal(t, "datapoint", string(actualCfg.MetricStatements[0].Context)) + require.Len(t, actualCfg.MetricStatements[0].Statements, 5) + assert.Equal(t, "set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0", actualCfg.MetricStatements[0].Statements[0]) + assert.Equal(t, `replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU")`, actualCfg.MetricStatements[0].Statements[1]) + assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[4]) +} + func TestDbiResourceTranslate(t *testing.T) { stmts := []string{ `set(resource.attributes["db.system.name"], "postgresql")`, diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go new file mode 100644 index 00000000000..1c8a5df4ed4 --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -0,0 +1,99 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "strconv" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/receiver" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type Option func(*translator) + +type translator struct { + factory receiver.Factory + name string + endpoint string + username string + passfile string + index int + topQueryInterval time.Duration +} + +func WithName(name string) Option { return func(t *translator) { t.name = name } } +func WithEndpoint(ep string) Option { return func(t *translator) { t.endpoint = ep } } +func WithUsername(u string) Option { return func(t *translator) { t.username = u } } +func WithPassfile(p string) Option { return func(t *translator) { t.passfile = p } } +func WithIndex(i int) Option { return func(t *translator) { t.index = i } } +func WithTopQueryInterval(d time.Duration) Option { + return func(t *translator) { t.topQueryInterval = d } +} + +func NewTranslator(opts ...Option) common.ComponentTranslator { + t := &translator{ + factory: mysqlreceiver.NewFactory(), + name: "metrics", + topQueryInterval: 60 * time.Second, + } + for _, opt := range opts { + opt(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.MustNewIDWithName("mysql", t.name+"_"+strconv.Itoa(t.index)) +} + +func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*mysqlreceiver.Config) + + cfg.Endpoint = t.endpoint + cfg.Username = t.username + cfg.Passfile = t.passfile + cfg.Transport = confignet.TransportTypeTCP + + // DBI only supports local MySQL instances, so the connection does not use + // TLS. Remote (TLS) connections are intentionally not supported. + cfg.TLS = configtls.ClientConfig{ + Insecure: true, + } + + // Enable metrics that are disabled by default in the receiver but required + // by Database Insights. + cfg.MetricsBuilderConfig.Metrics.MysqlTableSize.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlReplicaSQLDelay.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlReplicaTimeBehindSource.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlDeadlocks.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlConnectionCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlConnectionErrors.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlMaxUsedConnections.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlCommands.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQueryCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlHistoryListLength.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlActiveTransactions.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQueryClientCount.Enabled = true + cfg.MetricsBuilderConfig.Metrics.MysqlQuerySlowCount.Enabled = true + + cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true + cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true + + cfg.QuerySampleCollection.MaxRowsPerQuery = 500 + + cfg.TopQueryCollection.CollectionInterval = t.topQueryInterval + cfg.TopQueryCollection.TopQueryCount = 200 + cfg.TopQueryCollection.MaxQuerySampleCount = 5000 + cfg.TopQueryCollection.QueryPlanCacheSize = 1000 + cfg.TopQueryCollection.QueryPlanCacheTTL = time.Hour + + return cfg, nil +} diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go new file mode 100644 index 00000000000..10abf0ec57d --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslator_ID(t *testing.T) { + tr := NewTranslator(WithName("metrics"), WithIndex(0)) + assert.Equal(t, "mysql/metrics_0", tr.ID().String()) + + tr = NewTranslator(WithName("events"), WithIndex(1)) + assert.Equal(t, "mysql/events_1", tr.ID().String()) +} + +func TestTranslator_Translate_Localhost(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithIndex(0), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, "localhost:3306", mysqlCfg.Endpoint) + assert.Equal(t, "cw_monitor", mysqlCfg.Username) + assert.Equal(t, "/etc/.mysql_credentials", mysqlCfg.Passfile) + assert.True(t, mysqlCfg.TLS.Insecure) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) + assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.Equal(t, uint64(200), mysqlCfg.TopQueryCollection.TopQueryCount) + assert.Equal(t, uint64(5000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) + assert.Equal(t, 1000, mysqlCfg.TopQueryCollection.QueryPlanCacheSize) + assert.Equal(t, time.Hour, mysqlCfg.TopQueryCollection.QueryPlanCacheTTL) + assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) +} + +func TestTranslator_Translate_CustomTopQueryInterval(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithTopQueryInterval(30*time.Second), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, 30*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) +}