From df3ef08ff9e3f2b58246b1f52644f4728ece6717 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 2 Apr 2026 08:08:28 +0000 Subject: [PATCH 001/246] 1st test push interface --- internal/apiserver/apiserver.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index f31abaa..8c4cbf9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -2,6 +2,7 @@ package apiserver import ( "encoding/json" + "log" "net/http" "github.com/gnmic/operator/internal/controller" @@ -27,6 +28,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServe func (a *APIServer) routes(mux *http.ServeMux) { mux.HandleFunc("GET /clusters/{namespace}/{name}/plan", a.getClusterPlan) + mux.HandleFunc("POST /clusters/{namespace}/{name}/createTarget", a.postCreateTarget) } func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { @@ -43,3 +45,11 @@ func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { return } } + +// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +// curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget + +func (a *APIServer) postCreateTarget(w http.ResponseWriter, r *http.Request) { + log.Printf("received POST request: path=%s method=%s remote=%s", r.URL.Path, r.Method, r.RemoteAddr) + w.WriteHeader(http.StatusAccepted) +} From fe8dc49a025c00313d67916129917d23673c7e95 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 2 Apr 2026 14:14:35 +0000 Subject: [PATCH 002/246] change to gin-gonic --- cmd/main.go | 48 +++++++-------- go.mod | 46 +++++++++++---- go.sum | 100 +++++++++++++++++++++++++------- internal/apiserver/apiserver.go | 42 +++++--------- 4 files changed, 155 insertions(+), 81 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4c37a0d..d9758fa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,10 +17,8 @@ limitations under the License. package main import ( - "context" "flag" "os" - "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,7 +31,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -219,26 +216,31 @@ func main() { } if apiAddr != "" { - apiServer := apiserver.New(apiAddr, clusterReconciler) - err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - errCh := make(chan error) - go func() { - errCh <- apiServer.Server.ListenAndServe() - }() - select { - case err := <-errCh: - return err - case <-ctx.Done(): - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - return apiServer.Server.Shutdown(ctx) - } - })) - if err != nil { - setupLog.Error(err, "unable to add api server") - os.Exit(1) - } - } + apiserver.New(apiAddr, clusterReconciler) + + } + + // if apiAddr != "" { + // apiServer := apiserver.New(apiAddr, clusterReconciler) + // err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + // errCh := make(chan error) + // go func() { + // errCh <- apiServer.Server.ListenAndServe() + // }() + // select { + // case err := <-errCh: + // return err + // case <-ctx.Done(): + // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + // defer cancel() + // return apiServer.Server.Shutdown(ctx) + // } + // })) + // if err != nil { + // setupLog.Error(err, "unable to add api server") + // os.Exit(1) + // } + // } // start manager setupLog.Info("starting manager") diff --git a/go.mod b/go.mod index f236ded..208f424 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 github.com/go-logr/logr v1.4.3 - github.com/onsi/ginkgo/v2 v2.27.3 - github.com/onsi/gomega v1.38.3 + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.0 github.com/openconfig/gnmic/pkg/api v0.1.10 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.35.1 @@ -17,6 +17,31 @@ require ( sigs.k8s.io/controller-runtime v0.22.4 ) +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect +) + require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect @@ -27,6 +52,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gin-gonic/gin v1.12.0 github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect github.com/go-openapi/jsonreference v0.21.4 // indirect @@ -46,7 +72,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -63,15 +89,15 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.41.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.79.3 // indirect @@ -83,6 +109,6 @@ require ( sigs.k8s.io/gateway-api v1.4.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 8a613b4..8b1369f 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,18 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs= github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -22,6 +30,12 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -68,10 +82,20 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxE github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -83,8 +107,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f h1:HU1RgM6NALf/KW9HEY6zry3ADbDKcmpQ+hJedoNGQYQ= -github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f/go.mod h1:67FPmZWbr+KDT/VlpWtw6sO9XSjpJmLuHpoLmWiTGgY= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -93,14 +117,20 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -111,12 +141,14 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.27.3 h1:ICsZJ8JoYafeXFFlFAG75a7CxMsJHwgKwtO+82SE9L8= -github.com/onsi/ginkgo/v2 v2.27.3/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= -github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= -github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= github.com/openconfig/gnmic/pkg/api v0.1.10/go.mod h1:6PntONfjCMq3XzsDfWMkLeoVuBRbkm2foQO5m6PeYo0= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -130,14 +162,24 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -148,8 +190,14 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -164,6 +212,8 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= @@ -172,24 +222,29 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -209,6 +264,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.35.1 h1:0PO/1FhlK/EQNVK5+txc4FuhQibV25VLSdLMmGpDE/Q= @@ -233,7 +289,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8c4cbf9..7370be9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,55 +1,45 @@ package apiserver import ( - "encoding/json" "log" - "net/http" + "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" ) type APIServer struct { - Server *http.Server + Server *gin.Engine clusterReconciler *controller.ClusterReconciler } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { - mux := http.NewServeMux() + _ = addr a := &APIServer{ - Server: &http.Server{ - Addr: addr, - Handler: mux, - }, + Server: gin.Default(), clusterReconciler: clusterReconciler, } - a.routes(mux) - return a -} -func (a *APIServer) routes(mux *http.ServeMux) { - mux.HandleFunc("GET /clusters/{namespace}/{name}/plan", a.getClusterPlan) - mux.HandleFunc("POST /clusters/{namespace}/{name}/createTarget", a.postCreateTarget) + a.Server.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) + a.Server.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) + a.Server.Run(":8082") + return a } -func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { - namespace, name := r.PathValue("namespace"), r.PathValue("name") +func (a *APIServer) getClusterPlan(c *gin.Context) { + log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) + namespace, name := c.Param("namespace"), c.Param("name") plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(plan) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + c.String(404, err.Error()) return } + c.JSON(200, plan) } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget -func (a *APIServer) postCreateTarget(w http.ResponseWriter, r *http.Request) { - log.Printf("received POST request: path=%s method=%s remote=%s", r.URL.Path, r.Method, r.RemoteAddr) - w.WriteHeader(http.StatusAccepted) +func (a *APIServer) postCreateTarget(c *gin.Context) { + log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) + c.Status(202) } From c902d0578cef25685c2a0846dd68018b6b81ecdf Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 18:45:02 -0600 Subject: [PATCH 003/246] first draft to create targets --- api/v1alpha1/targetsource_types.go | 7 +++ .../operator.gnmic.dev_targetsources.yaml | 11 +++++ .../controller/targetsource_controller.go | 41 +++++++++++++--- internal/discovery/client.go | 49 +++++++++++++++++++ internal/discovery/mapper.go | 21 ++++++++ 5 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 internal/discovery/client.go create mode 100644 internal/discovery/mapper.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 99da5d5..93ec890 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -27,10 +27,17 @@ type TargetSourceSpec struct { ConfigMap string `json:"configMap,omitempty"` PodSelector metav1.LabelSelector `json:"podSelector,omitempty"` ServiceSelector metav1.LabelSelector `json:"serviceSelector,omitempty"` + Manual []ManualTarget `json:"manual,omitempty"` // Labels map[string]string `json:"labels,omitempty"` } +type ManualTarget struct { + Name string `json:"name,omitempty"` + Address string `json:"address,omitempty"` + TargetProfile string `json:"targetProfile,omitempty"` +} + type HTTPConfig struct { URL string `json:"url,omitempty"` } diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 789ff3f..a4daeb8 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -55,6 +55,17 @@ spec: additionalProperties: type: string type: object + manual: + items: + properties: + address: + type: string + name: + type: string + targetProfile: + type: string + type: object + type: array podSelector: description: |- A label selector is a label query over a set of resources. The result of matchLabels and diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 032b103..60bb8af 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -22,9 +22,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/discovery" ) // TargetSourceReconciler reconciles a TargetSource object @@ -50,12 +52,39 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request logger.Info("reconciling TargetSource", "name", targetSource.Name) - // TODO: Implement target discovery logic based on spec: - // - HTTP: fetch targets from HTTP endpoint - // - Consul: discover from Consul - // - ConfigMap: read from ConfigMap - // - PodSelector: select Kubernetes pods - // - ServiceSelector: select Kubernetes services + // VD: Approach for the reconciliation loop: + // 1. Fetch objects from TargetSource + // 2. Build desired state + // 3. Get actual state (only targets owned by TargetSource) + // 4. Compute diff + // 5. Apply changes (create, update, delete) + + discoveredTargets, err := discovery.FetchNewTargets(ctx, targetSource) + if err != nil { + logger.Error(err, "error getting discovered targets") + return ctrl.Result{}, err + } + + existingTargets, err := discovery.GetExistingTargets(ctx, r.Client, targetSource) + if err != nil { + logger.Error(err, "error fetching existing targets") + return ctrl.Result{}, err + } + + newTargets, err := discovery.GetNewTargets(existingTargets, discoveredTargets) + + for _, t := range newTargets { + err = controllerutil.SetControllerReference(&targetSource, &t, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + + err = r.Client.Create(ctx, &t) + if err != nil { + logger.Error(err, "error when creating target") + return ctrl.Result{}, err + } + } return ctrl.Result{}, nil } diff --git a/internal/discovery/client.go b/internal/discovery/client.go new file mode 100644 index 0000000..61a94dd --- /dev/null +++ b/internal/discovery/client.go @@ -0,0 +1,49 @@ +package discovery + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +func FetchNewTargets(ctx context.Context, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { + var targets []gnmicv1alpha1.Target + + for _, e := range ts.Spec.Manual { + target := &gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: e.Name, + Namespace: ts.Namespace, + Labels: map[string]string{ + "gnmic.io/source": ts.Name, + }, + }, + Spec: gnmicv1alpha1.TargetSpec{ + Address: e.Address, + Profile: e.TargetProfile, + }, + } + targets = append(targets, *target) + } + + return targets, nil +} + +func GetExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList + + err := c.List(ctx, &targetList, + client.InNamespace(ts.Namespace), + client.MatchingLabels{ + "gnmic.io/source": ts.Name, + }, + ) + if err != nil { + return nil, err + } + + return targetList.Items, nil +} diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go new file mode 100644 index 0000000..744a98d --- /dev/null +++ b/internal/discovery/mapper.go @@ -0,0 +1,21 @@ +package discovery + +import ( + "slices" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +func GetNewTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { + var new []gnmicv1alpha1.Target + + for _, t := range discovered { + if !slices.ContainsFunc(existing, func(e gnmicv1alpha1.Target) bool { + return e.ObjectMeta.Name == t.ObjectMeta.Name && e.ObjectMeta.Namespace == t.ObjectMeta.Namespace + }) { + new = append(new, t) + } + } + + return new, nil +} From 95be5ab5262f30fafd94cef7d298dccddb5603d8 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 18:52:18 -0600 Subject: [PATCH 004/246] added deletion of targets --- internal/controller/targetsource_controller.go | 10 ++++++++++ internal/discovery/mapper.go | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 60bb8af..63ee455 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -86,6 +86,16 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } } + deletedTargets, err := discovery.GetDeletedTargets(existingTargets, discoveredTargets) + + for _, t := range deletedTargets { + err = r.Client.Delete(ctx, &t) + if err != nil { + logger.Error(err, "error deleting the object") + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil } diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index 744a98d..6fa7f9f 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -19,3 +19,17 @@ func GetNewTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1 return new, nil } + +func GetDeletedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { + var deleted []gnmicv1alpha1.Target + + for _, e := range existing { + if !slices.ContainsFunc(discovered, func(d gnmicv1alpha1.Target) bool { + return d.ObjectMeta.Name == e.ObjectMeta.Name && d.ObjectMeta.Namespace == e.ObjectMeta.Namespace + }) { + deleted = append(deleted, e) + } + } + + return deleted, nil +} From 4e3104e80689c40d53c71b8ede6ae0d9fa460a08 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 20:07:57 -0600 Subject: [PATCH 005/246] renamed client functions --- internal/controller/targetsource_controller.go | 4 ++-- internal/discovery/client.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 63ee455..aaf758c 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -59,13 +59,13 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request // 4. Compute diff // 5. Apply changes (create, update, delete) - discoveredTargets, err := discovery.FetchNewTargets(ctx, targetSource) + discoveredTargets, err := discovery.FetchDiscoveryTargets(ctx, targetSource) if err != nil { logger.Error(err, "error getting discovered targets") return ctrl.Result{}, err } - existingTargets, err := discovery.GetExistingTargets(ctx, r.Client, targetSource) + existingTargets, err := discovery.FetchExistingTargets(ctx, r.Client, targetSource) if err != nil { logger.Error(err, "error fetching existing targets") return ctrl.Result{}, err diff --git a/internal/discovery/client.go b/internal/discovery/client.go index 61a94dd..fccb571 100644 --- a/internal/discovery/client.go +++ b/internal/discovery/client.go @@ -9,7 +9,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" ) -func FetchNewTargets(ctx context.Context, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { +func FetchDiscoveryTargets(ctx context.Context, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { var targets []gnmicv1alpha1.Target for _, e := range ts.Spec.Manual { @@ -32,7 +32,7 @@ func FetchNewTargets(ctx context.Context, ts gnmicv1alpha1.TargetSource) ([]gnmi return targets, nil } -func GetExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { +func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { var targetList gnmicv1alpha1.TargetList err := c.List(ctx, &targetList, From 32eb936c9ac57918e531a7b8a1f187464dfc26e9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 20:31:34 -0600 Subject: [PATCH 006/246] added update functionality --- .../controller/targetsource_controller.go | 25 +++++++++++++++++++ internal/discovery/mapper.go | 23 +++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index aaf758c..907d9b0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -20,6 +20,7 @@ import ( "context" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -86,6 +87,30 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } } + updatedTargets, err := discovery.GetUpdatedTargets(existingTargets, discoveredTargets) + + for _, t := range updatedTargets { + existing := &gnmicv1alpha1.Target{} + + err := r.Get(ctx, types.NamespacedName{ + Name: t.ObjectMeta.Name, + Namespace: t.ObjectMeta.Namespace, + }, existing) + + if err != nil { + logger.Error(err, "error fetching existing object") + return ctrl.Result{}, err + } + + existing.Spec = t.Spec + + err = r.Update(ctx, existing) + if err != nil { + logger.Error(err, "error updating object") + return ctrl.Result{}, err + } + } + deletedTargets, err := discovery.GetDeletedTargets(existingTargets, discoveredTargets) for _, t := range deletedTargets { diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index 6fa7f9f..a4ba5e6 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -20,6 +20,29 @@ func GetNewTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1 return new, nil } +func GetUpdatedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { + var updated []gnmicv1alpha1.Target + + existingMap := make(map[string]gnmicv1alpha1.Target) + + for _, e := range existing { + key := e.Namespace + "/" + e.Name + existingMap[key] = e + } + + for _, t := range discovered { + key := t.Namespace + "/" + t.Name + + if e, found := existingMap[key]; found { + if e.Spec.Address != t.Spec.Address || e.Spec.Profile != t.Spec.Address { + updated = append(updated, t) + } + } + } + + return updated, nil +} + func GetDeletedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { var deleted []gnmicv1alpha1.Target From 3c4141f479ed76716be27585826ae716961c73dc Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 20:54:15 -0600 Subject: [PATCH 007/246] refactored crud options into separate function with struct --- .../controller/targetsource_controller.go | 12 ++---- internal/discovery/mapper.go | 38 +++++++------------ 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 907d9b0..a76836d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -72,9 +72,9 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - newTargets, err := discovery.GetNewTargets(existingTargets, discoveredTargets) + diff := discovery.BuildDiff(existingTargets, discoveredTargets) - for _, t := range newTargets { + for _, t := range diff.ToCreate { err = controllerutil.SetControllerReference(&targetSource, &t, r.Scheme) if err != nil { return ctrl.Result{}, err @@ -87,9 +87,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } } - updatedTargets, err := discovery.GetUpdatedTargets(existingTargets, discoveredTargets) - - for _, t := range updatedTargets { + for _, t := range diff.ToUpdate { existing := &gnmicv1alpha1.Target{} err := r.Get(ctx, types.NamespacedName{ @@ -111,9 +109,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } } - deletedTargets, err := discovery.GetDeletedTargets(existingTargets, discoveredTargets) - - for _, t := range deletedTargets { + for _, t := range diff.ToDelete { err = r.Client.Delete(ctx, &t) if err != nil { logger.Error(err, "error deleting the object") diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index a4ba5e6..f8d6468 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -4,27 +4,19 @@ import ( "slices" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/equality" ) -func GetNewTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { - var new []gnmicv1alpha1.Target - - for _, t := range discovered { - if !slices.ContainsFunc(existing, func(e gnmicv1alpha1.Target) bool { - return e.ObjectMeta.Name == t.ObjectMeta.Name && e.ObjectMeta.Namespace == t.ObjectMeta.Namespace - }) { - new = append(new, t) - } - } - - return new, nil +type Diff struct { + ToCreate []gnmicv1alpha1.Target + ToUpdate []gnmicv1alpha1.Target + ToDelete []gnmicv1alpha1.Target } -func GetUpdatedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { - var updated []gnmicv1alpha1.Target +func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { + var diff Diff existingMap := make(map[string]gnmicv1alpha1.Target) - for _, e := range existing { key := e.Namespace + "/" + e.Name existingMap[key] = e @@ -34,25 +26,21 @@ func GetUpdatedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1al key := t.Namespace + "/" + t.Name if e, found := existingMap[key]; found { - if e.Spec.Address != t.Spec.Address || e.Spec.Profile != t.Spec.Address { - updated = append(updated, t) + if !equality.Semantic.DeepEqual(e.Spec, t.Spec) { + diff.ToUpdate = append(diff.ToUpdate, t) } + } else { + diff.ToCreate = append(diff.ToCreate, t) } } - return updated, nil -} - -func GetDeletedTargets(existing, discovered []gnmicv1alpha1.Target) ([]gnmicv1alpha1.Target, error) { - var deleted []gnmicv1alpha1.Target - for _, e := range existing { if !slices.ContainsFunc(discovered, func(d gnmicv1alpha1.Target) bool { return d.ObjectMeta.Name == e.ObjectMeta.Name && d.ObjectMeta.Namespace == e.ObjectMeta.Namespace }) { - deleted = append(deleted, e) + diff.ToDelete = append(diff.ToDelete, e) } } - return deleted, nil + return diff } From 590a494d3b02cab3f6136b2635672fac31e2e239 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 20:57:36 -0600 Subject: [PATCH 008/246] refactored delete to use map lookup --- internal/discovery/mapper.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index f8d6468..6d1c271 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -1,8 +1,6 @@ package discovery import ( - "slices" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/api/equality" ) @@ -22,6 +20,12 @@ func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { existingMap[key] = e } + discoveredMap := make(map[string]gnmicv1alpha1.Target) + for _, e := range discovered { + key := e.Namespace + "/" + e.Name + discoveredMap[key] = e + } + for _, t := range discovered { key := t.Namespace + "/" + t.Name @@ -35,9 +39,9 @@ func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { } for _, e := range existing { - if !slices.ContainsFunc(discovered, func(d gnmicv1alpha1.Target) bool { - return d.ObjectMeta.Name == e.ObjectMeta.Name && d.ObjectMeta.Namespace == e.ObjectMeta.Namespace - }) { + key := e.Namespace + "/" + e.Name + + if e, found := discoveredMap[key]; !found { diff.ToDelete = append(diff.ToDelete, e) } } From 26669868c647484ee389232a6babf8adc315d2f8 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 21:01:09 -0600 Subject: [PATCH 009/246] added comments --- internal/controller/targetsource_controller.go | 11 +++++++---- internal/discovery/mapper.go | 6 +++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index a76836d..cfc0502 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -55,25 +55,28 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request // VD: Approach for the reconciliation loop: // 1. Fetch objects from TargetSource - // 2. Build desired state - // 3. Get actual state (only targets owned by TargetSource) - // 4. Compute diff - // 5. Apply changes (create, update, delete) + // 2. Get actual state (only targets owned by TargetSource) + // 3. Compute diff + // 4. Apply changes (create, update, delete) + // Step 1: Get desired state from discovery source discoveredTargets, err := discovery.FetchDiscoveryTargets(ctx, targetSource) if err != nil { logger.Error(err, "error getting discovered targets") return ctrl.Result{}, err } + // Step 2: Get current state from Kubernetes cluster (lookup by label of TargetSource) existingTargets, err := discovery.FetchExistingTargets(ctx, r.Client, targetSource) if err != nil { logger.Error(err, "error fetching existing targets") return ctrl.Result{}, err } + // Step 3: Compute diff diff := discovery.BuildDiff(existingTargets, discoveredTargets) + // Step 4: Iterate over each list and do create, update, delete respectively for _, t := range diff.ToCreate { err = controllerutil.SetControllerReference(&targetSource, &t, r.Scheme) if err != nil { diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index 6d1c271..1ec2931 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -26,18 +26,22 @@ func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { discoveredMap[key] = e } + // Loop for targets to create + update for _, t := range discovered { key := t.Namespace + "/" + t.Name + // Check if target already exists if e, found := existingMap[key]; found { + // Check if the spec of the target changed if !equality.Semantic.DeepEqual(e.Spec, t.Spec) { diff.ToUpdate = append(diff.ToUpdate, t) } - } else { + } else { // Target is new diff.ToCreate = append(diff.ToCreate, t) } } + // Loop for targets to delete for _, e := range existing { key := e.Namespace + "/" + e.Name From b9cd85ef6ca79c4672cb4ff4d337ff11c25bb211 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 21:23:03 -0600 Subject: [PATCH 010/246] fixed bug with deleted object name being empty --- internal/discovery/mapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/discovery/mapper.go b/internal/discovery/mapper.go index 1ec2931..dc686a4 100644 --- a/internal/discovery/mapper.go +++ b/internal/discovery/mapper.go @@ -45,7 +45,7 @@ func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { for _, e := range existing { key := e.Namespace + "/" + e.Name - if e, found := discoveredMap[key]; !found { + if _, found := discoveredMap[key]; !found { diff.ToDelete = append(diff.ToDelete, e) } } From 109a85e504a7c02b0ee74dbc8bc981843a45a2ed Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 6 Apr 2026 21:23:14 -0600 Subject: [PATCH 011/246] added info logs --- internal/controller/targetsource_controller.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index cfc0502..d863e48 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -80,14 +81,16 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request for _, t := range diff.ToCreate { err = controllerutil.SetControllerReference(&targetSource, &t, r.Scheme) if err != nil { + logger.Error(err, "error setting the owner reference") return ctrl.Result{}, err } err = r.Client.Create(ctx, &t) if err != nil { - logger.Error(err, "error when creating target") + logger.Error(err, "error creating target object") return ctrl.Result{}, err } + logger.Info(fmt.Sprintf("created new target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) } for _, t := range diff.ToUpdate { @@ -99,7 +102,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request }, existing) if err != nil { - logger.Error(err, "error fetching existing object") + logger.Error(err, "error fetching existing target object") return ctrl.Result{}, err } @@ -110,14 +113,17 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request logger.Error(err, "error updating object") return ctrl.Result{}, err } + logger.Info(fmt.Sprintf("updated existing target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) } for _, t := range diff.ToDelete { err = r.Client.Delete(ctx, &t) + logger.Info(fmt.Sprintf("resource name to be deleted: %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) if err != nil { logger.Error(err, "error deleting the object") return ctrl.Result{}, err } + logger.Info(fmt.Sprintf("deleted target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) } return ctrl.Result{}, nil From 5e9aedc8e90d411ea220efede79413e53a7815b4 Mon Sep 17 00:00:00 2001 From: Janooski Date: Tue, 7 Apr 2026 20:08:55 +0000 Subject: [PATCH 012/246] change to gin --- cmd/main.go | 70 ++++++++++++------------ config/default/manager_config_patch.yaml | 10 +++- internal/apiserver/apiserver.go | 27 +++++++-- 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d9758fa..496e4a7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,8 +17,12 @@ limitations under the License. package main import ( + "context" + "errors" "flag" + "net/http" "os" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,8 +33,8 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -64,7 +68,7 @@ func main() { flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "Disabled: health and readiness probes are served by the gin API server.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -206,41 +210,35 @@ func main() { } //+kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - if apiAddr != "" { - apiserver.New(apiAddr, clusterReconciler) - - } - - // if apiAddr != "" { - // apiServer := apiserver.New(apiAddr, clusterReconciler) - // err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - // errCh := make(chan error) - // go func() { - // errCh <- apiServer.Server.ListenAndServe() - // }() - // select { - // case err := <-errCh: - // return err - // case <-ctx.Done(): - // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - // defer cancel() - // return apiServer.Server.Shutdown(ctx) - // } - // })) - // if err != nil { - // setupLog.Error(err, "unable to add api server") - // os.Exit(1) - // } - // } + apiServer := apiserver.New(apiAddr, clusterReconciler) + err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + errCh := make(chan error, 1) + go func() { + err := apiServer.Server.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + close(errCh) + }() + + select { + case err, ok := <-errCh: + if !ok { + return nil + } + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return apiServer.Server.Shutdown(shutdownCtx) + } + })) + if err != nil { + setupLog.Error(err, "unable to add api server") + os.Exit(1) + } + } // start manager setupLog.Info("starting manager") diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 691d2c3..d88bad0 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -9,10 +9,18 @@ spec: containers: - name: manager args: - - "--health-probe-bind-address=:8081" + - "--health-probe-bind-address=0" - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" + livenessProbe: + httpGet: + path: /healthz + port: 8082 + readinessProbe: + httpGet: + path: /readyz + port: 8082 ports: - containerPort: 8080 protocol: TCP diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 7370be9..5ffa29d 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -2,29 +2,44 @@ package apiserver import ( "log" + "net/http" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" ) type APIServer struct { - Server *gin.Engine + Server *http.Server + router *gin.Engine clusterReconciler *controller.ClusterReconciler } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { - _ = addr + router := gin.Default() a := &APIServer{ - Server: gin.Default(), + Server: &http.Server{ + Addr: addr, + Handler: router, + }, + router: router, clusterReconciler: clusterReconciler, } - a.Server.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) - a.Server.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) - a.Server.Run(":8082") + a.router.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) + a.router.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) + a.router.GET("/healthz", a.getHealthz) + a.router.GET("/readyz", a.getReadyz) return a } +func (a *APIServer) getHealthz(c *gin.Context) { + c.Status(http.StatusOK) +} + +func (a *APIServer) getReadyz(c *gin.Context) { + c.Status(http.StatusOK) +} + func (a *APIServer) getClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") From b7e12d7b6e9c958bf523614c931a6c7fe7f1acd8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 18:42:37 +0000 Subject: [PATCH 013/246] adding openapi.yaml and generated code, still has bugs --- cmd/main.go | 8 +- go.mod | 16 +++ go.sum | 125 ++++++++++++++++++++++ internal/apiserver/apiserver.go | 29 +++-- internal/apiserver/cfg.yaml | 6 ++ internal/apiserver/gen.go | 184 ++++++++++++++++++++++++++++++++ internal/apiserver/openapi.yaml | 45 ++++++++ 7 files changed, 399 insertions(+), 14 deletions(-) create mode 100644 internal/apiserver/cfg.yaml create mode 100644 internal/apiserver/gen.go create mode 100644 internal/apiserver/openapi.yaml diff --git a/cmd/main.go b/cmd/main.go index 496e4a7..ba4c3a3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -211,11 +211,13 @@ func main() { //+kubebuilder:scaffold:builder if apiAddr != "" { - apiServer := apiserver.New(apiAddr, clusterReconciler) + server, r := apiserver.New(apiAddr, clusterReconciler) + apiserver.RegisterHandlers(r, server) + err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { - err := apiServer.Server.ListenAndServe() + err := server.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { errCh <- err } @@ -231,7 +233,7 @@ func main() { case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return apiServer.Server.Shutdown(shutdownCtx) + return server.Server.Shutdown(shutdownCtx) } })) if err != nil { diff --git a/go.mod b/go.mod index 208f424..36c110d 100644 --- a/go.mod +++ b/go.mod @@ -22,24 +22,38 @@ require ( github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/getkin/kin-openapi v0.133.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect + github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect + github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect + github.com/speakeasy-api/jsonpath v0.6.0 // indirect + github.com/speakeasy-api/openapi-overlay v0.10.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect + github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect + github.com/woodsbury/decimal128 v1.3.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/crypto v0.48.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -112,3 +126,5 @@ require ( sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen diff --git a/go.sum b/go.sum index 8b1369f..c9059d3 100644 --- a/go.sum +++ b/go.sum @@ -14,24 +14,34 @@ github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3 github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= +github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= @@ -90,27 +100,46 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -119,14 +148,19 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -139,16 +173,38 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= github.com/openconfig/gnmic/pkg/api v0.1.10/go.mod h1:6PntONfjCMq3XzsDfWMkLeoVuBRbkm2foQO5m6PeYo0= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -168,6 +224,11 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/speakeasy-api/jsonpath v0.6.0 h1:IhtFOV9EbXplhyRqsVhHoBmmYjblIRh5D1/g8DHMXJ8= +github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= +github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj6gST0P3SGTX95GU= +github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -176,6 +237,8 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -194,8 +257,13 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= +github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= +github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -224,27 +292,68 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -253,17 +362,33 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 5ffa29d..0b7d278 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,6 +1,9 @@ package apiserver +//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml + import ( + "fmt" "log" "net/http" @@ -14,7 +17,7 @@ type APIServer struct { clusterReconciler *controller.ClusterReconciler } -func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { +func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -24,12 +27,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServe router: router, clusterReconciler: clusterReconciler, } - - a.router.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) - a.router.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) - a.router.GET("/healthz", a.getHealthz) - a.router.GET("/readyz", a.getReadyz) - return a + return a, router } func (a *APIServer) getHealthz(c *gin.Context) { @@ -40,7 +38,7 @@ func (a *APIServer) getReadyz(c *gin.Context) { c.Status(http.StatusOK) } -func (a *APIServer) getClusterPlan(c *gin.Context) { +func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) @@ -54,7 +52,16 @@ func (a *APIServer) getClusterPlan(c *gin.Context) { // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget -func (a *APIServer) postCreateTarget(c *gin.Context) { - log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - c.Status(202) +func (a *APIServer) CreateTargets(c *gin.Context) { + var payload []Target + if err := c.ShouldBindJSON(&payload); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + // testing + for _, target := range payload { + fmt.Printf("name: %s, address: %s, profile: %s, tags: %s\n", *target.Name, *target.Address, *target.Profile, *target.Tags) + } + + c.JSON(http.StatusOK, payload) } diff --git a/internal/apiserver/cfg.yaml b/internal/apiserver/cfg.yaml new file mode 100644 index 0000000..4bc7f02 --- /dev/null +++ b/internal/apiserver/cfg.yaml @@ -0,0 +1,6 @@ +package: apiserver +output: gen.go +generate: + gin-server: true + models: true + embedded-spec: true \ No newline at end of file diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go new file mode 100644 index 0000000..88ed5ab --- /dev/null +++ b/internal/apiserver/gen.go @@ -0,0 +1,184 @@ +// Package apiserver provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.6.0 DO NOT EDIT. +package apiserver + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "net/url" + "path" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gin-gonic/gin" +) + +// Target defines model for Target. +type Target struct { + Address *string `json:"address,omitempty"` + Name *string `json:"name,omitempty"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` +} + +// ServerInterface represents all server handlers. +type ServerInterface interface { + // Create targets in the gNMIc Operator + // (POST /createTargets) + CreateTargets(c *gin.Context) + // Get cluster plan + // (GET /plan) + GetClusterPlan(c *gin.Context) +} + +// ServerInterfaceWrapper converts contexts to parameters. +type ServerInterfaceWrapper struct { + Handler ServerInterface + HandlerMiddlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +type MiddlewareFunc func(c *gin.Context) + +// CreateTargets operation middleware +func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.CreateTargets(c) +} + +// GetClusterPlan operation middleware +func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetClusterPlan(c) +} + +// GinServerOptions provides options for the Gin server. +type GinServerOptions struct { + BaseURL string + Middlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +// RegisterHandlers creates http.Handler with routing matching OpenAPI spec. +func RegisterHandlers(router gin.IRouter, si ServerInterface) { + RegisterHandlersWithOptions(router, si, GinServerOptions{}) +} + +// RegisterHandlersWithOptions creates http.Handler with additional options +func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options GinServerOptions) { + errorHandler := options.ErrorHandler + if errorHandler == nil { + errorHandler = func(c *gin.Context, err error, statusCode int) { + c.JSON(statusCode, gin.H{"msg": err.Error()}) + } + } + + wrapper := ServerInterfaceWrapper{ + Handler: si, + HandlerMiddlewares: options.Middlewares, + ErrorHandler: errorHandler, + } + + router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) + router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) +} + +// Base64 encoded, gzipped, json marshaled Swagger object +var swaggerSpec = []string{ + + "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", + "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", + "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", + "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", + "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", + "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", +} + +// GetSwagger returns the content of the embedded swagger specification file +// or error if failed to decode +func decodeSpec() ([]byte, error) { + zipped, err := base64.StdEncoding.DecodeString(strings.Join(swaggerSpec, "")) + if err != nil { + return nil, fmt.Errorf("error base64 decoding spec: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(zipped)) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + var buf bytes.Buffer + _, err = buf.ReadFrom(zr) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + + return buf.Bytes(), nil +} + +var rawSpec = decodeSpecCached() + +// a naive cached of a decoded swagger spec +func decodeSpecCached() func() ([]byte, error) { + data, err := decodeSpec() + return func() ([]byte, error) { + return data, err + } +} + +// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. +func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { + res := make(map[string]func() ([]byte, error)) + if len(pathToFile) > 0 { + res[pathToFile] = rawSpec + } + + return res +} + +// GetSwagger returns the Swagger specification corresponding to the generated code +// in this file. The external references of Swagger specification are resolved. +// The logic of resolving external references is tightly connected to "import-mapping" feature. +// Externally referenced files must be embedded in the corresponding golang packages. +// Urls can be supported but this task was out of the scope. +func GetSwagger() (swagger *openapi3.T, err error) { + resolvePath := PathToRawSpec("") + + loader := openapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { + pathToFile := url.String() + pathToFile = path.Clean(pathToFile) + getSpec, ok := resolvePath[pathToFile] + if !ok { + err1 := fmt.Errorf("path not found: %s", pathToFile) + return nil, err1 + } + return getSpec() + } + var specData []byte + specData, err = rawSpec() + if err != nil { + return + } + swagger, err = loader.LoadFromData(specData) + if err != nil { + return + } + return +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml new file mode 100644 index 0000000..fccf33e --- /dev/null +++ b/internal/apiserver/openapi.yaml @@ -0,0 +1,45 @@ +openapi: 3.2.0 +info: + title: "gNMIc Operator REST API" + version: "0.0.1" +paths: + /plan: + get: + summary: "Get cluster plan" + operationId: "getClusterPlan" + responses: + '200': + description: "ClusterPlan returned" + /createTargets: + post: + summary: "Create targets in the gNMIc Operator" + operationId: "createTargets" + responses: + '201': + description: "Targets created successfully" + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Target' + +components: + schemas: + Target: + type: object + properties: + name: + type: string + address: + type: string # in the format "IP:port" + profile: + type: string + tags: + type: array + items: + type: string + # username: + # type: string + # password: # not sure if the password is needed + # type: string \ No newline at end of file From 5e50abaea49ea05d8f0fb7ae752e4b09bd0b0c7e Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 20:14:12 +0000 Subject: [PATCH 014/246] add healthz and readyz interfaces to avoid pod crashing --- internal/apiserver/apiserver.go | 4 +-- internal/apiserver/gen.go | 47 ++++++++++++++++++++++++++++----- internal/apiserver/openapi.yaml | 12 +++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 0b7d278..6d2ca5b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -30,11 +30,11 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ return a, router } -func (a *APIServer) getHealthz(c *gin.Context) { +func (a *APIServer) GetHealthz(c *gin.Context) { c.Status(http.StatusOK) } -func (a *APIServer) getReadyz(c *gin.Context) { +func (a *APIServer) GetReadyz(c *gin.Context) { c.Status(http.StatusOK) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 88ed5ab..77b2ecb 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -29,9 +29,15 @@ type ServerInterface interface { // Create targets in the gNMIc Operator // (POST /createTargets) CreateTargets(c *gin.Context) + + // (GET /healthz) + GetHealthz(c *gin.Context) // Get cluster plan // (GET /plan) GetClusterPlan(c *gin.Context) + + // (GET /readyz) + GetReadyz(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -56,6 +62,19 @@ func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { siw.Handler.CreateTargets(c) } +// GetHealthz operation middleware +func (siw *ServerInterfaceWrapper) GetHealthz(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetHealthz(c) +} + // GetClusterPlan operation middleware func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { @@ -69,6 +88,19 @@ func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { siw.Handler.GetClusterPlan(c) } +// GetReadyz operation middleware +func (siw *ServerInterfaceWrapper) GetReadyz(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetReadyz(c) +} + // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -97,18 +129,21 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options } router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) + router.GET(options.BaseURL+"/healthz", wrapper.GetHealthz) router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) + router.GET(options.BaseURL+"/readyz", wrapper.GetReadyz) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", - "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", - "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", - "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", - "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", - "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", + "H4sIAAAAAAAC/4xSwW7UMBD9FWvgGG3ScssNVajsAahKf2BwZhNXiW2NJ0ihyr8jTwxL6KraXGyN35t5", + "701ewIYpBk9eErQvkOxAE+r1CbknybfIIRKLI61j1zElvcoSCVpIws73sFbgcaKLD5HDyY2X3wR77eaE", + "psttSwGZcYH1XAg/nskKrLnk/Cko2UmeA/3XL0drvkVilMDm8dP3J/Px4QgV/CROLnhooTk0h5s8IETy", + "GB208OFwe2iggogyqJjaMqHQloZWYkgaS9DeLvhjBy3c7WAVMKUYfNoyu21u8mGDF/JKxhhHZ5VeP6es", + "5k/4uyjeM52ghXf1eU112VFdFvQ6n7WCjpJlF2XzWVSZzUpn0mwtpXSax3HLM83ThLz8tWGkMJw3MpDZ", + "h6mUeiAcZfiVVZb/ZB/IPcnnAnmVRpOPvcb+DF/VQh1H9G91vxvnJMQPGXbNhH/whklm9tT95/6exNgN", + "ZnS8CmHCbnnT6OOGuNJnQa/6/Q4AAP//KM8oJIIDAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index fccf33e..9471a8b 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -3,6 +3,18 @@ info: title: "gNMIc Operator REST API" version: "0.0.1" paths: + /readyz: + get: + operationId: "getReadyz" + responses: + '200': + description: "getReadyz" + /healthz: + get: + operationId: "getHealthz" + responses: + '200': + description: "getHealthz" /plan: get: summary: "Get cluster plan" From 0f230c2f848284e6a652189f07802e3010924342 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 20:51:58 +0000 Subject: [PATCH 015/246] move healthz and readyz back to port 8081 --- cmd/main.go | 14 ++++++- config/default/manager_config_patch.yaml | 6 +-- internal/apiserver/apiserver.go | 8 ---- internal/apiserver/gen.go | 47 +++--------------------- internal/apiserver/openapi.yaml | 12 ------ 5 files changed, 21 insertions(+), 66 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ba4c3a3..51dba83 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -65,10 +66,10 @@ func main() { var probeAddr string var devMode bool var apiAddr string - flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") + flag.StringVar(&apiAddr, "api-bind-address", ":8082", "The address the operator API endpoint binds to.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "Disabled: health and readiness probes are served by the gin API server.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -210,6 +211,15 @@ func main() { } //+kubebuilder:scaffold:builder + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + if apiAddr != "" { server, r := apiserver.New(apiAddr, clusterReconciler) apiserver.RegisterHandlers(r, server) diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index d88bad0..c6d4e57 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -9,18 +9,18 @@ spec: containers: - name: manager args: - - "--health-probe-bind-address=0" + - "--health-probe-bind-address=:8081" - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" livenessProbe: httpGet: path: /healthz - port: 8082 + port: 8081 readinessProbe: httpGet: path: /readyz - port: 8082 + port: 8081 ports: - containerPort: 8080 protocol: TCP diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 6d2ca5b..3ab21d1 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -30,14 +30,6 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ return a, router } -func (a *APIServer) GetHealthz(c *gin.Context) { - c.Status(http.StatusOK) -} - -func (a *APIServer) GetReadyz(c *gin.Context) { - c.Status(http.StatusOK) -} - func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 77b2ecb..88ed5ab 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -29,15 +29,9 @@ type ServerInterface interface { // Create targets in the gNMIc Operator // (POST /createTargets) CreateTargets(c *gin.Context) - - // (GET /healthz) - GetHealthz(c *gin.Context) // Get cluster plan // (GET /plan) GetClusterPlan(c *gin.Context) - - // (GET /readyz) - GetReadyz(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -62,19 +56,6 @@ func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { siw.Handler.CreateTargets(c) } -// GetHealthz operation middleware -func (siw *ServerInterfaceWrapper) GetHealthz(c *gin.Context) { - - for _, middleware := range siw.HandlerMiddlewares { - middleware(c) - if c.IsAborted() { - return - } - } - - siw.Handler.GetHealthz(c) -} - // GetClusterPlan operation middleware func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { @@ -88,19 +69,6 @@ func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { siw.Handler.GetClusterPlan(c) } -// GetReadyz operation middleware -func (siw *ServerInterfaceWrapper) GetReadyz(c *gin.Context) { - - for _, middleware := range siw.HandlerMiddlewares { - middleware(c) - if c.IsAborted() { - return - } - } - - siw.Handler.GetReadyz(c) -} - // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -129,21 +97,18 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options } router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) - router.GET(options.BaseURL+"/healthz", wrapper.GetHealthz) router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) - router.GET(options.BaseURL+"/readyz", wrapper.GetReadyz) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4xSwW7UMBD9FWvgGG3ScssNVajsAahKf2BwZhNXiW2NJ0ihyr8jTwxL6KraXGyN35t5", - "701ewIYpBk9eErQvkOxAE+r1CbknybfIIRKLI61j1zElvcoSCVpIws73sFbgcaKLD5HDyY2X3wR77eaE", - "psttSwGZcYH1XAg/nskKrLnk/Cko2UmeA/3XL0drvkVilMDm8dP3J/Px4QgV/CROLnhooTk0h5s8IETy", - "GB208OFwe2iggogyqJjaMqHQloZWYkgaS9DeLvhjBy3c7WAVMKUYfNoyu21u8mGDF/JKxhhHZ5VeP6es", - "5k/4uyjeM52ghXf1eU112VFdFvQ6n7WCjpJlF2XzWVSZzUpn0mwtpXSax3HLM83ThLz8tWGkMJw3MpDZ", - "h6mUeiAcZfiVVZb/ZB/IPcnnAnmVRpOPvcb+DF/VQh1H9G91vxvnJMQPGXbNhH/whklm9tT95/6exNgN", - "ZnS8CmHCbnnT6OOGuNJnQa/6/Q4AAP//KM8oJIIDAAA=", + "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", + "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", + "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", + "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", + "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", + "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 9471a8b..fccf33e 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -3,18 +3,6 @@ info: title: "gNMIc Operator REST API" version: "0.0.1" paths: - /readyz: - get: - operationId: "getReadyz" - responses: - '200': - description: "getReadyz" - /healthz: - get: - operationId: "getHealthz" - responses: - '200': - description: "getHealthz" /plan: get: summary: "Get cluster plan" From 4d3dc8d949514a4a09da8707e8d805119aebcf56 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 21:41:21 +0000 Subject: [PATCH 016/246] url paths dynamic --- cmd/main.go | 15 +++++++++++---- config/manager/manager.yaml | 9 +++++++++ go.mod | 2 ++ go.sum | 8 ++++++++ helm/templates/deployment.yaml | 13 +++++++++++++ internal/apiserver/apiserver.go | 10 +++++++--- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 51dba83..be73c0f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -221,13 +221,20 @@ func main() { } if apiAddr != "" { - server, r := apiserver.New(apiAddr, clusterReconciler) - apiserver.RegisterHandlers(r, server) + apiNamespace := os.Getenv("POD_NAMESPACE") + apiClusterName := os.Getenv("CLUSTER_NAME") + if apiNamespace == "" || apiClusterName == "" { + setupLog.Error(errors.New("missing runtime API identity"), "POD_NAMESPACE and CLUSTER_NAME must be set") + os.Exit(1) + } + apiBaseURL := "/api/v1/" + apiNamespace + "/" + apiClusterName + api, gin := apiserver.New(apiAddr, apiNamespace, apiClusterName, clusterReconciler) + apiserver.RegisterHandlersWithOptions(gin, api, apiserver.GinServerOptions{BaseURL: apiBaseURL}) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { - err := server.Server.ListenAndServe() + err := api.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { errCh <- err } @@ -243,7 +250,7 @@ func main() { case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return server.Server.Shutdown(shutdownCtx) + return api.Server.Shutdown(shutdownCtx) } })) if err != nil { diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2cd79f0..900f373 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -73,6 +73,15 @@ spec: - --leader-elect image: controller:latest name: manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/go.mod b/go.mod index 36c110d..d268635 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( ) require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -38,6 +39,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect + github.com/oapi-codegen/runtime v1.3.1 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index c9059d3..81a33fb 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -144,6 +148,7 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -181,6 +186,8 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oapi-codegen/runtime v1.3.1 h1:RgDY6J4OGQLbRXhG/Xpt3vSVqYpHQS7hN4m85+5xB9g= +github.com/oapi-codegen/runtime v1.3.1/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= @@ -231,6 +238,7 @@ github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index b85e661..250c873 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -41,6 +41,19 @@ spec: {{- if .Values.api.port }} - --api-bind-address=:{{ .Values.api.port }} {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] ports: {{- if .Values.webhook.enabled }} - name: webhook diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 3ab21d1..a47fbf9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -15,9 +15,11 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler + namespace string + clusterName string } -func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { +func New(addr string, namespace string, clusterName string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -26,14 +28,15 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ }, router: router, clusterReconciler: clusterReconciler, + namespace: namespace, + clusterName: clusterName, } return a, router } func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - namespace, name := c.Param("namespace"), c.Param("name") - plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) + plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) if err != nil { c.String(404, err.Error()) return @@ -45,6 +48,7 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget func (a *APIServer) CreateTargets(c *gin.Context) { + log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) From 34be165c89d8be83f5f35300a1a031e5ee36b4e1 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 11:55:32 +0000 Subject: [PATCH 017/246] improve logging for troubleshooting --- internal/apiserver/apiserver.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index a47fbf9..fe4f8aa 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -36,12 +36,13 @@ func New(addr string, namespace string, clusterName string, clusterReconciler *c func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) - if err != nil { - c.String(404, err.Error()) - return - } - c.JSON(200, plan) + // plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) + // if err != nil { + // c.String(404, err.Error()) + // return + // } + // c.JSON(200, plan) + c.JSON(http.StatusOK, "GetClusterPlan") } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 @@ -54,10 +55,24 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // testing + // For testing, to see the payload that is being sent for _, target := range payload { - fmt.Printf("name: %s, address: %s, profile: %s, tags: %s\n", *target.Name, *target.Address, *target.Profile, *target.Tags) + if target.Name != nil { + fmt.Printf("name: %s, ", *target.Name) + } + if target.Address != nil { + fmt.Printf("address: %s, ", *target.Address) + } + if target.Profile != nil { + fmt.Printf("profile: %s, ", *target.Profile) + } + if target.Tags != nil { + fmt.Printf("tags: %s", *target.Tags) + } + fmt.Printf("\n") } + // TODO: send target received from interface to autodiscover logic via channel. + c.JSON(http.StatusOK, payload) } From 7c0e2bbdde08addbb28f9d7284a7514623e091bb Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 13:23:54 +0000 Subject: [PATCH 018/246] move gin router creation to apiserver package --- cmd/main.go | 10 +++------ internal/apiserver/apiserver.go | 40 +++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index be73c0f..2e3ba40 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -221,16 +221,12 @@ func main() { } if apiAddr != "" { - apiNamespace := os.Getenv("POD_NAMESPACE") - apiClusterName := os.Getenv("CLUSTER_NAME") - if apiNamespace == "" || apiClusterName == "" { - setupLog.Error(errors.New("missing runtime API identity"), "POD_NAMESPACE and CLUSTER_NAME must be set") + api, err := apiserver.New(apiAddr, clusterReconciler) + if err != nil { + setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - apiBaseURL := "/api/v1/" + apiNamespace + "/" + apiClusterName - api, gin := apiserver.New(apiAddr, apiNamespace, apiClusterName, clusterReconciler) - apiserver.RegisterHandlersWithOptions(gin, api, apiserver.GinServerOptions{BaseURL: apiBaseURL}) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index fe4f8aa..81b8572 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,11 +1,13 @@ package apiserver //go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( + "errors" "fmt" - "log" "net/http" + "os" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" @@ -19,7 +21,7 @@ type APIServer struct { clusterName string } -func New(addr string, namespace string, clusterName string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { +func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -28,28 +30,32 @@ func New(addr string, namespace string, clusterName string, clusterReconciler *c }, router: router, clusterReconciler: clusterReconciler, - namespace: namespace, - clusterName: clusterName, + namespace: os.Getenv("POD_NAMESPACE"), + clusterName: os.Getenv("CLUSTER_NAME"), } - return a, router -} -func (a *APIServer) GetClusterPlan(c *gin.Context) { - log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - // plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) - // if err != nil { - // c.String(404, err.Error()) - // return - // } - // c.JSON(200, plan) - c.JSON(http.StatusOK, "GetClusterPlan") + if a.namespace == "" || a.clusterName == "" { + return nil, errors.New("POD_NAMESPACE and CLUSTER_NAME must be set") + } + apiBaseURL := "/api/v1/" + a.namespace + "/" + a.clusterName + RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + return a, nil } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 -// curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget +// GetClusterPlan returns cluster plan +func (a *APIServer) GetClusterPlan(c *gin.Context) { + plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) + if err != nil { + c.String(404, err.Error()) + return + } + c.JSON(200, plan) +} + +// CreateTargets binds payload to Target struct defined in openapi.yaml and TBD... func (a *APIServer) CreateTargets(c *gin.Context) { - log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) From 4dd7c8cd2e5a3905bb4e9ffd23536c843cc31145 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 13:38:35 +0000 Subject: [PATCH 019/246] small refactor --- cmd/main.go | 6 +++--- config/default/manager_config_patch.yaml | 8 -------- helm/templates/deployment.yaml | 4 ---- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 2e3ba40..ec6ebe0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -228,7 +228,7 @@ func main() { } err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - errCh := make(chan error, 1) + errCh := make(chan error) go func() { err := api.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -244,9 +244,9 @@ func main() { } return err case <-ctx.Done(): - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return api.Server.Shutdown(shutdownCtx) + return api.Server.Shutdown(ctx) } })) if err != nil { diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index c6d4e57..691d2c3 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -13,14 +13,6 @@ spec: - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - readinessProbe: - httpGet: - path: /readyz - port: 8081 ports: - containerPort: 8080 protocol: TCP diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 250c873..314c565 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -46,10 +46,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - name: CLUSTER_NAME valueFrom: fieldRef: From 402c70a06b85c6d891bc26017272cac6a6fcb66a Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 13 Apr 2026 14:46:40 -0600 Subject: [PATCH 020/246] changed NewLoader function call --- internal/controller/targetsource/loaders.go | 9 ++++++--- internal/controller/targetsource_controller.go | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource/loaders.go b/internal/controller/targetsource/loaders.go index 309bf1a..c7e967e 100644 --- a/internal/controller/targetsource/loaders.go +++ b/internal/controller/targetsource/loaders.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "sync" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" ) // Loader defines a pluggable TargetSource loader interface @@ -39,13 +41,14 @@ func Register(name string, factory func() Loader) { } // NewLoader creates a loader by name -func NewLoader(name string) (Loader, error) { +func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (Loader, error) { registryMu.RLock() defer registryMu.RUnlock() - factory, ok := registry[name] + loaderName := namespace + "/" + name + factory, ok := registry[loaderName] if !ok { - return nil, fmt.Errorf("unknown targetsource loader: %q", name) + return nil, fmt.Errorf("unknown targetsource loader: %q", loaderName) } return factory(), nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 58afd52..16d5d64 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -72,13 +72,13 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request _, exists := r.running[req.NamespacedName] r.mu.Unlock() - // If an targetsource loader exists, return immediately without starting + // If a targetsource loader exists, return immediately without starting // any new loader or target manager if exists { return ctrl.Result{}, nil } - loader, err := targetsource.NewLoader(targetSource.Spec.Type) // TODO: pass configuration to loader based on spec + loader, err := targetsource.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec) // TODO: pass configuration to loader based on spec if err != nil { return ctrl.Result{}, err } From 7f8328502c49073b181770fd3f3a984bb3674bb6 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 13 Apr 2026 15:14:06 -0600 Subject: [PATCH 021/246] added discovery message to types --- internal/controller/targetsource/types.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/controller/targetsource/types.go b/internal/controller/targetsource/types.go index 51f7468..9f1c150 100644 --- a/internal/controller/targetsource/types.go +++ b/internal/controller/targetsource/types.go @@ -10,6 +10,19 @@ type DiscoveredTarget struct { Labels map[string]string } +const ( + DELETE DiscoveryEvent = 0 + CREATE DiscoveryEvent = 1 + UPDATE DiscoveryEvent = 2 +) + +type DiscoveryEvent int + +type DiscoveryMessage struct { + Target DiscoveredTarget + Event DiscoveryEvent +} + // TargetManager consumes discovered targets and applies them to Kubernetes. type TargetManager struct { client client.Client From 7d03411e7b7eb96ebc9be336acf2e75ca408e0aa Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 13 Apr 2026 15:16:45 -0600 Subject: [PATCH 022/246] changed target source channel type --- internal/controller/targetsource/loaders.go | 2 +- internal/controller/targetsource/target_manager.go | 2 +- internal/controller/targetsource/types.go | 2 +- internal/controller/targetsource_controller.go | 7 ++++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/controller/targetsource/loaders.go b/internal/controller/targetsource/loaders.go index c7e967e..62344ca 100644 --- a/internal/controller/targetsource/loaders.go +++ b/internal/controller/targetsource/loaders.go @@ -19,7 +19,7 @@ type Loader interface { Start( ctx context.Context, targetsourceName string, - out chan<- []DiscoveredTarget, + out chan<- []DiscoveryMessage, ) error } diff --git a/internal/controller/targetsource/target_manager.go b/internal/controller/targetsource/target_manager.go index e6b20b9..be34fa2 100644 --- a/internal/controller/targetsource/target_manager.go +++ b/internal/controller/targetsource/target_manager.go @@ -8,7 +8,7 @@ import ( ) // NewTargetManager wires a TargetManager instance. -func NewTargetManager(c client.Client, sourceName string, in <-chan []DiscoveredTarget) *TargetManager { +func NewTargetManager(c client.Client, sourceName string, in <-chan []DiscoveryMessage) *TargetManager { return &TargetManager{ client: c, targetsource: sourceName, diff --git a/internal/controller/targetsource/types.go b/internal/controller/targetsource/types.go index 9f1c150..43928f2 100644 --- a/internal/controller/targetsource/types.go +++ b/internal/controller/targetsource/types.go @@ -27,5 +27,5 @@ type DiscoveryMessage struct { type TargetManager struct { client client.Client targetsource string - in <-chan []DiscoveredTarget + in <-chan []DiscoveryMessage } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 16d5d64..b4218b6 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -84,16 +84,17 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } runtimeCtx, cancel := context.WithCancel(context.Background()) - target_channel := make(chan []targetsource.DiscoveredTarget) + + targetChannel := make(chan []targetsource.DiscoveryMessage, 10) // start loader - go loader.Start(runtimeCtx, targetSource.Name, target_channel) + go loader.Start(runtimeCtx, targetSource.Name, targetChannel) // start target manager manager := targetsource.NewTargetManager( r.Client, targetSource.Name, - target_channel, + targetChannel, ) go manager.Run(runtimeCtx) From 4fb037319f88de0f957d3d52f54b411537532501 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 13 Apr 2026 15:20:15 -0600 Subject: [PATCH 023/246] fixed http_pull implementation based on new types --- .../targetsource/loaders/http_pull/loader.go | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/controller/targetsource/loaders/http_pull/loader.go b/internal/controller/targetsource/loaders/http_pull/loader.go index 58efba4..dee5e13 100644 --- a/internal/controller/targetsource/loaders/http_pull/loader.go +++ b/internal/controller/targetsource/loaders/http_pull/loader.go @@ -23,7 +23,7 @@ func (l *Loader) Name() string { func (l *Loader) Start( ctx context.Context, targetsourceName string, - out chan<- []targetsource.DiscoveredTarget, + out chan<- []targetsource.DiscoveryMessage, ) error { logger := log.FromContext(ctx).WithValues("loader", l.Name()) @@ -41,16 +41,22 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - targets := []targetsource.DiscoveredTarget{ + targets := []targetsource.DiscoveryMessage{ { - Name: "ceos1", - Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Target: targetsource.DiscoveredTarget{ + Name: "ceos1", + Address: "clab-3-nodes-ceos1:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + Event: 1, }, { - Name: "leaf1", - Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, + Target: targetsource.DiscoveredTarget{ + Name: "leaf1", + Address: "clab-3-nodes-leaf1:57400", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + Event: 1, }, } From 2294f1e71e3efc60671879902f8eb11d087b084f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 13 Apr 2026 16:06:52 -0600 Subject: [PATCH 024/246] implemented first draft of target creation using loaders --- internal/controller/targetsource/loaders.go | 2 +- .../targetsource/loaders/http_pull/loader.go | 4 +- .../controller/targetsource/target_manager.go | 44 ++++++++++++++++--- internal/controller/targetsource/types.go | 10 ++++- .../controller/targetsource_controller.go | 3 +- 5 files changed, 52 insertions(+), 11 deletions(-) diff --git a/internal/controller/targetsource/loaders.go b/internal/controller/targetsource/loaders.go index 62344ca..b591acd 100644 --- a/internal/controller/targetsource/loaders.go +++ b/internal/controller/targetsource/loaders.go @@ -46,7 +46,7 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe defer registryMu.RUnlock() loaderName := namespace + "/" + name - factory, ok := registry[loaderName] + factory, ok := registry[spec.Type] if !ok { return nil, fmt.Errorf("unknown targetsource loader: %q", loaderName) } diff --git a/internal/controller/targetsource/loaders/http_pull/loader.go b/internal/controller/targetsource/loaders/http_pull/loader.go index dee5e13..4157107 100644 --- a/internal/controller/targetsource/loaders/http_pull/loader.go +++ b/internal/controller/targetsource/loaders/http_pull/loader.go @@ -48,7 +48,7 @@ func (l *Loader) Start( Address: "clab-3-nodes-ceos1:6030", Labels: map[string]string{"TargetSource": targetsourceName}, }, - Event: 1, + Event: targetsource.CREATE, }, { Target: targetsource.DiscoveredTarget{ @@ -56,7 +56,7 @@ func (l *Loader) Start( Address: "clab-3-nodes-leaf1:57400", Labels: map[string]string{"TargetSource": targetsourceName}, }, - Event: 1, + Event: targetsource.CREATE, }, } diff --git a/internal/controller/targetsource/target_manager.go b/internal/controller/targetsource/target_manager.go index be34fa2..b69a9c9 100644 --- a/internal/controller/targetsource/target_manager.go +++ b/internal/controller/targetsource/target_manager.go @@ -2,16 +2,22 @@ package targetsource import ( "context" + "fmt" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" ) // NewTargetManager wires a TargetManager instance. -func NewTargetManager(c client.Client, sourceName string, in <-chan []DiscoveryMessage) *TargetManager { +func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []DiscoveryMessage) *TargetManager { return &TargetManager{ client: c, - targetsource: sourceName, + scheme: s, + targetSource: ts, in: in, } } @@ -20,7 +26,7 @@ func NewTargetManager(c client.Client, sourceName string, in <-chan []DiscoveryM // and reconciles Target CRs accordingly func (m *TargetManager) Run(ctx context.Context) error { logger := log.FromContext(ctx). - WithValues("targetSource", m.targetsource) + WithValues("targetSource", m.targetSource) logger.Info("target manager started") @@ -30,12 +36,40 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Info("target manager stopped") return nil - case targets := <-m.in: + case messages := <-m.in: logger.Info( "received discovered targets", - "count", len(targets), + "count", len(messages), ) + for _, msg := range messages { + if msg.Event == CREATE { + target := &gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: msg.Target.Name, + Namespace: m.targetSource.ObjectMeta.Namespace, + Labels: map[string]string{ + "gnmic.io/source": m.targetSource.ObjectMeta.Name, + }, + }, + Spec: gnmicv1alpha1.TargetSpec{ + Address: msg.Target.Address, + Profile: "default", + }, + } + err := controllerutil.SetControllerReference(m.targetSource, target, m.scheme) + if err != nil { + logger.Error(err, "error setting the owner reference") + } + + err = m.client.Create(ctx, target) + if err != nil { + logger.Error(err, "error creating target object") + } + logger.Info(fmt.Sprintf("created new target object %s/%s", target.ObjectMeta.Namespace, target.ObjectMeta.Name)) + } + } + // List existing Target CRs owned by this TargetSource // var existing gnmicv1alpha1.TargetList // if err := m.client.List( diff --git a/internal/controller/targetsource/types.go b/internal/controller/targetsource/types.go index 43928f2..6c2b283 100644 --- a/internal/controller/targetsource/types.go +++ b/internal/controller/targetsource/types.go @@ -1,6 +1,11 @@ package targetsource -import "sigs.k8s.io/controller-runtime/pkg/client" +import ( + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR @@ -26,6 +31,7 @@ type DiscoveryMessage struct { // TargetManager consumes discovered targets and applies them to Kubernetes. type TargetManager struct { client client.Client - targetsource string + scheme *runtime.Scheme + targetSource *gnmicv1alpha1.TargetSource in <-chan []DiscoveryMessage } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index b4218b6..a7b887c 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -93,7 +93,8 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request // start target manager manager := targetsource.NewTargetManager( r.Client, - targetSource.Name, + r.Scheme, + &targetSource, targetChannel, ) go manager.Run(runtimeCtx) From 912e05a9945048de6518ec3684d11b0d4e05dd28 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Tue, 14 Apr 2026 15:12:58 -0600 Subject: [PATCH 025/246] removed manual targetsource spec --- api/v1alpha1/targetsource_types.go | 7 ------- internal/controller/targetsource/client.go | 24 ---------------------- 2 files changed, 31 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 30aa6bd..037b581 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -28,18 +28,11 @@ type TargetSourceSpec struct { ConfigMap string `json:"configMap,omitempty"` PodSelector metav1.LabelSelector `json:"podSelector,omitempty"` ServiceSelector metav1.LabelSelector `json:"serviceSelector,omitempty"` - Manual []ManualTarget `json:"manual,omitempty"` // Type string `json:"type,omitempty"` Labels map[string]string `json:"labels,omitempty"` } -type ManualTarget struct { - Name string `json:"name,omitempty"` - Address string `json:"address,omitempty"` - TargetProfile string `json:"targetProfile,omitempty"` -} - type HTTPConfig struct { URL string `json:"url,omitempty"` } diff --git a/internal/controller/targetsource/client.go b/internal/controller/targetsource/client.go index b190adc..c918ba1 100644 --- a/internal/controller/targetsource/client.go +++ b/internal/controller/targetsource/client.go @@ -3,35 +3,11 @@ package targetsource import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" ) -func FetchDiscoveryTargets(ctx context.Context, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { - var targets []gnmicv1alpha1.Target - - for _, e := range ts.Spec.Manual { - target := &gnmicv1alpha1.Target{ - ObjectMeta: metav1.ObjectMeta{ - Name: e.Name, - Namespace: ts.Namespace, - Labels: map[string]string{ - "gnmic.io/source": ts.Name, - }, - }, - Spec: gnmicv1alpha1.TargetSpec{ - Address: e.Address, - Profile: e.TargetProfile, - }, - } - targets = append(targets, *target) - } - - return targets, nil -} - func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { var targetList gnmicv1alpha1.TargetList From bf28aad5fd34de0043e52c144aa51d719ec7be5b Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Tue, 14 Apr 2026 15:36:02 -0600 Subject: [PATCH 026/246] cleaned up reconciliation loop and refactored into target manager --- .../operator.gnmic.dev_targetsources.yaml | 11 --- .../controller/targetsource/target_manager.go | 68 +++++++++++++------ .../controller/targetsource_controller.go | 67 +----------------- 3 files changed, 49 insertions(+), 97 deletions(-) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index c5604d8..3070c0c 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,17 +60,6 @@ spec: additionalProperties: type: string type: object - manual: - items: - properties: - address: - type: string - name: - type: string - targetProfile: - type: string - type: object - type: array podSelector: description: |- A label selector is a label query over a set of resources. The result of matchLabels and diff --git a/internal/controller/targetsource/target_manager.go b/internal/controller/targetsource/target_manager.go index b69a9c9..ebb3bf5 100644 --- a/internal/controller/targetsource/target_manager.go +++ b/internal/controller/targetsource/target_manager.go @@ -4,12 +4,14 @@ import ( "context" "fmt" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" ) // NewTargetManager wires a TargetManager instance. @@ -22,7 +24,7 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ } } -// Run is a long‑running loop that receives target snapshots +// Run is a long‑running loop that receives target event messages // and reconciles Target CRs accordingly func (m *TargetManager) Run(ctx context.Context) error { logger := log.FromContext(ctx). @@ -43,7 +45,24 @@ func (m *TargetManager) Run(ctx context.Context) error { ) for _, msg := range messages { - if msg.Event == CREATE { + switch msg.Event { + case DELETE: + existing := &gnmicv1alpha1.Target{} + err := m.client.Get(ctx, types.NamespacedName{ + Name: msg.Target.Name, + Namespace: m.targetSource.Namespace, + }, existing) + if err != nil { + logger.Error(err, "error fetching existing target object") + } + + err = m.client.Delete(ctx, existing) + if err != nil { + logger.Error(err, "error deleting the object") + } + logger.Info(fmt.Sprintf("deleted target object %s/%s", m.targetSource.Namespace, msg.Target.Name)) + + case CREATE: target := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ Name: msg.Target.Name, @@ -67,25 +86,34 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Error(err, "error creating target object") } logger.Info(fmt.Sprintf("created new target object %s/%s", target.ObjectMeta.Namespace, target.ObjectMeta.Name)) + + case UPDATE: + existing := &gnmicv1alpha1.Target{} + newSpec := gnmicv1alpha1.TargetSpec{ + Address: msg.Target.Address, + Profile: "default", + } + + err := m.client.Get(ctx, types.NamespacedName{ + Name: msg.Target.Name, + Namespace: m.targetSource.Namespace, + }, existing) + if err != nil { + logger.Error(err, "error fetching existing target object") + } + + existing.Spec = newSpec + + err = m.client.Update(ctx, existing) + if err != nil { + logger.Error(err, "error updating object") + } + logger.Info(fmt.Sprintf("updated existing target object %s/%s", existing.ObjectMeta.Namespace, existing.ObjectMeta.Name)) + + default: + logger.Error(nil, "unknown discovery event received") } } - - // List existing Target CRs owned by this TargetSource - // var existing gnmicv1alpha1.TargetList - // if err := m.client.List( - // ctx, - // &existing, - // client.MatchingLabels{ - // "gnmic.dev/targetsource": m.targetsource, - // }, - // ); err != nil { - // return err - // } - - // TODO: Target Lifecycle Management - // 1. Compare and determine which Targets to create/update/delete - // 2. Create/update/delete Target CRs accordingly - // 3. Update TargetSource status with sync results } } } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index a7b887c..79bc38b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -74,6 +74,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request // If a targetsource loader exists, return immediately without starting // any new loader or target manager + // TODO: check for spec changes and handle running process accordingly if exists { return ctrl.Result{}, nil } @@ -105,72 +106,6 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request logger.Info("TargetSource pipeline started", "name", targetSource.Name) - // // Step 1: Get desired state from discovery source - // discoveredTargets, err := targetsource.FetchDiscoveryTargets(ctx, targetSource) - // if err != nil { - // logger.Error(err, "error getting discovered targets") - // return ctrl.Result{}, err - // } - - // // Step 2: Get current state from Kubernetes cluster (lookup by label of TargetSource) - // existingTargets, err := targetsource.FetchExistingTargets(ctx, r.Client, targetSource) - // if err != nil { - // logger.Error(err, "error fetching existing targets") - // return ctrl.Result{}, err - // } - - // // Step 3: Compute diff - // diff := targetsource.BuildDiff(existingTargets, discoveredTargets) - - // // Step 4: Iterate over each list and do create, update, delete respectively - // for _, t := range diff.ToCreate { - // err = controllerutil.SetControllerReference(&targetSource, &t, r.Scheme) - // if err != nil { - // logger.Error(err, "error setting the owner reference") - // return ctrl.Result{}, err - // } - - // err = r.Client.Create(ctx, &t) - // if err != nil { - // logger.Error(err, "error creating target object") - // return ctrl.Result{}, err - // } - // logger.Info(fmt.Sprintf("created new target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) - // } - - // for _, t := range diff.ToUpdate { - // existing := &gnmicv1alpha1.Target{} - - // err := r.Get(ctx, types.NamespacedName{ - // Name: t.ObjectMeta.Name, - // Namespace: t.ObjectMeta.Namespace, - // }, existing) - - // if err != nil { - // logger.Error(err, "error fetching existing target object") - // return ctrl.Result{}, err - // } - - // existing.Spec = t.Spec - - // err = r.Update(ctx, existing) - // if err != nil { - // logger.Error(err, "error updating object") - // return ctrl.Result{}, err - // } - // logger.Info(fmt.Sprintf("updated existing target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) - // } - - // for _, t := range diff.ToDelete { - // err = r.Client.Delete(ctx, &t) - // logger.Info(fmt.Sprintf("resource name to be deleted: %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) - // if err != nil { - // logger.Error(err, "error deleting the object") - // return ctrl.Result{}, err - // } - // logger.Info(fmt.Sprintf("deleted target object %s/%s", t.ObjectMeta.Namespace, t.ObjectMeta.Name)) - // } - return ctrl.Result{}, nil } From 022dbaef5a3f604e0e52e0c82798febb26a525e6 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 15 Apr 2026 09:28:49 -0600 Subject: [PATCH 027/246] restructured project to introduce new architecture --- .../operator.gnmic.dev_targetsources.yaml | 2 + .../controller/targetsource/core/loaders.go | 20 +++++++ .../targetsource/core/loaders_test.go | 1 + .../targetsource/{ => core}/target_manager.go | 2 +- .../targetsource/{ => core}/types.go | 2 +- internal/controller/targetsource/factory.go | 24 +++++++++ internal/controller/targetsource/loaders.go | 54 ------------------- .../targetsource/loaders/http_pull/loader.go | 20 +++---- .../controller/targetsource/loaders_test.go | 1 - .../controller/targetsource_controller.go | 5 +- 10 files changed, 60 insertions(+), 71 deletions(-) create mode 100644 internal/controller/targetsource/core/loaders.go create mode 100644 internal/controller/targetsource/core/loaders_test.go rename internal/controller/targetsource/{ => core}/target_manager.go (99%) rename internal/controller/targetsource/{ => core}/types.go (97%) create mode 100644 internal/controller/targetsource/factory.go delete mode 100644 internal/controller/targetsource/loaders.go delete mode 100644 internal/controller/targetsource/loaders_test.go diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 1212ff9..0129a88 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,6 +60,8 @@ spec: type: string type: object type: object + type: + type: string required: - profile - provider diff --git a/internal/controller/targetsource/core/loaders.go b/internal/controller/targetsource/core/loaders.go new file mode 100644 index 0000000..7007349 --- /dev/null +++ b/internal/controller/targetsource/core/loaders.go @@ -0,0 +1,20 @@ +package core + +import ( + "context" +) + +// Loader defines a pluggable TargetSource loader interface +// Loaders observe external Sources of Truth and emit target snapshots through a channel +type Loader interface { + // Name returns the unique loader identifier e.g. "http_pull" + Name() string + + // Start begins discovery and pushes target snapshots into the out channel + // The loader must stop cleanly when ctx is cancelled + Start( + ctx context.Context, + targetsourceName string, + out chan<- []DiscoveryMessage, + ) error +} diff --git a/internal/controller/targetsource/core/loaders_test.go b/internal/controller/targetsource/core/loaders_test.go new file mode 100644 index 0000000..9a8bc95 --- /dev/null +++ b/internal/controller/targetsource/core/loaders_test.go @@ -0,0 +1 @@ +package core diff --git a/internal/controller/targetsource/target_manager.go b/internal/controller/targetsource/core/target_manager.go similarity index 99% rename from internal/controller/targetsource/target_manager.go rename to internal/controller/targetsource/core/target_manager.go index ebb3bf5..e41a9ad 100644 --- a/internal/controller/targetsource/target_manager.go +++ b/internal/controller/targetsource/core/target_manager.go @@ -1,4 +1,4 @@ -package targetsource +package core import ( "context" diff --git a/internal/controller/targetsource/types.go b/internal/controller/targetsource/core/types.go similarity index 97% rename from internal/controller/targetsource/types.go rename to internal/controller/targetsource/core/types.go index 6c2b283..2b39606 100644 --- a/internal/controller/targetsource/types.go +++ b/internal/controller/targetsource/core/types.go @@ -1,4 +1,4 @@ -package targetsource +package core import ( "k8s.io/apimachinery/pkg/runtime" diff --git a/internal/controller/targetsource/factory.go b/internal/controller/targetsource/factory.go new file mode 100644 index 0000000..1421390 --- /dev/null +++ b/internal/controller/targetsource/factory.go @@ -0,0 +1,24 @@ +package targetsource + +import ( + "fmt" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/targetsource/core" + "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_pull" +) + +// NewLoader creates a loader by name +func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { + loaderName := namespace + "/" + name + + switch { + case spec.Provider.HTTP != nil: + return http_pull.New(), nil + case spec.Provider.Consul != nil: + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + default: + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + } + +} diff --git a/internal/controller/targetsource/loaders.go b/internal/controller/targetsource/loaders.go deleted file mode 100644 index b591acd..0000000 --- a/internal/controller/targetsource/loaders.go +++ /dev/null @@ -1,54 +0,0 @@ -package targetsource - -import ( - "context" - "fmt" - "sync" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" -) - -// Loader defines a pluggable TargetSource loader interface -// Loaders observe external Sources of Truth and emit target snapshots through a channel -type Loader interface { - // Name returns the unique loader identifier e.g. "http_pull" - Name() string - - // Start begins discovery and pushes target snapshots into the out channel - // The loader must stop cleanly when ctx is cancelled - Start( - ctx context.Context, - targetsourceName string, - out chan<- []DiscoveryMessage, - ) error -} - -var ( - registryMu sync.RWMutex - registry = make(map[string]func() Loader) -) - -// Register registers a loader implementation -// It panics on duplicate registrations to fail fast during startup rather than at runtime -func Register(name string, factory func() Loader) { - registryMu.Lock() - defer registryMu.Unlock() - - if _, exists := registry[name]; exists { - panic(fmt.Sprintf("targetsource loader %q already registered", name)) - } - registry[name] = factory -} - -// NewLoader creates a loader by name -func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (Loader, error) { - registryMu.RLock() - defer registryMu.RUnlock() - - loaderName := namespace + "/" + name - factory, ok := registry[spec.Type] - if !ok { - return nil, fmt.Errorf("unknown targetsource loader: %q", loaderName) - } - return factory(), nil -} diff --git a/internal/controller/targetsource/loaders/http_pull/loader.go b/internal/controller/targetsource/loaders/http_pull/loader.go index 4157107..269db18 100644 --- a/internal/controller/targetsource/loaders/http_pull/loader.go +++ b/internal/controller/targetsource/loaders/http_pull/loader.go @@ -6,13 +6,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/gnmic/operator/internal/controller/targetsource" + "github.com/gnmic/operator/internal/controller/targetsource/core" ) type Loader struct{} // New instantiates the http_pull loader -func New() targetsource.Loader { +func New() core.Loader { return &Loader{} } @@ -23,7 +23,7 @@ func (l *Loader) Name() string { func (l *Loader) Start( ctx context.Context, targetsourceName string, - out chan<- []targetsource.DiscoveryMessage, + out chan<- []core.DiscoveryMessage, ) error { logger := log.FromContext(ctx).WithValues("loader", l.Name()) @@ -41,22 +41,22 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - targets := []targetsource.DiscoveryMessage{ + targets := []core.DiscoveryMessage{ { - Target: targetsource.DiscoveredTarget{ + Target: core.DiscoveredTarget{ Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", Labels: map[string]string{"TargetSource": targetsourceName}, }, - Event: targetsource.CREATE, + Event: core.CREATE, }, { - Target: targetsource.DiscoveredTarget{ + Target: core.DiscoveredTarget{ Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", Labels: map[string]string{"TargetSource": targetsourceName}, }, - Event: targetsource.CREATE, + Event: core.CREATE, }, } @@ -74,7 +74,3 @@ func (l *Loader) Start( } } } - -func init() { - targetsource.Register("http_pull", New) -} diff --git a/internal/controller/targetsource/loaders_test.go b/internal/controller/targetsource/loaders_test.go deleted file mode 100644 index 603b690..0000000 --- a/internal/controller/targetsource/loaders_test.go +++ /dev/null @@ -1 +0,0 @@ -package targetsource diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 79bc38b..7980a9b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -27,6 +27,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/targetsource" + "github.com/gnmic/operator/internal/controller/targetsource/core" _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/all" ) @@ -86,13 +87,13 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request runtimeCtx, cancel := context.WithCancel(context.Background()) - targetChannel := make(chan []targetsource.DiscoveryMessage, 10) + targetChannel := make(chan []core.DiscoveryMessage, 10) // start loader go loader.Start(runtimeCtx, targetSource.Name, targetChannel) // start target manager - manager := targetsource.NewTargetManager( + manager := core.NewTargetManager( r.Client, r.Scheme, &targetSource, From 4dc2eb31da6bc8ab0a8fab7faae2109ca8e596d4 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 15 Apr 2026 10:52:36 -0600 Subject: [PATCH 028/246] renamed targetsource package to discovery --- .../{targetsource => discovery}/client.go | 4 +++- .../core/loader_interface.go} | 0 .../{targetsource => discovery}/core/types.go | 15 --------------- .../factory.go => discovery/loader.go} | 6 +++--- .../loaders/all/all.go | 2 +- .../loaders/http_pull/loader.go | 2 +- .../loaders/http_pull/loader_test.go | 0 .../loaders/http_push/loader.go | 0 .../loaders/http_push/loader_test.go | 0 .../{targetsource => discovery}/mapper.go | 2 +- internal/controller/discovery/mapper_test.go | 1 + .../core => discovery}/target_manager.go | 19 ++++++++++++++----- .../targetsource/core/loaders_test.go | 1 - .../controller/targetsource/mapper_test.go | 1 - .../controller/targetsource_controller.go | 10 +++++----- 15 files changed, 29 insertions(+), 34 deletions(-) rename internal/controller/{targetsource => discovery}/client.go (79%) rename internal/controller/{targetsource/core/loaders.go => discovery/core/loader_interface.go} (100%) rename internal/controller/{targetsource => discovery}/core/types.go (52%) rename internal/controller/{targetsource/factory.go => discovery/loader.go} (77%) rename internal/controller/{targetsource => discovery}/loaders/all/all.go (57%) rename internal/controller/{targetsource => discovery}/loaders/http_pull/loader.go (95%) rename internal/controller/{targetsource => discovery}/loaders/http_pull/loader_test.go (100%) rename internal/controller/{targetsource => discovery}/loaders/http_push/loader.go (100%) rename internal/controller/{targetsource => discovery}/loaders/http_push/loader_test.go (100%) rename internal/controller/{targetsource => discovery}/mapper.go (98%) create mode 100644 internal/controller/discovery/mapper_test.go rename internal/controller/{targetsource/core => discovery}/target_manager.go (87%) delete mode 100644 internal/controller/targetsource/core/loaders_test.go delete mode 100644 internal/controller/targetsource/mapper_test.go diff --git a/internal/controller/targetsource/client.go b/internal/controller/discovery/client.go similarity index 79% rename from internal/controller/targetsource/client.go rename to internal/controller/discovery/client.go index c918ba1..3bc7ef7 100644 --- a/internal/controller/targetsource/client.go +++ b/internal/controller/discovery/client.go @@ -1,4 +1,6 @@ -package targetsource +package discovery + +// File may become obsolete, depends on how the logic to compare desired vs. existing state will get implemented import ( "context" diff --git a/internal/controller/targetsource/core/loaders.go b/internal/controller/discovery/core/loader_interface.go similarity index 100% rename from internal/controller/targetsource/core/loaders.go rename to internal/controller/discovery/core/loader_interface.go diff --git a/internal/controller/targetsource/core/types.go b/internal/controller/discovery/core/types.go similarity index 52% rename from internal/controller/targetsource/core/types.go rename to internal/controller/discovery/core/types.go index 2b39606..406a22b 100644 --- a/internal/controller/targetsource/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,12 +1,5 @@ package core -import ( - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" -) - // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { @@ -27,11 +20,3 @@ type DiscoveryMessage struct { Target DiscoveredTarget Event DiscoveryEvent } - -// TargetManager consumes discovered targets and applies them to Kubernetes. -type TargetManager struct { - client client.Client - scheme *runtime.Scheme - targetSource *gnmicv1alpha1.TargetSource - in <-chan []DiscoveryMessage -} diff --git a/internal/controller/targetsource/factory.go b/internal/controller/discovery/loader.go similarity index 77% rename from internal/controller/targetsource/factory.go rename to internal/controller/discovery/loader.go index 1421390..ad1e83f 100644 --- a/internal/controller/targetsource/factory.go +++ b/internal/controller/discovery/loader.go @@ -1,11 +1,11 @@ -package targetsource +package discovery import ( "fmt" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/targetsource/core" - "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_pull" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" ) // NewLoader creates a loader by name diff --git a/internal/controller/targetsource/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go similarity index 57% rename from internal/controller/targetsource/loaders/all/all.go rename to internal/controller/discovery/loaders/all/all.go index 629c5d9..c53b98a 100644 --- a/internal/controller/targetsource/loaders/all/all.go +++ b/internal/controller/discovery/loaders/all/all.go @@ -1,6 +1,6 @@ package all import ( - _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_pull" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" // _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_push" ) diff --git a/internal/controller/targetsource/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go similarity index 95% rename from internal/controller/targetsource/loaders/http_pull/loader.go rename to internal/controller/discovery/loaders/http_pull/loader.go index 269db18..e987d78 100644 --- a/internal/controller/targetsource/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -6,7 +6,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/gnmic/operator/internal/controller/targetsource/core" + "github.com/gnmic/operator/internal/controller/discovery/core" ) type Loader struct{} diff --git a/internal/controller/targetsource/loaders/http_pull/loader_test.go b/internal/controller/discovery/loaders/http_pull/loader_test.go similarity index 100% rename from internal/controller/targetsource/loaders/http_pull/loader_test.go rename to internal/controller/discovery/loaders/http_pull/loader_test.go diff --git a/internal/controller/targetsource/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go similarity index 100% rename from internal/controller/targetsource/loaders/http_push/loader.go rename to internal/controller/discovery/loaders/http_push/loader.go diff --git a/internal/controller/targetsource/loaders/http_push/loader_test.go b/internal/controller/discovery/loaders/http_push/loader_test.go similarity index 100% rename from internal/controller/targetsource/loaders/http_push/loader_test.go rename to internal/controller/discovery/loaders/http_push/loader_test.go diff --git a/internal/controller/targetsource/mapper.go b/internal/controller/discovery/mapper.go similarity index 98% rename from internal/controller/targetsource/mapper.go rename to internal/controller/discovery/mapper.go index dbdfbcf..6f8dbf0 100644 --- a/internal/controller/targetsource/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -1,4 +1,4 @@ -package targetsource +package discovery // This file makes diff between existing and new targets // file decides which targets to create/update/delete diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go new file mode 100644 index 0000000..5844159 --- /dev/null +++ b/internal/controller/discovery/mapper_test.go @@ -0,0 +1 @@ +package discovery diff --git a/internal/controller/targetsource/core/target_manager.go b/internal/controller/discovery/target_manager.go similarity index 87% rename from internal/controller/targetsource/core/target_manager.go rename to internal/controller/discovery/target_manager.go index e41a9ad..c13b1ef 100644 --- a/internal/controller/targetsource/core/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -1,4 +1,4 @@ -package core +package discovery import ( "context" @@ -12,10 +12,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" ) +// TargetManager consumes discovered targets and applies them to Kubernetes. +type TargetManager struct { + client client.Client + scheme *runtime.Scheme + targetSource *gnmicv1alpha1.TargetSource + in <-chan []core.DiscoveryMessage +} + // NewTargetManager wires a TargetManager instance. -func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []DiscoveryMessage) *TargetManager { +func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager { return &TargetManager{ client: c, scheme: s, @@ -46,7 +55,7 @@ func (m *TargetManager) Run(ctx context.Context) error { for _, msg := range messages { switch msg.Event { - case DELETE: + case core.DELETE: existing := &gnmicv1alpha1.Target{} err := m.client.Get(ctx, types.NamespacedName{ Name: msg.Target.Name, @@ -62,7 +71,7 @@ func (m *TargetManager) Run(ctx context.Context) error { } logger.Info(fmt.Sprintf("deleted target object %s/%s", m.targetSource.Namespace, msg.Target.Name)) - case CREATE: + case core.CREATE: target := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ Name: msg.Target.Name, @@ -87,7 +96,7 @@ func (m *TargetManager) Run(ctx context.Context) error { } logger.Info(fmt.Sprintf("created new target object %s/%s", target.ObjectMeta.Namespace, target.ObjectMeta.Name)) - case UPDATE: + case core.UPDATE: existing := &gnmicv1alpha1.Target{} newSpec := gnmicv1alpha1.TargetSpec{ Address: msg.Target.Address, diff --git a/internal/controller/targetsource/core/loaders_test.go b/internal/controller/targetsource/core/loaders_test.go deleted file mode 100644 index 9a8bc95..0000000 --- a/internal/controller/targetsource/core/loaders_test.go +++ /dev/null @@ -1 +0,0 @@ -package core diff --git a/internal/controller/targetsource/mapper_test.go b/internal/controller/targetsource/mapper_test.go deleted file mode 100644 index 603b690..0000000 --- a/internal/controller/targetsource/mapper_test.go +++ /dev/null @@ -1 +0,0 @@ -package targetsource diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 7980a9b..64c9909 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -26,9 +26,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/targetsource" - "github.com/gnmic/operator/internal/controller/targetsource/core" - _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/all" + "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" ) type runningSource struct { @@ -80,7 +80,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - loader, err := targetsource.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec) // TODO: pass configuration to loader based on spec + loader, err := discovery.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec) // TODO: pass configuration to loader based on spec if err != nil { return ctrl.Result{}, err } @@ -93,7 +93,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request go loader.Start(runtimeCtx, targetSource.Name, targetChannel) // start target manager - manager := core.NewTargetManager( + manager := discovery.NewTargetManager( r.Client, r.Scheme, &targetSource, From ad172c929ed5a6052f27d7b4be21820f51255aec Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 15 Apr 2026 11:44:38 -0600 Subject: [PATCH 029/246] removed unnecessary files and updated gitignore --- .gitignore | 1 + lab/dev/3-nodes.clab.yaml.annotations.json | 35 ---------------------- lab/dev/netbox/readme.txt | 3 -- lab/dev/temp | 0 4 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 lab/dev/3-nodes.clab.yaml.annotations.json delete mode 100644 lab/dev/netbox/readme.txt delete mode 100644 lab/dev/temp diff --git a/.gitignore b/.gitignore index c04366f..29d31af 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ Dockerfile.cross *.swo *~ private/ +lab/dev/**/*.annotations.json lab/dev/clab-* lab/dev/netbox/secrets design/ diff --git a/lab/dev/3-nodes.clab.yaml.annotations.json b/lab/dev/3-nodes.clab.yaml.annotations.json deleted file mode 100644 index 9188d4c..0000000 --- a/lab/dev/3-nodes.clab.yaml.annotations.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "freeTextAnnotations": [], - "freeShapeAnnotations": [], - "groupStyleAnnotations": [], - "networkNodeAnnotations": [], - "nodeAnnotations": [ - { - "id": "spine1", - "interfacePattern": "e1-{n}", - "position": { - "x": 360, - "y": 380 - } - }, - { - "id": "leaf1", - "interfacePattern": "e1-{n}", - "position": { - "x": 480, - "y": 260 - } - }, - { - "id": "leaf2", - "interfacePattern": "e1-{n}", - "position": { - "x": 520, - "y": 400 - } - } - ], - "edgeAnnotations": [], - "aliasEndpointAnnotations": [], - "viewerSettings": {} -} \ No newline at end of file diff --git a/lab/dev/netbox/readme.txt b/lab/dev/netbox/readme.txt deleted file mode 100644 index c0edbe1..0000000 --- a/lab/dev/netbox/readme.txt +++ /dev/null @@ -1,3 +0,0 @@ -# All files within operator/lab/dev/netbox are -# only for development and testing purposes -# is generally vibe coded and will be removed after development \ No newline at end of file diff --git a/lab/dev/temp b/lab/dev/temp deleted file mode 100644 index e69de29..0000000 From 181ea95a26c2f88be3b8b01f0b4e6de0c176d39d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 16 Apr 2026 12:29:50 +0000 Subject: [PATCH 030/246] extend TargetSource CRD by http token --- api/v1alpha1/targetsource_types.go | 3 ++- config/crd/bases/operator.gnmic.dev_targetsources.yaml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 3cf029b..057bbb2 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -39,7 +39,8 @@ type ProviderSpec struct { } type HTTPConfig struct { - URL string `json:"url,omitempty"` + URL string `json:"url,omitempty"` + Token string `json:"token,omitempty"` } type ConsulConfig struct { diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 0129a88..7aa6084 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -56,6 +56,8 @@ spec: type: object http: properties: + token: + type: string url: type: string type: object From 2fddddf3e33ba4112550c59b59583729e548bc0d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 16 Apr 2026 12:30:08 +0000 Subject: [PATCH 031/246] add pull logic as poc --- .../discovery/loaders/http_pull/loader.go | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 22868b2..7162119 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -2,6 +2,9 @@ package http_pull import ( "context" + "encoding/json" + "fmt" + "net/http" "time" "sigs.k8s.io/controller-runtime/pkg/log" @@ -31,6 +34,10 @@ func (l *Loader) Start( logger.Info("HTTP pull loader started") + client := &http.Client{ + Timeout: 30 * time.Second, + } + // Only for debugging: emit a static snapshot every 30 seconds ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() @@ -42,32 +49,26 @@ func (l *Loader) Start( return nil case <-ticker.C: - // Example snapshot (placeholder) - targets := []core.DiscoveryMessage{ - { - Target: core.DiscoveredTarget{ - Name: "ceos1", - Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, - }, - { - Target: core.DiscoveredTarget{ - Name: "leaf1", - Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, - }, + targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, spec.Provider.HTTP.URL, spec.Provider.HTTP.Token) + if err != nil { + logger.Error(err, "failed to fetch targets from HTTP endpoint") + continue + } + + var messages []core.DiscoveryMessage + for _, target := range targets { + messages = append(messages, core.DiscoveryMessage{ + Target: target, + Event: core.CREATE, + }) } // Non-blocking context-aware send select { - case out <- targets: - logger.V(1).Info( + case out <- messages: + logger.Info( "emitted target snapshot", - "count", len(targets), + "count", len(messages), ) case <-ctx.Done(): logger.Info("context cancelled while emitting targets") @@ -76,3 +77,34 @@ func (l *Loader) Start( } } } + +func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http.Client, url string, token string) ([]core.DiscoveredTarget, error) { + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + url, + nil, + ) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", "Token +"+token) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var targets []core.DiscoveredTarget + if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil { + return nil, err + } + + return targets, nil +} From 64a83cd28ab91846126ae612f09f0292cd2fd62d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 16 Apr 2026 13:47:48 +0000 Subject: [PATCH 032/246] fix request header typo --- internal/controller/discovery/loaders/http_pull/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 7162119..7bd0bd1 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -89,7 +89,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http. return nil, err } req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", "Token +"+token) + req.Header.Set("Authorization", "Token "+token) resp, err := client.Do(req) if err != nil { From 98823e83dc124853258357e34c4e1571dafe66a5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 16 Apr 2026 14:25:18 +0000 Subject: [PATCH 033/246] refactor pull implementation --- .../discovery/loaders/http_pull/loader.go | 107 +++++++++++------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 7bd0bd1..fb081f8 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -3,6 +3,7 @@ package http_pull import ( "context" "encoding/json" + "errors" "fmt" "net/http" "time" @@ -13,9 +14,14 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) +const ( + defaultPollInterval = 30 * time.Second +) + +// Loader implements the HTTP pull discovery mechanism type Loader struct{} -// New instantiates the http_pull loader +// New returns a new http_pull loader instance func New() core.Loader { return &Loader{} } @@ -30,18 +36,59 @@ func (l *Loader) Start( spec gnmicv1alpha1.TargetSourceSpec, out chan<- []core.DiscoveryMessage, ) error { - logger := log.FromContext(ctx).WithValues("loader", l.Name()) + logger := log.FromContext(ctx).WithValues( + "loader", l.Name(), + "targetSource", targetsourceName, + ) - logger.Info("HTTP pull loader started") + // Input Validation of spec + if spec.Provider == nil || spec.Provider.HTTP == nil { + return errors.New("http_pull loader requires spec.provider.http to be set") + } client := &http.Client{ Timeout: 30 * time.Second, } - // Only for debugging: emit a static snapshot every 30 seconds - ticker := time.NewTicker(30 * time.Second) + interval := defaultPollInterval + ticker := time.NewTicker(interval) defer ticker.Stop() + logger.Info("HTTP pull loader started", "interval", interval.String()) + + // helper function to fetch targets and emit discovery messages + fetchAndEmit := func() { + targets, err := l.fetchTargetsFromHTTPEndpoint( + ctx, + client, + spec.Provider.HTTP.URL, + spec.Provider.HTTP.Token, + ) + if err != nil { + logger.Error(err, "failed to fetch targets from HTTP endpoint") + return + } + + messages := make([]core.DiscoveryMessage, 0, len(targets)) + for _, target := range targets { + messages = append(messages, core.DiscoveryMessage{ + Target: target, + Event: core.CREATE, + }) + } + + select { + case out <- messages: + logger.Info("emitted target snapshot", "count", len(messages)) + case <-ctx.Done(): + logger.Info("context cancelled while emitting targets") + } + } + + // Immediate fetch on startup + fetchAndEmit() + + // Periodic fetch for { select { case <-ctx.Done(): @@ -49,61 +96,39 @@ func (l *Loader) Start( return nil case <-ticker.C: - targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, spec.Provider.HTTP.URL, spec.Provider.HTTP.Token) - if err != nil { - logger.Error(err, "failed to fetch targets from HTTP endpoint") - continue - } - - var messages []core.DiscoveryMessage - for _, target := range targets { - messages = append(messages, core.DiscoveryMessage{ - Target: target, - Event: core.CREATE, - }) - } - - // Non-blocking context-aware send - select { - case out <- messages: - logger.Info( - "emitted target snapshot", - "count", len(messages), - ) - case <-ctx.Done(): - logger.Info("context cancelled while emitting targets") - return nil - } + fetchAndEmit() } } } -func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http.Client, url string, token string) ([]core.DiscoveredTarget, error) { - req, err := http.NewRequestWithContext( - ctx, - http.MethodGet, - url, - nil, - ) +func (l *Loader) fetchTargetsFromHTTPEndpoint( + ctx context.Context, + client *http.Client, + url string, + token string, +) ([]core.DiscoveredTarget, error) { + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("creating HTTP request failed: %w", err) } + req.Header.Set("Accept", "application/json") req.Header.Set("Authorization", "Token "+token) resp, err := client.Do(req) if err != nil { - return nil, err + return nil, fmt.Errorf("HTTP request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) } var targets []core.DiscoveredTarget if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil { - return nil, err + return nil, fmt.Errorf("failed to decode HTTP response: %w", err) } return targets, nil From e76c6f35c7d9bf06ae79a7677e09e78cc5bedebf Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:10:07 +0000 Subject: [PATCH 034/246] restructure discovery structs --- .../discovery/core/loader_interface.go | 2 +- .../discovery/core/message_interface.go | 8 ++++++++ internal/controller/discovery/core/types.go | 19 +++++++++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 internal/controller/discovery/core/message_interface.go diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 2b87a0a..f8e343b 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -12,7 +12,7 @@ type Loader interface { // Name returns the unique loader identifier e.g. "http_pull" Name() string - // Start begins discovery and pushes target snapshots into the out channel + // Start begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is cancelled Start( ctx context.Context, diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go new file mode 100644 index 0000000..0836bc6 --- /dev/null +++ b/internal/controller/discovery/core/message_interface.go @@ -0,0 +1,8 @@ +package core + +type DiscoveryMessage interface { + isDiscoveryMessage() +} + +func (DiscoveryEvent) isDiscoveryMessage() {} +func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 406a22b..f56eaa2 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -9,14 +9,21 @@ type DiscoveredTarget struct { } const ( - DELETE DiscoveryEvent = 0 - CREATE DiscoveryEvent = 1 - UPDATE DiscoveryEvent = 2 + DELETE EventAction = 0 + CREATE EventAction = 1 + UPDATE EventAction = 2 ) -type DiscoveryEvent int +type EventAction int -type DiscoveryMessage struct { +type DiscoveryEvent struct { Target DiscoveredTarget - Event DiscoveryEvent + Event EventAction +} + +type DiscoverySnapshot struct { + Target []DiscoveredTarget + Event EventAction + SnapshotID string + IsLastChunk bool } From 3c18fb54fbb78db867ebba48ef3ff7e0b58e5e0a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:11:42 +0000 Subject: [PATCH 035/246] offload sending logic from loader implementation --- internal/controller/discovery/core/sender.go | 69 +++++++++++++++++++ .../discovery/loaders/http_pull/loader.go | 45 ++++++------ 2 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 internal/controller/discovery/core/sender.go diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go new file mode 100644 index 0000000..84de206 --- /dev/null +++ b/internal/controller/discovery/core/sender.go @@ -0,0 +1,69 @@ +package core + +import ( + "context" +) + +// sendMessages sends discovery messages over a channel in a context-aware manner +func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error { + select { + case <-ctx.Done(): + return ctx.Err() + case out <- messages: + } + return nil +} + +// createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots +func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { + if chunkSize <= 0 { + chunkSize = 1 + } + + var snapshots []DiscoverySnapshot + totalTargets := len(targets) + + for i := 0; i < totalTargets; i += chunkSize { + end := i + chunkSize + if end > totalTargets { + end = totalTargets + } + + chunk := targets[i:end] + snapshots = append(snapshots, DiscoverySnapshot{ + Target: chunk, + SnapshotID: snapshotID, + IsLastChunk: (end == totalTargets), + }) + } + + return snapshots +} + +// SendSnapshot sends discovered targets as a snapshot over a channel in chunks +func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { + snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) + + for _, snapshot := range snapshots { + // Convert DiscoverySnapshot to DiscoveryMessage interface + messages := make([]DiscoveryMessage, 1) + messages[0] = snapshot + + if err := sendMessages(ctx, out, messages); err != nil { + return err + } + } + + return nil +} + +// SendEvents sends discovery messages over channel in a context-aware manner +func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error { + // Convert DiscoveryEvent slice to DiscoveryMessage slice + messages := make([]DiscoveryMessage, len(events)) + for i, msg := range events { + messages[i] = msg + } + + return sendMessages(ctx, out, messages) +} diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 22868b2..94660d0 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -2,12 +2,18 @@ package http_pull import ( "context" + "fmt" "time" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/google/uuid" +) + +const ( + chunkSize = 100 ) type Loader struct{} @@ -27,7 +33,11 @@ func (l *Loader) Start( spec gnmicv1alpha1.TargetSourceSpec, out chan<- []core.DiscoveryMessage, ) error { - logger := log.FromContext(ctx).WithValues("loader", l.Name()) + logger := log.FromContext(ctx).WithValues( + "component", "loader", + "name", l.Name(), + "targetsource", targetsourceName, + ) logger.Info("HTTP pull loader started") @@ -43,35 +53,22 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - targets := []core.DiscoveryMessage{ + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + targets := []core.DiscoveredTarget{ { - Target: core.DiscoveredTarget{ - Name: "ceos1", - Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, + Name: "ceos1", + Address: "clab-3-nodes-ceos1:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, }, { - Target: core.DiscoveredTarget{ - Name: "leaf1", - Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, + Name: "leaf1", + Address: "clab-3-nodes-leaf1:57400", + Labels: map[string]string{"TargetSource": targetsourceName}, }, } - // Non-blocking context-aware send - select { - case out <- targets: - logger.V(1).Info( - "emitted target snapshot", - "count", len(targets), - ) - case <-ctx.Done(): - logger.Info("context cancelled while emitting targets") - return nil + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + return err } } } From 86ab0f35818be90b429177c013a78b7c3fed083f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:16:08 +0000 Subject: [PATCH 036/246] implement type assertion based on received message --- .../controller/discovery/target_manager.go | 75 +++++++++++++------ 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 245942d..f44e33c 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -9,23 +9,26 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" ) -// TargetManager consumes discovered targets and applies them to Kubernetes. +// TargetManager consumes discovered targets and applies them to Kubernetes type TargetManager struct { client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource in <-chan []core.DiscoveryMessage + collected map[string][]core.DiscoveredTarget } -// NewTargetManager wires a TargetManager instance. +// NewTargetManager wires a TargetManager instance func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager { return &TargetManager{ client: c, scheme: s, targetSource: ts, in: in, + collected: make(map[string][]core.DiscoveredTarget), } } @@ -43,28 +46,54 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Info("target manager stopped") return nil - case targets := <-m.in: - logger.Info( - "received discovered targets", - "count", len(targets), - ) + case messages := <-m.in: + for _, message := range messages { + // Type assert to determine if this is a snapshot or event + switch msg := message.(type) { + case core.DiscoverySnapshot: + // Collect snapshot chunks + logger.Info( + "received snapshot chunk", + "snapshotID", msg.SnapshotID, + "targetCount", len(msg.Target), + ) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...) + if msg.IsLastChunk { + m.processSnapshot(msg.SnapshotID, logger) + } - // List existing Target CRs owned by this TargetSource - // var existing gnmicv1alpha1.TargetList - // if err := m.client.List( - // ctx, - // &existing, - // client.MatchingLabels{ - // "gnmic.dev/targetsource": m.targetsource, - // }, - // ); err != nil { - // return err - // } - - // TODO: Target Lifecycle Management - // 1. Compare and determine which Targets to create/update/delete - // 2. Create/update/delete Target CRs accordingly - // 3. Update TargetSource status with sync results + case core.DiscoveryEvent: + // Process individual event-driven update + logger.Info( + "received discovery event", + "target", msg.Target.Name, + ) + switch msg.Event { + case core.CREATE: + logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", msg.Target.Name) + } + } + } } } } + +// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly +func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { + targets := m.collected[snapshotID] + delete(m.collected, snapshotID) + + logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets)) + + if m.targetSource.Spec.Provider.HTTP != nil { + logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name) + } + + for _, target := range targets { + logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels) + } +} From 8b36d7dd34e1200a50cc1c9c1176e9cbfbf97371 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:28:51 +0000 Subject: [PATCH 037/246] add http_push skeleton --- .../discovery/loaders/http_push/loader.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 95dc1e9..2e4ae0e 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -1,4 +1,53 @@ package http_push +import ( + "context" + "errors" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "sigs.k8s.io/controller-runtime/pkg/log" +) + // this file implements the logic receive target updates via HTTP push // REST API defined internal/apiserver + +// Loader implements the HTTP pull discovery mechanism +type Loader struct{} + +// New returns a new http_pull loader instance +func New() core.Loader { + return &Loader{} +} + +func (l *Loader) Name() string { + return "http_push" +} + +func (l *Loader) Start( + ctx context.Context, + targetsourceName string, + spec gnmicv1alpha1.TargetSourceSpec, + out chan<- []core.DiscoveryMessage, +) error { + logger := log.FromContext(ctx).WithValues( + "component", "loader", + "name", l.Name(), + "targetsource", targetsourceName, + ) + logger.Info("HTTP push loader started") + + // Input Validation of spec + if spec.Provider == nil || spec.Provider.HTTP == nil { + return errors.New("http_push loader requires spec.provider.http to be set") + } + + // Receive target updates via HTTP push + var targetEvents []core.DiscoveryEvent + + if err := core.SendEvents(ctx, out, targetEvents); err != nil { + logger.Error(err, "failed to send events") + return nil + } + return nil +} From efbf727aed95de42e0a582333e90262689a2a3e5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:30:22 +0000 Subject: [PATCH 038/246] add http_push skeleton --- .../discovery/loaders/http_push/loader.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 95dc1e9..2e4ae0e 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -1,4 +1,53 @@ package http_push +import ( + "context" + "errors" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "sigs.k8s.io/controller-runtime/pkg/log" +) + // this file implements the logic receive target updates via HTTP push // REST API defined internal/apiserver + +// Loader implements the HTTP pull discovery mechanism +type Loader struct{} + +// New returns a new http_pull loader instance +func New() core.Loader { + return &Loader{} +} + +func (l *Loader) Name() string { + return "http_push" +} + +func (l *Loader) Start( + ctx context.Context, + targetsourceName string, + spec gnmicv1alpha1.TargetSourceSpec, + out chan<- []core.DiscoveryMessage, +) error { + logger := log.FromContext(ctx).WithValues( + "component", "loader", + "name", l.Name(), + "targetsource", targetsourceName, + ) + logger.Info("HTTP push loader started") + + // Input Validation of spec + if spec.Provider == nil || spec.Provider.HTTP == nil { + return errors.New("http_push loader requires spec.provider.http to be set") + } + + // Receive target updates via HTTP push + var targetEvents []core.DiscoveryEvent + + if err := core.SendEvents(ctx, out, targetEvents); err != nil { + logger.Error(err, "failed to send events") + return nil + } + return nil +} From 60a5eb3a34a741077ec465b20266ecc58eecc59b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:43:15 +0000 Subject: [PATCH 039/246] refactor targetsource_controller.go --- .../controller/targetsource_controller.go | 130 +++++++++++------- lab/dev/resources/targetsources/ctest1.yaml | 3 +- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8cd6f68..9fb587f 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -55,92 +55,124 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) + logger := log.FromContext(ctx).WithValues( + "Name", req.NamespacedName, + ) + targetSource, err := r.getTargetSource(ctx, req.NamespacedName) + if err != nil { + return ctrl.Result{}, err + } + + // Handle deletion with finalizer + if !targetSource.DeletionTimestamp.IsZero() { + return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource) + } + + // Ensure finalizer is set + if err := r.ensureFinalizer(ctx, targetSource); err != nil { + return ctrl.Result{}, err + } + + // Check if pipeline is already running + if r.isPipelineRunning(req.NamespacedName) { + return ctrl.Result{}, nil + } + + // Start discovery pipeline + if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil { + return ctrl.Result{}, err + } + + logger.Info("TargetSource pipeline started") + return ctrl.Result{}, nil +} + +// getTargetSource retrieves a TargetSource by name, handling cleanup if not found +func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource - if err := r.Get(ctx, req.NamespacedName, &targetSource); err != nil { + if err := r.Get(ctx, key, &targetSource); err != nil { // If the TargetSource no longer exists, ensure runtime cleanup if client.IgnoreNotFound(err) == nil { - r.stopDiscovery(req.NamespacedName) + r.stopDiscovery(key) } - return ctrl.Result{}, client.IgnoreNotFound(err) + return nil, client.IgnoreNotFound(err) } + return &targetSource, nil +} - logger.Info("reconciling TargetSource", "name", targetSource.Name) - - // Handle deletion with finalizer - if !targetSource.DeletionTimestamp.IsZero() { - logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) +// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer +func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) - r.stopDiscovery(req.NamespacedName) + r.stopDiscovery(key) - // Remove finalizer if exists - if controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) { - controllerutil.RemoveFinalizer(&targetSource, targetSourceFinalizer) - if err := r.Update(ctx, &targetSource); err != nil { - return ctrl.Result{}, err - } + // Remove finalizer if exists + if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer) + if err := r.Update(ctx, targetSource); err != nil { + return ctrl.Result{}, err } + } - return ctrl.Result{}, nil + return ctrl.Result{}, nil +} + +// ensureFinalizer adds the finalizer if not present and updates the TargetSource +func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error { + if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + return nil } - // Ensure finalizer is set - if !controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) { - controllerutil.AddFinalizer(&targetSource, targetSourceFinalizer) - if err := r.Update(ctx, &targetSource); err != nil { - return ctrl.Result{}, err - } - // Requeue to continue with a clean state - return ctrl.Result{}, nil + controllerutil.AddFinalizer(targetSource, targetSourceFinalizer) + if err := r.Update(ctx, targetSource); err != nil { + return err } - // TODO: - // 1. Check if a pipeline is already running for this TargetSource - // 2. If not, create and start a new pipeline: - // a. Create a Loader based on TargetSource spec - // b. Start the Loader in a new goroutine, passing a channel for discovered targets - // c. Start a TargetManager in another goroutine to consume discovered targets and manage Target CRs - // 3. If yes, check if the spec has changed and restart the pipeline if needed + return nil +} +// isPipelineRunning checks if a discovery pipeline is already running for the given key +func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { r.mu.Lock() - _, exists := r.running[req.NamespacedName] - r.mu.Unlock() + defer r.mu.Unlock() - // If a targetsource loader exists, return immediately without starting - // any new loader or target manager - if exists { - return ctrl.Result{}, nil - } + _, exists := r.running[key] + return exists +} - loader, err := discovery.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec) +// startDiscoveryPipeline creates and starts the loader and target manager +func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { + loader, err := discovery.NewLoader( + targetSource.ObjectMeta.Name, + targetSource.ObjectMeta.Namespace, + targetSource.Spec, + ) if err != nil { - return ctrl.Result{}, err + return err } runtimeCtx, cancel := context.WithCancel(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, 10) - // start loader + // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) - // start target manager + // Start target manager manager := discovery.NewTargetManager( r.Client, r.Scheme, - &targetSource, + targetSource, targetChannel, ) go manager.Run(runtimeCtx) r.mu.Lock() - r.running[req.NamespacedName] = runningSource{cancel: cancel} + r.running[key] = runningSource{cancel: cancel} r.mu.Unlock() - logger.Info("TargetSource pipeline started", "name", targetSource.Name) - - return ctrl.Result{}, nil + return nil } // stopDiscovery stops and removes a running discovery pipeline diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml index e0aea43..bdb1bf8 100644 --- a/lab/dev/resources/targetsources/ctest1.yaml +++ b/lab/dev/resources/targetsources/ctest1.yaml @@ -5,7 +5,8 @@ metadata: spec: provider: http: - url: http://inventory-service:8080/targets + url: http://srbsci-121:8081/api/dcim/devices/?export=test + token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1 labels: source: inventory type: http From 1bc5d2be5e429076d9bb95578cf56eb2a42fda14 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:49:59 +0000 Subject: [PATCH 040/246] remove targetsource ressource to not impact main --- lab/dev/resources/targetsources/ctest1.yaml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 lab/dev/resources/targetsources/ctest1.yaml diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml deleted file mode 100644 index bdb1bf8..0000000 --- a/lab/dev/resources/targetsources/ctest1.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: http-discovery -spec: - provider: - http: - url: http://srbsci-121:8081/api/dcim/devices/?export=test - token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1 - labels: - source: inventory - type: http - profile: eos \ No newline at end of file From 14e7765ae44c19dad961dc367a7a2da4ff818190 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 18:09:17 +0000 Subject: [PATCH 041/246] add batching to DiscoveryEvent's --- internal/controller/discovery/core/sender.go | 29 +++++++++++++++---- .../discovery/loaders/http_push/loader.go | 3 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index 84de206..cc8e3c1 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -58,12 +58,29 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] } // SendEvents sends discovery messages over channel in a context-aware manner -func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error { - // Convert DiscoveryEvent slice to DiscoveryMessage slice - messages := make([]DiscoveryMessage, len(events)) - for i, msg := range events { - messages[i] = msg +func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { + if chunkSize <= 0 { + chunkSize = 1 } - return sendMessages(ctx, out, messages) + totalEvents := len(events) + for i := 0; i < totalEvents; i += chunkSize { + end := i + chunkSize + if end > totalEvents { + end = totalEvents + } + + chunk := events[i:end] + // Convert DiscoveryEvent chunk to DiscoveryMessage slice + messages := make([]DiscoveryMessage, len(chunk)) + for j, event := range chunk { + messages[j] = event + } + + if err := sendMessages(ctx, out, messages); err != nil { + return err + } + } + + return nil } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 2e4ae0e..572df1d 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -44,8 +44,9 @@ func (l *Loader) Start( // Receive target updates via HTTP push var targetEvents []core.DiscoveryEvent + const chunkSize = 100 - if err := core.SendEvents(ctx, out, targetEvents); err != nil { + if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil { logger.Error(err, "failed to send events") return nil } From b4337ead8b4eb7f3bb3b764f2141707f69698483 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 18:26:33 +0000 Subject: [PATCH 042/246] refactored sender.go --- internal/controller/discovery/core/sender.go | 65 ++++++++++--------- internal/controller/discovery/core/types.go | 2 +- .../controller/discovery/target_manager.go | 4 +- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index cc8e3c1..3e6b4aa 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -14,6 +14,24 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [ return nil } +// forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize +func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { + if chunkSize <= 0 { + chunkSize = 1 + } + + for i := 0; i < total; i += chunkSize { + end := i + chunkSize + if end > total { + end = total + } + if err := fn(i, end); err != nil { + return err + } + } + return nil +} + // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { if chunkSize <= 0 { @@ -23,19 +41,15 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu var snapshots []DiscoverySnapshot totalTargets := len(targets) - for i := 0; i < totalTargets; i += chunkSize { - end := i + chunkSize - if end > totalTargets { - end = totalTargets - } - + _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] snapshots = append(snapshots, DiscoverySnapshot{ - Target: chunk, + Targets: chunk, SnapshotID: snapshotID, IsLastChunk: (end == totalTargets), }) - } + return nil + }) return snapshots } @@ -45,7 +59,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { - // Convert DiscoverySnapshot to DiscoveryMessage interface + // Convert DiscoverySnapshot to DiscoveryMessage messages := make([]DiscoveryMessage, 1) messages[0] = snapshot @@ -57,30 +71,23 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] return nil } +func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { + message := make([]DiscoveryMessage, len(events)) + for i, event := range events { + message[i] = event + } + return message +} + // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { if chunkSize <= 0 { chunkSize = 1 } + messages := eventsToMessages(events) + total := len(messages) - totalEvents := len(events) - for i := 0; i < totalEvents; i += chunkSize { - end := i + chunkSize - if end > totalEvents { - end = totalEvents - } - - chunk := events[i:end] - // Convert DiscoveryEvent chunk to DiscoveryMessage slice - messages := make([]DiscoveryMessage, len(chunk)) - for j, event := range chunk { - messages[j] = event - } - - if err := sendMessages(ctx, out, messages); err != nil { - return err - } - } - - return nil + return forEachChunk(total, chunkSize, func(i, end int) error { + return sendMessages(ctx, out, messages[i:end]) + }) } diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index f56eaa2..cac249d 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -22,7 +22,7 @@ type DiscoveryEvent struct { } type DiscoverySnapshot struct { - Target []DiscoveredTarget + Targets []DiscoveredTarget Event EventAction SnapshotID string IsLastChunk bool diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index f44e33c..153723c 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -55,9 +55,9 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Info( "received snapshot chunk", "snapshotID", msg.SnapshotID, - "targetCount", len(msg.Target), + "targetCount", len(msg.Targets), ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) if msg.IsLastChunk { m.processSnapshot(msg.SnapshotID, logger) } From 30f3ecb6f291c55d7cdd2b73e9257189acacd106 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 19:20:56 +0000 Subject: [PATCH 043/246] load buffer and chunk size from env variable --- cmd/main.go | 10 ++++++++-- internal/controller/discovery/core/sender.go | 11 ----------- internal/controller/discovery/core/types.go | 4 ++++ internal/controller/discovery/loader.go | 4 ++-- .../discovery/loaders/http_pull/loader.go | 16 +++++++--------- .../discovery/loaders/http_push/loader.go | 13 +++++++------ internal/controller/targetsource_controller.go | 10 +++++++++- 7 files changed, 37 insertions(+), 31 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4c37a0d..eacdee5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -64,6 +64,8 @@ func main() { var probeAddr string var devMode bool var apiAddr string + var discoveryChunkSize int + var discoveryBufferSize int flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") @@ -71,6 +73,8 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.IntVar(&discoveryChunkSize, "discovery-chunk-size", 100, "Maximum number of targets/events sent in a single discovery message.") + flag.IntVar(&discoveryBufferSize, "discovery-buffer-size", 10, "Amount of discovery messages that can be queued in the channel buffer.") opts := zap.Options{ Development: devMode, } @@ -117,8 +121,10 @@ func main() { os.Exit(1) } if err := (&controller.TargetSourceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + BufferSize: discoveryBufferSize, + ChunkSize: discoveryChunkSize, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index 3e6b4aa..843f30e 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -16,10 +16,6 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [ // forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { - if chunkSize <= 0 { - chunkSize = 1 - } - for i := 0; i < total; i += chunkSize { end := i + chunkSize if end > total { @@ -34,10 +30,6 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { - if chunkSize <= 0 { - chunkSize = 1 - } - var snapshots []DiscoverySnapshot totalTargets := len(targets) @@ -81,9 +73,6 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { - if chunkSize <= 0 { - chunkSize = 1 - } messages := eventsToMessages(events) total := len(messages) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index cac249d..69a407e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,5 +1,9 @@ package core +type LoaderConfig struct { + ChunkSize int +} + // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index ad1e83f..e0834c0 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -9,12 +9,12 @@ import ( ) // NewLoader creates a loader by name -func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { loaderName := namespace + "/" + name switch { case spec.Provider.HTTP != nil: - return http_pull.New(), nil + return http_pull.New(cfg), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 94660d0..8213c8a 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -12,15 +12,13 @@ import ( "github.com/google/uuid" ) -const ( - chunkSize = 100 -) - -type Loader struct{} +type Loader struct { + cfg core.LoaderConfig +} -// New instantiates the http_pull loader -func New() core.Loader { - return &Loader{} +// New instantiates the http_pull loader with the provided config +func New(cfg core.LoaderConfig) core.Loader { + return &Loader{cfg: cfg} } func (l *Loader) Name() string { @@ -67,7 +65,7 @@ func (l *Loader) Start( }, } - if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 572df1d..025176f 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -13,11 +13,13 @@ import ( // REST API defined internal/apiserver // Loader implements the HTTP pull discovery mechanism -type Loader struct{} +type Loader struct{ + cfg core.LoaderConfig +} -// New returns a new http_pull loader instance -func New() core.Loader { - return &Loader{} +// New returns a new http_push loader instance configured with cfg +func New(cfg core.LoaderConfig) core.Loader { + return &Loader{cfg: cfg} } func (l *Loader) Name() string { @@ -44,9 +46,8 @@ func (l *Loader) Start( // Receive target updates via HTTP push var targetEvents []core.DiscoveryEvent - const chunkSize = 100 - if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil { + if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { logger.Error(err, "failed to send events") return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9fb587f..fce6742 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -45,6 +45,9 @@ type TargetSourceReconciler struct { mu sync.Mutex running map[client.ObjectKey]runningSource + + BufferSize int + ChunkSize int } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -144,17 +147,22 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { // startDiscoveryPipeline creates and starts the loader and target manager func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { + cfg := core.LoaderConfig{ + ChunkSize: r.ChunkSize, + } + loader, err := discovery.NewLoader( targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec, + cfg, ) if err != nil { return err } runtimeCtx, cancel := context.WithCancel(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, 10) + targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) From 6684aa9c33e9a21108218e4765dfb8be9bd6c929 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 17 Apr 2026 17:18:47 -0600 Subject: [PATCH 044/246] ran go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f236ded..827da2a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 github.com/go-logr/logr v1.4.3 + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.27.3 github.com/onsi/gomega v1.38.3 github.com/openconfig/gnmic/pkg/api v0.1.10 @@ -47,7 +48,6 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect - github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect From c952cbf267a36aa4dc1acebc89ffafed34890700 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 17 Apr 2026 18:39:33 -0600 Subject: [PATCH 045/246] changed events to delete/apply and implemented draft with snapshots --- internal/controller/discovery/core/types.go | 4 +- .../controller/discovery/target_manager.go | 134 +++++++++--------- 2 files changed, 65 insertions(+), 73 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index cac249d..8368d06 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -10,8 +10,7 @@ type DiscoveredTarget struct { const ( DELETE EventAction = 0 - CREATE EventAction = 1 - UPDATE EventAction = 2 + APPLY EventAction = 1 ) type EventAction int @@ -23,7 +22,6 @@ type DiscoveryEvent struct { type DiscoverySnapshot struct { Targets []DiscoveredTarget - Event EventAction SnapshotID string IsLastChunk bool } diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 0d9bd94..e34a272 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -3,7 +3,9 @@ package discovery import ( "context" "fmt" + "maps" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -73,77 +75,20 @@ func (m *TargetManager) Run(ctx context.Context) error { case core.DiscoveryEvent: // Process individual event-driven update - logger.Info( - "received discovery event", - "target", msg.Target.Name, - ) - switch msg.Event { - case core.CREATE: - logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - target := &gnmicv1alpha1.Target{ - ObjectMeta: metav1.ObjectMeta{ - Name: msg.Target.Name, - Namespace: m.targetSource.ObjectMeta.Namespace, - Labels: map[string]string{ - "gnmic.io/source": m.targetSource.ObjectMeta.Name, - }, - }, - Spec: gnmicv1alpha1.TargetSpec{ - Address: msg.Target.Address, - Profile: "default", - }, - } - err := controllerutil.SetControllerReference(m.targetSource, target, m.scheme) - if err != nil { - logger.Error(err, "error setting the owner reference") - } - - err = m.client.Create(ctx, target) - if err != nil { - logger.Error(err, "error creating target object") - } - logger.Info(fmt.Sprintf("created new target object %s/%s", target.ObjectMeta.Namespace, target.ObjectMeta.Name)) - - case core.UPDATE: - logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - existing := &gnmicv1alpha1.Target{} - newSpec := gnmicv1alpha1.TargetSpec{ - Address: msg.Target.Address, - Profile: "default", - } - - err := m.client.Get(ctx, types.NamespacedName{ - Name: msg.Target.Name, - Namespace: m.targetSource.Namespace, - }, existing) - if err != nil { - logger.Error(err, "error fetching existing target object") - } - - existing.Spec = newSpec - - err = m.client.Update(ctx, existing) - if err != nil { - logger.Error(err, "error updating object") - } - logger.Info(fmt.Sprintf("updated existing target object %s/%s", existing.ObjectMeta.Namespace, existing.ObjectMeta.Name)) + logger.Info(fmt.Sprintf("received discovery event for target %s", msg.Target.Name)) + switch msg.Event { case core.DELETE: - logger.Info("Would delete target", "name", msg.Target.Name) - existing := &gnmicv1alpha1.Target{} - err := m.client.Get(ctx, types.NamespacedName{ - Name: msg.Target.Name, - Namespace: m.targetSource.Namespace, - }, existing) + err := m.deleteTarget(ctx, msg.Target.Name) if err != nil { - logger.Error(err, "error fetching existing target object") + logger.Error(err, fmt.Sprintf("error deleting target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) } - - err = m.client.Delete(ctx, existing) + logger.Info(fmt.Sprintf("deleted target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) + case core.APPLY: + err := m.applyTarget(ctx, logger, msg.Target.Name, msg.Target.Address) if err != nil { - logger.Error(err, "error deleting the object") + logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) } - logger.Info(fmt.Sprintf("deleted target object %s/%s", m.targetSource.Namespace, msg.Target.Name)) } } } @@ -156,13 +101,62 @@ func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { targets := m.collected[snapshotID] delete(m.collected, snapshotID) - logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets)) + logger.Info(fmt.Sprintf("Processing full snapshot ID: %s, targets: %d", snapshotID, len(targets))) - if m.targetSource.Spec.Provider.HTTP != nil { - logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name) + for _, target := range targets { + err := m.applyTarget(context.Background(), logger, target.Name, target.Address) + if err != nil { + logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, target.Name)) + } } +} - for _, target := range targets { - logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels) +func (m *TargetManager) applyTarget(ctx context.Context, logger logr.Logger, name string, address string) error { + target := &gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: m.targetSource.Namespace, + }, } + + _, err := controllerutil.CreateOrUpdate(ctx, m.client, target, func() error { + labels := map[string]string{ + "gnmic.io/source": m.targetSource.Name, + } + + maps.Copy(labels, m.targetSource.Spec.TargetLabels) + + target.Labels = labels + + target.Spec = gnmicv1alpha1.TargetSpec{ + Address: address, + Profile: m.targetSource.Spec.TargetProfile, + } + + return controllerutil.SetControllerReference(m.targetSource, target, m.scheme) + }) + + logger.Info(fmt.Sprintf("applied target object %s/%s", m.targetSource.ObjectMeta.Namespace, name)) + + return err +} + +func (m *TargetManager) deleteTarget(ctx context.Context, name string) error { + existing := &gnmicv1alpha1.Target{} + err := m.client.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: m.targetSource.Namespace, + }, existing) + if apierrors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + + err = m.client.Delete(ctx, existing) + if apierrors.IsNotFound(err) { + return nil + } + + return err } From dbbcb4f1b47f8f90f86628dae0ded11dfc4c8816 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 20 Apr 2026 14:53:12 +0000 Subject: [PATCH 046/246] can start push loader --- Makefile | 2 +- api/v1alpha1/targetsource_types.go | 7 +++- .../operator.gnmic.dev_targetsources.yaml | 5 +++ internal/controller/discovery/loader.go | 3 ++ .../controller/discovery/loaders/all/all.go | 3 +- .../discovery/loaders/http_push/loader.go | 32 +++++++++++++++++-- .../loaders/http_push/loader_test.go | 2 +- .../resources/targetsources/ctestPull.yaml | 12 +++++++ lab/dev/temp | 0 9 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 lab/dev/resources/targetsources/ctestPull.yaml delete mode 100644 lab/dev/temp diff --git a/Makefile b/Makefile index fdcc2b2..46f1e49 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,7 @@ delete-outputs-dev-lab: ## Delete the outputs for the development lab cluster kubectl delete -f lab/dev/resources/outputs .PHONY: apply-pipelines-dev-lab -apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster + §apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster kubectl apply -f lab/dev/resources/pipelines .PHONY: delete-pipelines-dev-lab diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 3cf029b..057425c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -35,11 +35,16 @@ type TargetSourceSpec struct { // +kubebuilder:validation:MaxProperties=1 type ProviderSpec struct { HTTP *HTTPConfig `json:"http,omitempty"` + PULL *PULLConfig `json:"pull,omitempty"` Consul *ConsulConfig `json:"consul,omitempty"` } +type PULLConfig struct { + URL string `json:"url,omitempty"` // Placeholder for future settings, URL is not actually needed for this interface +} + type HTTPConfig struct { - URL string `json:"url,omitempty"` + URL string `json:"url,omitempty"` } type ConsulConfig struct { diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 0129a88..3369de7 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -59,6 +59,11 @@ spec: url: type: string type: object + pull: + properties: + url: + type: string + type: object type: object type: type: string diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index ad1e83f..24236ac 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -6,6 +6,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) // NewLoader creates a loader by name @@ -15,6 +16,8 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe switch { case spec.Provider.HTTP != nil: return http_pull.New(), nil + case spec.Provider.PULL != nil: + return http_push.New(), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go index c53b98a..731bffa 100644 --- a/internal/controller/discovery/loaders/all/all.go +++ b/internal/controller/discovery/loaders/all/all.go @@ -2,5 +2,6 @@ package all import ( _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" - // _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_push" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) + diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 95dc1e9..df7961f 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -1,4 +1,32 @@ package http_push -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver +import ( + "context" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type Loader struct{} + +// New instantiates the http_pull loader +func New() core.Loader { + return &Loader{} +} + +func (l *Loader) Name() string { + return "push" +} + +func (l *Loader) Start( + ctx context.Context, + targetsourceName string, + spec gnmicv1alpha1.TargetSourceSpec, + out chan<- []core.DiscoveryMessage, +) error { + logger := log.FromContext(ctx).WithValues("loader", l.Name()) + logger.Info("Push loader started") + + return nil +} diff --git a/internal/controller/discovery/loaders/http_push/loader_test.go b/internal/controller/discovery/loaders/http_push/loader_test.go index bb7d848..c75a5a0 100644 --- a/internal/controller/discovery/loaders/http_push/loader_test.go +++ b/internal/controller/discovery/loaders/http_push/loader_test.go @@ -1 +1 @@ -package http_push +package http_push \ No newline at end of file diff --git a/lab/dev/resources/targetsources/ctestPull.yaml b/lab/dev/resources/targetsources/ctestPull.yaml new file mode 100644 index 0000000..118e729 --- /dev/null +++ b/lab/dev/resources/targetsources/ctestPull.yaml @@ -0,0 +1,12 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: pull-loader +spec: + provider: + pull: + url: http://inventory-service:8080/targets + labels: + source: inventory + type: pull + profile: eos \ No newline at end of file diff --git a/lab/dev/temp b/lab/dev/temp deleted file mode 100644 index e69de29..0000000 From d4a9053433983bc43db3b2267c12fe2401902692 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 11:22:05 -0600 Subject: [PATCH 047/246] first implementation for full snapshot processing --- internal/controller/discovery/mapper.go | 47 +++++-------------- .../controller/discovery/target_manager.go | 20 ++++++-- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 6f8dbf0..07c3c01 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -5,51 +5,30 @@ package discovery import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/equality" + "github.com/gnmic/operator/internal/controller/discovery/core" ) type Diff struct { - ToCreate []gnmicv1alpha1.Target - ToUpdate []gnmicv1alpha1.Target - ToDelete []gnmicv1alpha1.Target + ToApply []core.DiscoveredTarget + ToDelete []core.DiscoveredTarget } -func BuildDiff(existing, discovered []gnmicv1alpha1.Target) Diff { +func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) Diff { var diff Diff - existingMap := make(map[string]gnmicv1alpha1.Target) - for _, e := range existing { - key := e.Namespace + "/" + e.Name - existingMap[key] = e - } - - discoveredMap := make(map[string]gnmicv1alpha1.Target) + discoveredMap := make(map[string]core.DiscoveredTarget) for _, e := range discovered { - key := e.Namespace + "/" + e.Name - discoveredMap[key] = e - } - - // Loop for targets to create + update - for _, t := range discovered { - key := t.Namespace + "/" + t.Name - - // Check if target already exists - if e, found := existingMap[key]; found { - // Check if the spec of the target changed - if !equality.Semantic.DeepEqual(e.Spec, t.Spec) { - diff.ToUpdate = append(diff.ToUpdate, t) - } - } else { // Target is new - diff.ToCreate = append(diff.ToCreate, t) - } + discoveredMap[e.Name] = e } - // Loop for targets to delete + // Loop for targets to delete, else they get applied for _, e := range existing { - key := e.Namespace + "/" + e.Name - - if _, found := discoveredMap[key]; !found { - diff.ToDelete = append(diff.ToDelete, e) + if t, found := discoveredMap[e.ObjectMeta.Name]; !found { + diff.ToDelete = append(diff.ToDelete, core.DiscoveredTarget{ + Name: e.ObjectMeta.Name, + }) + } else { + diff.ToApply = append(diff.ToApply, t) } } diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index e34a272..61ed16d 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -103,10 +103,24 @@ func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { logger.Info(fmt.Sprintf("Processing full snapshot ID: %s, targets: %d", snapshotID, len(targets))) - for _, target := range targets { - err := m.applyTarget(context.Background(), logger, target.Name, target.Address) + existing, err := FetchExistingTargets(context.Background(), m.client, *m.targetSource) + if err != nil { + logger.Error(err, "error fetching existing targets") + } + + diff := BuildDiff(existing, targets) + + for _, t := range diff.ToDelete { + err := m.deleteTarget(context.Background(), t.Name) + if err != nil { + logger.Error(err, fmt.Sprintf("error deleting target object %s/%s", m.targetSource.ObjectMeta.Namespace, t.Name)) + } + } + + for _, t := range diff.ToApply { + err := m.applyTarget(context.Background(), logger, t.Name, t.Address) if err != nil { - logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, target.Name)) + logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, t.Name)) } } } From 5af4f5e998f805e6ac7f6412063adae94be5243e Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 12:07:54 -0600 Subject: [PATCH 048/246] fixed mapper function to work for empty existing targets --- .../discovery/loaders/http_pull/loader.go | 57 +++++++++++++------ internal/controller/discovery/mapper.go | 22 +++++-- .../controller/discovery/target_manager.go | 6 ++ 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 94660d0..d9aacdc 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -45,6 +45,8 @@ func (l *Loader) Start( ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() + i := 1 + for { select { case <-ctx.Done(): @@ -52,24 +54,47 @@ func (l *Loader) Start( return nil case <-ticker.C: - // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) - targets := []core.DiscoveredTarget{ - { - Name: "ceos1", - Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - { - Name: "leaf1", - Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - } + if i == 1 { + // Example snapshot (placeholder) + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + targets := []core.DiscoveredTarget{ + { + Name: "ceos1", + Address: "clab-3-nodes-ceos1:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + { + Name: "leaf1", + Address: "clab-3-nodes-leaf1:57400", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + } - if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { - return err + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + return err + } + } else if i == 2 { + // Example snapshot (placeholder) + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + targets := []core.DiscoveredTarget{ + { + Name: "ceos1", + Address: "clab-3-nodes-ceos1:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + { + Name: "leaf2", + Address: "clab-3-nodes-leaf2:57400", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + } + + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + return err + } } + + i++ } } } diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 07c3c01..943ca25 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -16,19 +16,29 @@ type Diff struct { func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) Diff { var diff Diff + existingMap := make(map[string]gnmicv1alpha1.Target) + for _, e := range existing { + existingMap[e.ObjectMeta.Name] = e + } + discoveredMap := make(map[string]core.DiscoveredTarget) - for _, e := range discovered { - discoveredMap[e.Name] = e + for _, d := range discovered { + discoveredMap[d.Name] = d } - // Loop for targets to delete, else they get applied - for _, e := range existing { - if t, found := discoveredMap[e.ObjectMeta.Name]; !found { + for name, e := range existingMap { + if d, found := discoveredMap[name]; !found { diff.ToDelete = append(diff.ToDelete, core.DiscoveredTarget{ Name: e.ObjectMeta.Name, }) } else { - diff.ToApply = append(diff.ToApply, t) + diff.ToApply = append(diff.ToApply, d) + } + } + + for name, d := range discoveredMap { + if _, found := existingMap[name]; !found { + diff.ToApply = append(diff.ToApply, d) } } diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 61ed16d..b585aa7 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -108,8 +108,12 @@ func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { logger.Error(err, "error fetching existing targets") } + logger.Info("fetched targets") + diff := BuildDiff(existing, targets) + logger.Info(fmt.Sprintf("apply targets: %d, delete targets: %d", len(diff.ToApply), len(diff.ToDelete))) + for _, t := range diff.ToDelete { err := m.deleteTarget(context.Background(), t.Name) if err != nil { @@ -123,6 +127,8 @@ func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, t.Name)) } } + + logger.Info("end of snapshot processing") } func (m *TargetManager) applyTarget(ctx context.Context, logger logr.Logger, name string, address string) error { From d5ea4da50e3777ef7040ff5473a884177fb800fb Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 12:17:15 -0600 Subject: [PATCH 049/246] introduce observedGeneration for pipeline restart --- api/v1alpha1/targetsource_types.go | 7 ++++--- api/v1alpha1/zz_generated.deepcopy.go | 2 +- config/crd/bases/operator.gnmic.dev_targetsources.yaml | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index feea000..eb0be17 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -49,9 +49,10 @@ type ConsulConfig struct { // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { - Status string `json:"status"` - TargetsCount int32 `json:"targetsCount"` - LastSync metav1.Time `json:"lastSync"` + Status string `json:"status"` + ObservedGeneration int64 `json:"observedGeneration"` + TargetsCount int32 `json:"targetsCount"` + LastSync metav1.Time `json:"lastSync"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3848412..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index f373822..4c8c866 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -77,6 +77,9 @@ spec: lastSync: format: date-time type: string + observedGeneration: + format: int64 + type: integer status: type: string targetsCount: @@ -84,6 +87,7 @@ spec: type: integer required: - lastSync + - observedGeneration - status - targetsCount type: object From fe086e2204df82fd587a8edafdf374e209f01298 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 12:33:42 -0600 Subject: [PATCH 050/246] tests with targetsource status field to restart pipeline --- api/v1alpha1/targetsource_types.go | 6 +++--- .../operator.gnmic.dev_targetsources.yaml | 3 --- .../discovery/loaders/http_pull/loader.go | 19 +++++++++++++++++-- .../controller/targetsource_controller.go | 12 +++++++++++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index eb0be17..255732c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -49,10 +49,10 @@ type ConsulConfig struct { // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { - Status string `json:"status"` + Status string `json:"status,omitempty"` ObservedGeneration int64 `json:"observedGeneration"` - TargetsCount int32 `json:"targetsCount"` - LastSync metav1.Time `json:"lastSync"` + TargetsCount int32 `json:"targetsCount,omitempty"` + LastSync metav1.Time `json:"lastSync,omitempty"` } //+kubebuilder:object:root=true diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 4c8c866..68b669c 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -86,10 +86,7 @@ spec: format: int32 type: integer required: - - lastSync - observedGeneration - - status - - targetsCount type: object type: object served: true diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index d9aacdc..2d8a9e9 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -54,7 +54,8 @@ func (l *Loader) Start( return nil case <-ticker.C: - if i == 1 { + switch i { + case 1: // Example snapshot (placeholder) snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) targets := []core.DiscoveredTarget{ @@ -73,7 +74,7 @@ func (l *Loader) Start( if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { return err } - } else if i == 2 { + case 2: // Example snapshot (placeholder) snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) targets := []core.DiscoveredTarget{ @@ -89,6 +90,20 @@ func (l *Loader) Start( }, } + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + return err + } + + default: + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + targets := []core.DiscoveredTarget{ + { + Name: "ceos1", + Address: "clab-3-nodes-ceos2:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, + }, + } + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { return err } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9fb587f..993fa61 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -76,7 +76,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request // Check if pipeline is already running if r.isPipelineRunning(req.NamespacedName) { - return ctrl.Result{}, nil + if targetSource.Generation != targetSource.Status.ObservedGeneration { + r.stopDiscovery(req.NamespacedName) + } else { + return ctrl.Result{}, nil + } } // Start discovery pipeline @@ -84,6 +88,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } + // Update TargetSource ObservedGeneration Status field + targetSource.Status.ObservedGeneration = targetSource.Generation + if err := r.Status().Update(ctx, targetSource); err != nil { + return ctrl.Result{}, err + } + logger.Info("TargetSource pipeline started") return ctrl.Result{}, nil } From bd6056815134f7ac7d1e0a1581e1a08e8da6c061 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 12:59:58 -0600 Subject: [PATCH 051/246] added status update to targetmanager --- internal/controller/discovery/target_manager.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index b585aa7..0b630e7 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -91,6 +91,18 @@ func (m *TargetManager) Run(ctx context.Context) error { } } } + + existing, err := FetchExistingTargets(ctx, m.client, *m.targetSource) + if err != nil { + logger.Error(err, "error fetching existing targets") + } + + m.targetSource.Status.TargetsCount = int32(len(existing)) + m.targetSource.Status.LastSync = metav1.Now() + + if err := m.client.Status().Update(ctx, m.targetSource); err != nil { + logger.Error(err, "error updating targetSource status") + } } } } From 1e81e1102a43e8d4e43ad80e2c6cb0c725ab184f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 17:47:30 -0600 Subject: [PATCH 052/246] separated status updates & cleaned up functions --- .../controller/discovery/target_manager.go | 98 ++++++++++++------- .../controller/targetsource_controller.go | 40 +++++++- 2 files changed, 103 insertions(+), 35 deletions(-) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 0b630e7..9a7d2fd 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -2,7 +2,6 @@ package discovery import ( "context" - "fmt" "maps" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,7 +41,10 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // and reconciles Target CRs accordingly func (m *TargetManager) Run(ctx context.Context) error { logger := log.FromContext(ctx). - WithValues("targetSource", m.targetSource) + WithValues( + "targetSource", m.targetSource.ObjectMeta.Name, + "namespace", m.targetSource.ObjectMeta.Namespace, + ) logger.Info("target manager started") @@ -70,80 +72,112 @@ func (m *TargetManager) Run(ctx context.Context) error { ) m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) if msg.IsLastChunk { - m.processSnapshot(msg.SnapshotID, logger) + m.processSnapshot(ctx, msg.SnapshotID, logger) } case core.DiscoveryEvent: // Process individual event-driven update - logger.Info(fmt.Sprintf("received discovery event for target %s", msg.Target.Name)) + logger.Info("received discovery event", + "name", msg.Target.Name, + ) switch msg.Event { case core.DELETE: err := m.deleteTarget(ctx, msg.Target.Name) if err != nil { - logger.Error(err, fmt.Sprintf("error deleting target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) + logger.Error(err, "error deleting target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", msg.Target.Name, + ) + } else { + logger.Info("deleted target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", msg.Target.Name, + ) } - logger.Info(fmt.Sprintf("deleted target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) + case core.APPLY: - err := m.applyTarget(ctx, logger, msg.Target.Name, msg.Target.Address) + err := m.applyTarget(ctx, msg.Target.Name, msg.Target.Address) if err != nil { - logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, msg.Target.Name)) + logger.Error(err, "error applying target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", msg.Target.Name, + ) + + } else { + logger.Info("applied target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", msg.Target.Name, + ) } } } - - existing, err := FetchExistingTargets(ctx, m.client, *m.targetSource) - if err != nil { - logger.Error(err, "error fetching existing targets") - } - - m.targetSource.Status.TargetsCount = int32(len(existing)) - m.targetSource.Status.LastSync = metav1.Now() - - if err := m.client.Status().Update(ctx, m.targetSource); err != nil { - logger.Error(err, "error updating targetSource status") - } } } } } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { +func (m *TargetManager) processSnapshot(ctx context.Context, snapshotID string, logger logr.Logger) { targets := m.collected[snapshotID] delete(m.collected, snapshotID) - logger.Info(fmt.Sprintf("Processing full snapshot ID: %s, targets: %d", snapshotID, len(targets))) + logger.Info("processing full snapshot", + "id", snapshotID, + "numOfTargets", len(targets), + ) - existing, err := FetchExistingTargets(context.Background(), m.client, *m.targetSource) + existing, err := FetchExistingTargets(ctx, m.client, *m.targetSource) if err != nil { logger.Error(err, "error fetching existing targets") + } else { + logger.Info("fetched existing targets", + "numOfTargets", len(existing), + ) } - logger.Info("fetched targets") - diff := BuildDiff(existing, targets) - logger.Info(fmt.Sprintf("apply targets: %d, delete targets: %d", len(diff.ToApply), len(diff.ToDelete))) + logger.Info("built diff", + "numOfTargetsToApply", len(diff.ToApply), + "numOfTargetsToDelete", len(diff.ToDelete), + ) for _, t := range diff.ToDelete { - err := m.deleteTarget(context.Background(), t.Name) + err := m.deleteTarget(ctx, t.Name) if err != nil { - logger.Error(err, fmt.Sprintf("error deleting target object %s/%s", m.targetSource.ObjectMeta.Namespace, t.Name)) + logger.Error(err, "error deleting target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", t.Name, + ) + } else { + logger.Info("deleted target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", t.Name, + ) } } for _, t := range diff.ToApply { - err := m.applyTarget(context.Background(), logger, t.Name, t.Address) + err := m.applyTarget(ctx, t.Name, t.Address) if err != nil { - logger.Error(err, fmt.Sprintf("error applying target object %s/%s", m.targetSource.ObjectMeta.Namespace, t.Name)) + logger.Error(err, "error applying target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", t.Name, + ) + } else { + logger.Info("applied target object", + "namespace", m.targetSource.ObjectMeta.Namespace, + "name", t.Name, + ) } + } logger.Info("end of snapshot processing") } -func (m *TargetManager) applyTarget(ctx context.Context, logger logr.Logger, name string, address string) error { +func (m *TargetManager) applyTarget(ctx context.Context, name string, address string) error { target := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -168,8 +202,6 @@ func (m *TargetManager) applyTarget(ctx context.Context, logger logr.Logger, nam return controllerutil.SetControllerReference(m.targetSource, target, m.scheme) }) - logger.Info(fmt.Sprintf("applied target object %s/%s", m.targetSource.ObjectMeta.Namespace, name)) - return err } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 993fa61..da80030 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -20,7 +20,9 @@ import ( "context" "sync" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -79,7 +81,8 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request if targetSource.Generation != targetSource.Status.ObservedGeneration { r.stopDiscovery(req.NamespacedName) } else { - return ctrl.Result{}, nil + err := r.updateStatus(ctx, targetSource) + return ctrl.Result{}, err } } @@ -88,12 +91,15 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - // Update TargetSource ObservedGeneration Status field targetSource.Status.ObservedGeneration = targetSource.Generation if err := r.Status().Update(ctx, targetSource); err != nil { return ctrl.Result{}, err } + if err := r.updateStatus(ctx, targetSource); err != nil { + return ctrl.Result{}, err + } + logger.Info("TargetSource pipeline started") return ctrl.Result{}, nil } @@ -197,12 +203,42 @@ func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { } } +func (r *TargetSourceReconciler) updateStatus(ctx context.Context, ts *gnmicv1alpha1.TargetSource) error { + // Update TargetSource Status field + var targetList gnmicv1alpha1.TargetList + + err := r.Client.List(ctx, &targetList, + client.InNamespace(ts.Namespace), + client.MatchingLabels{ + "gnmic.io/source": ts.Name, + }, + ) + if err != nil { + return err + } + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &gnmicv1alpha1.TargetSource{} + if err := r.Get(ctx, client.ObjectKeyFromObject(ts), latest); err != nil { + return err + } + + latest.Status.TargetsCount = int32(len(targetList.Items)) + latest.Status.LastSync = metav1.Now() + + return r.Status().Update(ctx, latest) + }) + + return err +} + // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { r.running = make(map[client.ObjectKey]runningSource) return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). + Owns(&gnmicv1alpha1.Target{}). Named("targetsource"). Complete(r) } From d3e708cd1bfedd2797e2e48723fad6b232e7af96 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 18:28:57 -0600 Subject: [PATCH 053/246] restructured target manager logic to handle events --- internal/controller/discovery/core/types.go | 11 +++ internal/controller/discovery/mapper.go | 26 ++++--- .../controller/discovery/target_manager.go | 77 +++++-------------- 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 8368d06..d8ce3d1 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -20,6 +20,17 @@ type DiscoveryEvent struct { Event EventAction } +func (e EventAction) ToString() string { + switch e { + case DELETE: + return "DELETE" + case APPLY: + return "APPLY" + default: + return "UNKNOWN" + } +} + type DiscoverySnapshot struct { Targets []DiscoveredTarget SnapshotID string diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 943ca25..a49b7b4 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -13,8 +13,8 @@ type Diff struct { ToDelete []core.DiscoveredTarget } -func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) Diff { - var diff Diff +func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { + var events []core.DiscoveryEvent existingMap := make(map[string]gnmicv1alpha1.Target) for _, e := range existing { @@ -27,20 +27,22 @@ func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarg } for name, e := range existingMap { - if d, found := discoveredMap[name]; !found { - diff.ToDelete = append(diff.ToDelete, core.DiscoveredTarget{ - Name: e.ObjectMeta.Name, + if _, found := discoveredMap[name]; !found { + events = append(events, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: e.Name, + }, + Event: core.DELETE, }) - } else { - diff.ToApply = append(diff.ToApply, d) } } - for name, d := range discoveredMap { - if _, found := existingMap[name]; !found { - diff.ToApply = append(diff.ToApply, d) - } + for _, d := range discoveredMap { + events = append(events, core.DiscoveryEvent{ + Target: d, + Event: core.APPLY, + }) } - return diff + return events } diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 9a7d2fd..0b50773 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -79,38 +79,10 @@ func (m *TargetManager) Run(ctx context.Context) error { // Process individual event-driven update logger.Info("received discovery event", "name", msg.Target.Name, + "eventAction", msg.Event.ToString(), ) - switch msg.Event { - case core.DELETE: - err := m.deleteTarget(ctx, msg.Target.Name) - if err != nil { - logger.Error(err, "error deleting target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", msg.Target.Name, - ) - } else { - logger.Info("deleted target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", msg.Target.Name, - ) - } - - case core.APPLY: - err := m.applyTarget(ctx, msg.Target.Name, msg.Target.Address) - if err != nil { - logger.Error(err, "error applying target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", msg.Target.Name, - ) - - } else { - logger.Info("applied target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", msg.Target.Name, - ) - } - } + m.processEvent(ctx, msg, logger) } } } @@ -136,45 +108,38 @@ func (m *TargetManager) processSnapshot(ctx context.Context, snapshotID string, ) } - diff := BuildDiff(existing, targets) + events := BuildDiff(existing, targets) - logger.Info("built diff", - "numOfTargetsToApply", len(diff.ToApply), - "numOfTargetsToDelete", len(diff.ToDelete), - ) + for _, e := range events { + m.processEvent(ctx, e, logger) + } + + logger.Info("end of snapshot processing") +} - for _, t := range diff.ToDelete { - err := m.deleteTarget(ctx, t.Name) - if err != nil { - logger.Error(err, "error deleting target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", t.Name, +func (m *TargetManager) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) { + switch event.Event { + case core.DELETE: + if err := m.deleteTarget(ctx, event.Target.Name); err != nil { + logger.Error(err, "error deleting target", + "targetName", event.Target.Name, ) } else { logger.Info("deleted target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", t.Name, + "name", event.Target.Name, ) } - } - - for _, t := range diff.ToApply { - err := m.applyTarget(ctx, t.Name, t.Address) - if err != nil { - logger.Error(err, "error applying target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", t.Name, + case core.APPLY: + if err := m.deleteTarget(ctx, event.Target.Name); err != nil { + logger.Error(err, "error applying target", + "targetName", event.Target.Name, ) } else { logger.Info("applied target object", - "namespace", m.targetSource.ObjectMeta.Namespace, - "name", t.Name, + "name", event.Target.Name, ) } - } - - logger.Info("end of snapshot processing") } func (m *TargetManager) applyTarget(ctx context.Context, name string, address string) error { From 9a78cf4028027cbb0a7220959c01ad6882af99ef Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 20 Apr 2026 18:31:48 -0600 Subject: [PATCH 054/246] fixed small issues --- internal/controller/discovery/mapper.go | 7 +------ internal/controller/discovery/target_manager.go | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index a49b7b4..3c4414d 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -8,12 +8,7 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) -type Diff struct { - ToApply []core.DiscoveredTarget - ToDelete []core.DiscoveredTarget -} - -func BuildDiff(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { +func GenerateEvents(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { var events []core.DiscoveryEvent existingMap := make(map[string]gnmicv1alpha1.Target) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 0b50773..18d6d72 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -108,7 +108,7 @@ func (m *TargetManager) processSnapshot(ctx context.Context, snapshotID string, ) } - events := BuildDiff(existing, targets) + events := GenerateEvents(existing, targets) for _, e := range events { m.processEvent(ctx, e, logger) @@ -130,7 +130,7 @@ func (m *TargetManager) processEvent(ctx context.Context, event core.DiscoveryEv ) } case core.APPLY: - if err := m.deleteTarget(ctx, event.Target.Name); err != nil { + if err := m.applyTarget(ctx, event.Target.Name, event.Target.Address); err != nil { logger.Error(err, "error applying target", "targetName", event.Target.Name, ) From 3aae153c93e11e4080da8fbf1495704dd8cc1f78 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 22 Apr 2026 09:30:40 +0000 Subject: [PATCH 055/246] pass discovery message to loader --- go.mod | 4 +-- go.sum | 14 ++++----- internal/apiserver/apiserver.go | 31 +++++++------------ .../discovery/loaders/http_push/loader.go | 21 +++++++++++++ 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index d268635..507554e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 + github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.0 @@ -18,14 +19,12 @@ require ( ) require ( - github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect - github.com/getkin/kin-openapi v0.133.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -39,7 +38,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect - github.com/oapi-codegen/runtime v1.3.1 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index 81a33fb..63bb743 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= -github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -107,6 +103,8 @@ github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -148,7 +146,6 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -183,11 +180,10 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwd github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= -github.com/oapi-codegen/runtime v1.3.1 h1:RgDY6J4OGQLbRXhG/Xpt3vSVqYpHQS7hN4m85+5xB9g= -github.com/oapi-codegen/runtime v1.3.1/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= @@ -195,6 +191,7 @@ github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= @@ -231,6 +228,7 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/speakeasy-api/jsonpath v0.6.0 h1:IhtFOV9EbXplhyRqsVhHoBmmYjblIRh5D1/g8DHMXJ8= github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= @@ -238,7 +236,6 @@ github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -389,6 +386,7 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWM gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 81b8572..e7314ac 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,12 +5,13 @@ package apiserver import ( "errors" - "fmt" "net/http" "os" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) type APIServer struct { @@ -61,24 +62,16 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // For testing, to see the payload that is being sent - for _, target := range payload { - if target.Name != nil { - fmt.Printf("name: %s, ", *target.Name) - } - if target.Address != nil { - fmt.Printf("address: %s, ", *target.Address) - } - if target.Profile != nil { - fmt.Printf("profile: %s, ", *target.Profile) - } - if target.Tags != nil { - fmt.Printf("tags: %s", *target.Tags) - } - fmt.Printf("\n") + targets := []core.DiscoveryMessage{ + { + Target: core.DiscoveredTarget{ + Name: *payload[0].Name, + Address: *payload[0].Address + ":6030", + Labels: map[string]string{"TargetSource": "targetsourceName"}, + }, + Event: core.CREATE, + }, } - - // TODO: send target received from interface to autodiscover logic via channel. - + http_push.SendTargetToLoader(targets) c.JSON(http.StatusOK, payload) } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index df7961f..25aa9f8 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -2,7 +2,9 @@ package http_push import ( "context" + "fmt" + "github.com/bytedance/gopkg/util/logger" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" "sigs.k8s.io/controller-runtime/pkg/log" @@ -30,3 +32,22 @@ func (l *Loader) Start( return nil } + +func SendTargetToLoader(dm []core.DiscoveryMessage) { + logger.Info("SendTargetToLoader %s", dm) + // for _, target := range payload { + // if target.Name != nil { + // fmt.Printf("name: %s, ", *target.Name) + // } + // if target.Address != nil { + // fmt.Printf("address: %s, ", *target.Address) + // } + // if target.Profile != nil { + // fmt.Printf("profile: %s, ", *target.Profile) + // } + // if target.Tags != nil { + // fmt.Printf("tags: %s", *target.Tags) + // } + fmt.Printf("SentTargetToLoader called") + //} +} From 586001e963125cde484ddead4e16ef11c4939c7b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 22 Apr 2026 12:58:44 +0000 Subject: [PATCH 056/246] rename file to helpers --- internal/controller/discovery/core/{sender.go => helpers.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/controller/discovery/core/{sender.go => helpers.go} (100%) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/helpers.go similarity index 100% rename from internal/controller/discovery/core/sender.go rename to internal/controller/discovery/core/helpers.go From 7430815bb78b417702c6df5b8e85377e63193a4b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 22 Apr 2026 13:03:03 +0000 Subject: [PATCH 057/246] rebuild and reformat --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- internal/controller/discovery/loaders/push/loader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3848412..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go index 5b00081..ec70830 100644 --- a/internal/controller/discovery/loaders/push/loader.go +++ b/internal/controller/discovery/loaders/push/loader.go @@ -13,7 +13,7 @@ import ( // REST API defined internal/apiserver // Loader implements the HTTP pull discovery mechanism -type Loader struct{ +type Loader struct { cfg core.LoaderConfig } From 4fb6755f591f9d37792e43f17b3a1fd048ca1382 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 22 Apr 2026 08:28:22 -0600 Subject: [PATCH 058/246] removed unnecessary map --- internal/controller/discovery/mapper.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 3c4414d..bee897a 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -11,18 +11,13 @@ import ( func GenerateEvents(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { var events []core.DiscoveryEvent - existingMap := make(map[string]gnmicv1alpha1.Target) - for _, e := range existing { - existingMap[e.ObjectMeta.Name] = e - } - discoveredMap := make(map[string]core.DiscoveredTarget) for _, d := range discovered { discoveredMap[d.Name] = d } - for name, e := range existingMap { - if _, found := discoveredMap[name]; !found { + for _, e := range existing { + if _, found := discoveredMap[e.Name]; !found { events = append(events, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: e.Name, @@ -32,7 +27,7 @@ func GenerateEvents(existing []gnmicv1alpha1.Target, discovered []core.Discovere } } - for _, d := range discoveredMap { + for _, d := range discovered { events = append(events, core.DiscoveryEvent{ Target: d, Event: core.APPLY, From aaf9f2ba6545ac41ea5fb7387b937babee0214fa Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 22 Apr 2026 09:59:25 -0600 Subject: [PATCH 059/246] added prefix to targets --- .../controller/discovery/target_manager.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 18d6d72..c9ac079 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -70,6 +70,11 @@ func (m *TargetManager) Run(ctx context.Context) error { "snapshotID", msg.SnapshotID, "targetCount", len(msg.Targets), ) + + for i := range msg.Targets { + msg.Targets[i] = m.normalizeTarget(msg.Targets[i]) + } + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) if msg.IsLastChunk { m.processSnapshot(ctx, msg.SnapshotID, logger) @@ -82,6 +87,7 @@ func (m *TargetManager) Run(ctx context.Context) error { "eventAction", msg.Event.ToString(), ) + msg.Target = m.normalizeTarget(msg.Target) m.processEvent(ctx, msg, logger) } } @@ -110,6 +116,23 @@ func (m *TargetManager) processSnapshot(ctx context.Context, snapshotID string, events := GenerateEvents(existing, targets) + nApply := 0 + nDelete := 0 + + for _, e := range events { + switch e.Event { + case core.APPLY: + nApply++ + case core.DELETE: + nDelete++ + } + } + + logger.Info("generated events", + "numOfApply", nApply, + "numOfDelete", nDelete, + ) + for _, e := range events { m.processEvent(ctx, e, logger) } @@ -189,3 +212,8 @@ func (m *TargetManager) deleteTarget(ctx context.Context, name string) error { return err } + +func (m *TargetManager) normalizeTarget(t core.DiscoveredTarget) core.DiscoveredTarget { + t.Name = m.targetSource.Name + "-" + t.Name + return t +} From 255a1f3facb9f3c6b4e4ae17b4ad1afae0bcd0bd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 07:15:38 +0000 Subject: [PATCH 060/246] consolidate pull and push to http --- internal/controller/discovery/loader.go | 4 +- .../controller/discovery/loaders/all/all.go | 3 +- .../loaders/{pull => http}/loader.go | 10 ++-- .../discovery/loaders/http/loader_test.go | 1 + .../discovery/loaders/pull/loader_test.go | 1 - .../discovery/loaders/push/loader.go | 55 ------------------- .../discovery/loaders/push/loader_test.go | 1 - 7 files changed, 9 insertions(+), 66 deletions(-) rename internal/controller/discovery/loaders/{pull => http}/loader.go (89%) create mode 100644 internal/controller/discovery/loaders/http/loader_test.go delete mode 100644 internal/controller/discovery/loaders/pull/loader_test.go delete mode 100644 internal/controller/discovery/loaders/push/loader.go delete mode 100644 internal/controller/discovery/loaders/push/loader_test.go diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index 64dc8d3..42ce8da 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -5,7 +5,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" - pull "github.com/gnmic/operator/internal/controller/discovery/loaders/pull" + http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name @@ -14,7 +14,7 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe switch { case spec.Provider.HTTP != nil: - return pull.New(cfg), nil + return http.New(cfg), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go index d05604b..3590cda 100644 --- a/internal/controller/discovery/loaders/all/all.go +++ b/internal/controller/discovery/loaders/all/all.go @@ -1,6 +1,5 @@ package all import ( - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/pull" - // _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/push" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) diff --git a/internal/controller/discovery/loaders/pull/loader.go b/internal/controller/discovery/loaders/http/loader.go similarity index 89% rename from internal/controller/discovery/loaders/pull/loader.go rename to internal/controller/discovery/loaders/http/loader.go index 729233d..f014a2f 100644 --- a/internal/controller/discovery/loaders/pull/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -1,4 +1,4 @@ -package pull +package http import ( "context" @@ -16,13 +16,13 @@ type Loader struct { cfg core.LoaderConfig } -// New instantiates the pull loader with the provided config +// New instantiates the http loader with the provided config func New(cfg core.LoaderConfig) core.Loader { return &Loader{cfg: cfg} } func (l *Loader) Name() string { - return "pull" + return "http" } func (l *Loader) Start( @@ -37,7 +37,7 @@ func (l *Loader) Start( "targetsource", targetsourceName, ) - logger.Info("HTTP pull loader started") + logger.Info("HTTP loader started") // Only for debugging: emit a static snapshot every 30 seconds ticker := time.NewTicker(30 * time.Second) @@ -46,7 +46,7 @@ func (l *Loader) Start( for { select { case <-ctx.Done(): - logger.Info("HTTP pull loader stopped") + logger.Info("HTTP loader stopped") return nil case <-ticker.C: diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/loader_test.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/internal/controller/discovery/loaders/http/loader_test.go @@ -0,0 +1 @@ +package http diff --git a/internal/controller/discovery/loaders/pull/loader_test.go b/internal/controller/discovery/loaders/pull/loader_test.go deleted file mode 100644 index 0493bec..0000000 --- a/internal/controller/discovery/loaders/pull/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package pull diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go deleted file mode 100644 index ec70830..0000000 --- a/internal/controller/discovery/loaders/push/loader.go +++ /dev/null @@ -1,55 +0,0 @@ -package push - -import ( - "context" - "errors" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver - -// Loader implements the HTTP pull discovery mechanism -type Loader struct { - cfg core.LoaderConfig -} - -// New returns a new http_push loader instance configured with cfg -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} -} - -func (l *Loader) Name() string { - return "http_push" -} - -func (l *Loader) Start( - ctx context.Context, - targetsourceName string, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { - logger := log.FromContext(ctx).WithValues( - "component", "loader", - "name", l.Name(), - "targetsource", targetsourceName, - ) - logger.Info("HTTP push loader started") - - // Input Validation of spec - if spec.Provider == nil || spec.Provider.HTTP == nil { - return errors.New("http_push loader requires spec.provider.http to be set") - } - - // Receive target updates via HTTP push - var targetEvents []core.DiscoveryEvent - - if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { - logger.Error(err, "failed to send events") - return nil - } - return nil -} diff --git a/internal/controller/discovery/loaders/push/loader_test.go b/internal/controller/discovery/loaders/push/loader_test.go deleted file mode 100644 index 63fdf61..0000000 --- a/internal/controller/discovery/loaders/push/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package push From 9316ca3edffaab10f02ea90ad8e5c4bc2c524c0f Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 09:16:22 +0000 Subject: [PATCH 061/246] create multiple core.DiscoveredTarget --- internal/apiserver/apiserver.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index e7314ac..d760d3b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -62,15 +62,16 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := []core.DiscoveryMessage{ - { + targets := make([]core.DiscoveryMessage, 0, len(payload)) + for _, target := range payload { + targets = append(targets, core.DiscoveryMessage{ Target: core.DiscoveredTarget{ - Name: *payload[0].Name, - Address: *payload[0].Address + ":6030", - Labels: map[string]string{"TargetSource": "targetsourceName"}, + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"TargetSource": "*target.Tags to be"}, }, Event: core.CREATE, - }, + }) } http_push.SendTargetToLoader(targets) c.JSON(http.StatusOK, payload) From bd2b45f63366eaaba0170c37e1783e018049eaca Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 12:34:50 +0000 Subject: [PATCH 062/246] rename target manager to target applier --- .../controller/discovery/{target_manager.go => target_applier.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/controller/discovery/{target_manager.go => target_applier.go} (100%) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_applier.go similarity index 100% rename from internal/controller/discovery/target_manager.go rename to internal/controller/discovery/target_applier.go From 5a561a768f1a2d17e1ed09a40b82884bc512527f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 13:07:25 +0000 Subject: [PATCH 063/246] implement a generic registry --- .../controller/discovery/registry/registry.go | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 internal/controller/discovery/registry/registry.go diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go new file mode 100644 index 0000000..7da0757 --- /dev/null +++ b/internal/controller/discovery/registry/registry.go @@ -0,0 +1,61 @@ +package registry + +import ( + "fmt" + "sync" +) + +/* USAGE + +// create registry once in main.go +discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]() + +// inside targetsource controller, when starting discovery pipeline: +key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName) +if err := discoveryReg.Register(key, out); err != nil { + logger.Error(err, "could not register loader") + return err +} +defer discoveryReg.Unregister(key) + +// CHECK REGISTRY +ch, ok := discoveryReg.Get(ns + "/" + ts) +if !ok { + http.Error(w, "no loader for targetsource", http.StatusNotFound) + return +} +// then deliver payload to ch +*/ + +// Registry is a thread-safe map: key -> channel of T. +type Registry[T any] struct { + mu sync.RWMutex + m map[string]chan<- T +} + +func NewRegistry[T any]() *Registry[T] { + return &Registry[T]{m: make(map[string]chan<- T)} +} + +func (r *Registry[T]) Register(key string, ch chan<- T) error { + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.m[key]; exists { + return fmt.Errorf("already registered: %s", key) + } + r.m[key] = ch + return nil +} + +func (r *Registry[T]) Unregister(key string) { + r.mu.Lock() + delete(r.m, key) + r.mu.Unlock() +} + +func (r *Registry[T]) Get(key string) (chan<- T, bool) { + r.mu.RLock() + ch, ok := r.m[key] + r.mu.RUnlock() + return ch, ok +} From f5481b8f9c7627d9c499c9156afdb3c7c2346146 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 13:08:01 +0000 Subject: [PATCH 064/246] add a discoveryTegistry to share targetchannel between apiserver and target manager --- cmd/main.go | 14 +++++++++---- internal/apiserver/apiserver.go | 4 ++++ .../controller/targetsource_controller.go | 20 ++++++++++++++++--- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eacdee5..5cf8169 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -40,6 +40,8 @@ import ( operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/registry" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -83,6 +85,8 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]() + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsAddr}, @@ -121,10 +125,11 @@ func main() { os.Exit(1) } if err := (&controller.TargetSourceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - BufferSize: discoveryBufferSize, - ChunkSize: discoveryChunkSize, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + BufferSize: discoveryBufferSize, + ChunkSize: discoveryChunkSize, + DiscoveryRegistry: discoveryRegistry, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) @@ -226,6 +231,7 @@ func main() { if apiAddr != "" { apiServer := apiserver.New(apiAddr, clusterReconciler) + apiServer.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index f31abaa..b84eb9a 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,11 +5,15 @@ import ( "net/http" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/registry" ) type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler + + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index fce6742..c714acc 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -30,6 +30,7 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" + "github.com/gnmic/operator/internal/controller/discovery/registry" ) const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" @@ -48,6 +49,8 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int + + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -164,6 +167,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) + registryKey := key.Namespace + "/" + key.Name + if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil { + cancel() + return err + } + // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) @@ -187,12 +196,17 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // for the given TargetSource key func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { r.mu.Lock() - defer r.mu.Unlock() - - if running, ok := r.running[key]; ok { + running, ok := r.running[key] + if ok { running.cancel() delete(r.running, key) } + r.mu.Unlock() + + if ok { + registryKey := key.Namespace + "/" + key.Name + r.DiscoveryRegistry.Unregister(registryKey) + } } // SetupWithManager sets up the controller with the Manager. From e73a2533f915e1747f89ef6bfc28563365466d4f Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 13:45:20 +0000 Subject: [PATCH 065/246] pull interface only in apiserver --- internal/apiserver/apiserver.go | 20 +++-- .../discovery/loaders/push/loader.go | 78 ------------------- .../discovery/loaders/push/loader_test.go | 1 - 3 files changed, 9 insertions(+), 90 deletions(-) delete mode 100644 internal/controller/discovery/loaders/push/loader.go delete mode 100644 internal/controller/discovery/loaders/push/loader_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b984aa5..e4aaf97 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -11,7 +11,6 @@ import ( "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) type APIServer struct { @@ -55,24 +54,23 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to Target struct defined in openapi.yaml and TBD... +// CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := make([]core.DiscoveryMessage, 0, len(payload)) + targets := []core.DiscoveredTarget{} for _, target := range payload { - targets = append(targets, core.DiscoveryMessage{ - Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"TargetSource": "*target.Tags to be done"}, - }, - Event: core.CREATE, + targets = append(targets, core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, }) } - http_push.SendTargetToLoader(targets) + + // discovery / core / helpers / sendEvents to send received udpates to TagetManager + // loader push not needed c.JSON(http.StatusOK, payload) } diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go deleted file mode 100644 index 87938a6..0000000 --- a/internal/controller/discovery/loaders/push/loader.go +++ /dev/null @@ -1,78 +0,0 @@ -package push - -import ( - "context" - "fmt" - - "errors" - - "github.com/bytedance/gopkg/util/logger" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver - -// Loader implements the HTTP pull discovery mechanism -type Loader struct { - cfg core.LoaderConfig -} - -// New returns a new http_push loader instance configured with cfg -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} -} - -func (l *Loader) Name() string { - return "http_push" -} - -func (l *Loader) Start( - ctx context.Context, - targetsourceName string, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { - logger := log.FromContext(ctx).WithValues( - "component", "loader", - "name", l.Name(), - "targetsource", targetsourceName, - ) - logger.Info("HTTP push loader started") - - // Input Validation of spec - if spec.Provider == nil || spec.Provider.HTTP == nil { - return errors.New("http_push loader requires spec.provider.http to be set") - } - - // Receive target updates via HTTP push - var targetEvents []core.DiscoveryEvent - - if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { - logger.Error(err, "failed to send events") - return nil - } - return nil -} - -func SendTargetToLoader(dm []core.DiscoveryMessage) { - logger.Info("SendTargetToLoader %s", dm) - // for _, target := range payload { - // if target.Name != nil { - // fmt.Printf("name: %s, ", *target.Name) - // } - // if target.Address != nil { - // fmt.Printf("address: %s, ", *target.Address) - // } - // if target.Profile != nil { - // fmt.Printf("profile: %s, ", *target.Profile) - // } - // if target.Tags != nil { - // fmt.Printf("tags: %s", *target.Tags) - // } - fmt.Printf("SentTargetToLoader called") - //} -} diff --git a/internal/controller/discovery/loaders/push/loader_test.go b/internal/controller/discovery/loaders/push/loader_test.go deleted file mode 100644 index 63fdf61..0000000 --- a/internal/controller/discovery/loaders/push/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package push From 32cb07705b74192e4f66d69e28b28b42ded63b87 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 15:55:17 +0000 Subject: [PATCH 066/246] add core.SendEvents, ShouldBindJSON doesn't work --- cmd/main.go | 3 +- go.mod | 2 +- internal/apiserver/apiserver.go | 54 ++++++++++++++----- internal/apiserver/gen.go | 50 +++++++++++++---- internal/apiserver/openapi.yaml | 29 +++++++--- .../resources/targetsources/ctestPull.yaml | 12 ----- 6 files changed, 105 insertions(+), 45 deletions(-) delete mode 100644 lab/dev/resources/targetsources/ctestPull.yaml diff --git a/cmd/main.go b/cmd/main.go index 225f750..03d48d7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -233,10 +233,11 @@ func main() { if apiAddr != "" { api, err := apiserver.New(apiAddr, clusterReconciler) - if err != nil { + if err != nil { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } + api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/go.mod b/go.mod index 507554e..0a1d66c 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/uuid v1.6.0 github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index e112560..0aed77f 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -4,6 +4,8 @@ package apiserver // or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( + "context" + "fmt" "net/http" "github.com/gin-gonic/gin" @@ -16,8 +18,8 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler - - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + ChunkSize int + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] // change to lowercase? } func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { @@ -50,21 +52,47 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { - var payload []Target - if err := c.ShouldBindJSON(&payload); err != nil { + // logger.Info("Create Targets called") + var payloadTarget []Target + var payloadTargetSource TargetSource + fmt.Println("Binding Target to PayloadTarget") + if err := c.ShouldBindJSON(&payloadTarget); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + fmt.Printf("err: %s", err.Error) + return + } + fmt.Printf("payloadTarget: %s", payloadTarget) + if err := c.ShouldBindJSON(&payloadTargetSource); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := []core.DiscoveredTarget{} - for _, target := range payload { - targets = append(targets, core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, + + targets := []core.DiscoveryEvent{} + for _, target := range payloadTarget { + event := core.CREATE + switch *target.Operation { + case Create: + event = core.CREATE + case Delete: + event = core.DELETE + } + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, }) } - // discovery / core / helpers / sendEvents to send received udpates to TagetManager - // loader push not needed - c.JSON(http.StatusOK, payload) + ch, ok := a.DiscoveryRegistry.Get(*payloadTargetSource.Namespace + "/" + *payloadTargetSource.Name) + if !ok { + // Error message to be udpated!! + c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) + return + } + + core.SendEvents(context.Background(), ch, targets, a.ChunkSize) + c.JSON(http.StatusOK, payloadTarget) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 88ed5ab..acc383b 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -16,12 +16,40 @@ import ( "github.com/gin-gonic/gin" ) +// Defines values for TargetOperation. +const ( + Create TargetOperation = "create" + Delete TargetOperation = "delete" +) + +// Valid indicates whether the value is a known member of the TargetOperation enum. +func (e TargetOperation) Valid() bool { + switch e { + case Create: + return true + case Delete: + return true + default: + return false + } +} + // Target defines model for Target. type Target struct { - Address *string `json:"address,omitempty"` - Name *string `json:"name,omitempty"` - Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` + Address *string `json:"address,omitempty"` + Name *string `json:"name,omitempty"` + Operation *TargetOperation `json:"operation,omitempty"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` +} + +// TargetOperation defines model for Target.Operation. +type TargetOperation string + +// TargetSource defines model for TargetSource. +type TargetSource struct { + Name *string `json:"name,omitempty"` + Namespace *string `json:"namespace,omitempty"` } // ServerInterface represents all server handlers. @@ -103,12 +131,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", - "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", - "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", - "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", - "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", - "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", + "H4sIAAAAAAAC/4RSTW/bMAz9KwK3oxG73c23oRiKHLYVa2/DDprMJCpsSSOpAUHg/z7oY0k8Z+hJBPlI", + "ke+9Exg/Be/QCUN/AjYHnHQOXzTtUVIUyAcksZjzehgIOYdyDAg9sJB1e5gbcHrCm4U0QIv1LlXRxQn6", + "72AItSA0MOCIgvCjWTcG8js73h4qep/XsILT7X1qQhPpI8yXhP/5ikYSolz57CMZXN/633tSgYM2t6rr", + "b1LKup3PYCvpHNh/+bw16mvmxZP69un5RX182kIDv5E4MwXdptvcVfqcDhZ6+LC533TQQNByyCu2hcVy", + "R84Ez1m2M+fbAXp4WMAaIOTgHZc777u79BjvBF1u1iGM1uT29pWLbsUca5YuTjkr8Z5wBz28ay/2aqu3", + "2gpfybMW4+0pFZsIJvwVLeGQnLUo/p175a9rYQZkQzYUc1Yoq8LqoDgag8y7OI7FQRynSdPxzKiS2mGd", + "kgOqpa65pQ2jzgRWlpbCPKI8jJEF6SnBVsp06VkueYVXhBLJ4fDPco8oyhSYyt/P8zz/CQAA//+Shfmd", + "7gMAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index fccf33e..bd66bb8 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -20,12 +20,24 @@ paths: content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/Target' - + type: object + required: [TargetSource, Target] + properties: + TargetSource: + $ref: '#/components/schemas/TargetSource' + Target: + type: array + items: + $ref: '#/components/schemas/Target' components: schemas: + TargetSource: + type: object + properties: + namespace: + type: string + name: + type: string Target: type: object properties: @@ -39,7 +51,8 @@ components: type: array items: type: string - # username: - # type: string - # password: # not sure if the password is needed - # type: string \ No newline at end of file + operation: + type: string + enum: + - create + - delete \ No newline at end of file diff --git a/lab/dev/resources/targetsources/ctestPull.yaml b/lab/dev/resources/targetsources/ctestPull.yaml deleted file mode 100644 index 118e729..0000000 --- a/lab/dev/resources/targetsources/ctestPull.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: pull-loader -spec: - provider: - pull: - url: http://inventory-service:8080/targets - labels: - source: inventory - type: pull - profile: eos \ No newline at end of file From 8b4f5c9c840a0d01b8350c2ef698f4fdfb8619d2 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 16:20:37 +0000 Subject: [PATCH 067/246] tests with ShouldBindBodyJSON --- internal/apiserver/apiserver.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 0aed77f..7f125cf 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,6 +9,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" @@ -53,20 +54,24 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { // logger.Info("Create Targets called") + var payloadTarget []Target var payloadTargetSource TargetSource fmt.Println("Binding Target to PayloadTarget") - if err := c.ShouldBindJSON(&payloadTarget); err != nil { + // https://gin-gonic.com/en/docs/binding/bind-body-into-different-structs/ + if err := c.ShouldBindBodyWithJSON(&payloadTarget); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) fmt.Printf("err: %s", err.Error) return } fmt.Printf("payloadTarget: %s", payloadTarget) - if err := c.ShouldBindJSON(&payloadTargetSource); err != nil { + if err := c.ShouldBindBodyWithJSON(&payloadTargetSource); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } + // error {"error":"json: cannot unmarshal object into Go value of type []apiserver.Target"} + targets := []core.DiscoveryEvent{} for _, target := range payloadTarget { event := core.CREATE From 22683f4e4b0ee7853f45c8fce20c7d1646317162 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 16:37:08 +0000 Subject: [PATCH 068/246] remove unused event action from DiscoverySnapshot --- internal/controller/discovery/core/types.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 69a407e..61209fd 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -27,7 +27,6 @@ type DiscoveryEvent struct { type DiscoverySnapshot struct { Targets []DiscoveredTarget - Event EventAction SnapshotID string IsLastChunk bool } From 922bbc6a6be0900f27e9aed9c09d6bce1c19caf6 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 07:32:40 +0000 Subject: [PATCH 069/246] rename target manager to target applier --- .../controller/discovery/target_applier.go | 18 +++++++++--------- internal/controller/targetsource_controller.go | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 153723c..3babebf 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -12,8 +12,8 @@ import ( "github.com/go-logr/logr" ) -// TargetManager consumes discovered targets and applies them to Kubernetes -type TargetManager struct { +// TargetApplier consumes discovered targets and applies them to Kubernetes +type TargetApplier struct { client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource @@ -21,9 +21,9 @@ type TargetManager struct { collected map[string][]core.DiscoveredTarget } -// NewTargetManager wires a TargetManager instance -func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager { - return &TargetManager{ +// NewTargetApplier wires a TargetApplier instance +func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier { + return &TargetApplier{ client: c, scheme: s, targetSource: ts, @@ -34,16 +34,16 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *TargetManager) Run(ctx context.Context) error { +func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues("targetSource", m.targetSource) - logger.Info("target manager started") + logger.Info("target applier started") for { select { case <-ctx.Done(): - logger.Info("target manager stopped") + logger.Info("target applier stopped") return nil case messages := <-m.in: @@ -83,7 +83,7 @@ func (m *TargetManager) Run(ctx context.Context) error { } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { +func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) { targets := m.collected[snapshotID] delete(m.collected, snapshotID) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c714acc..78d64d0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -176,8 +176,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) - // Start target manager - manager := discovery.NewTargetManager( + // Start target applier + manager := discovery.NewTargetApplier( r.Client, r.Scheme, targetSource, From 733927fa680c2896c83ee2863f7d2c2b24575448 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 07:51:37 +0000 Subject: [PATCH 070/246] implement key for registry as a comparable --- cmd/main.go | 3 +- internal/apiserver/apiserver.go | 3 +- .../controller/discovery/registry/registry.go | 39 +++++-------------- .../controller/targetsource_controller.go | 13 +++---- 4 files changed, 19 insertions(+), 39 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5cf8169..e4bad31 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,6 +28,7 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -85,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]() + discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b84eb9a..17e5c82 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -7,13 +7,14 @@ import ( "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" + "k8s.io/apimachinery/pkg/types" ) type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 7da0757..1892d28e 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -5,39 +5,18 @@ import ( "sync" ) -/* USAGE - -// create registry once in main.go -discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]() - -// inside targetsource controller, when starting discovery pipeline: -key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName) -if err := discoveryReg.Register(key, out); err != nil { - logger.Error(err, "could not register loader") - return err -} -defer discoveryReg.Unregister(key) - -// CHECK REGISTRY -ch, ok := discoveryReg.Get(ns + "/" + ts) -if !ok { - http.Error(w, "no loader for targetsource", http.StatusNotFound) - return -} -// then deliver payload to ch -*/ - -// Registry is a thread-safe map: key -> channel of T. -type Registry[T any] struct { +// Registry is a thread-safe key -> channel registry +// K must be comparable so it can be used as a map key +type Registry[K comparable, V any] struct { mu sync.RWMutex - m map[string]chan<- T + m map[K]chan<- V } -func NewRegistry[T any]() *Registry[T] { - return &Registry[T]{m: make(map[string]chan<- T)} +func NewRegistry[K comparable, V any]() *Registry[K, V] { + return &Registry[K, V]{m: make(map[K]chan<- V)} } -func (r *Registry[T]) Register(key string, ch chan<- T) error { +func (r *Registry[K, V]) Register(key K, ch chan<- V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { @@ -47,13 +26,13 @@ func (r *Registry[T]) Register(key string, ch chan<- T) error { return nil } -func (r *Registry[T]) Unregister(key string) { +func (r *Registry[K, V]) Unregister(key K) { r.mu.Lock() delete(r.m, key) r.mu.Unlock() } -func (r *Registry[T]) Get(key string) (chan<- T, bool) { +func (r *Registry[K, V]) Get(key K) (chan<- V, bool) { r.mu.RLock() ch, ok := r.m[key] r.mu.RUnlock() diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 78d64d0..d97b3a6 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -21,6 +21,7 @@ import ( "sync" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -50,7 +51,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -149,7 +150,7 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { } // startDiscoveryPipeline creates and starts the loader and target manager -func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { +func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error { cfg := core.LoaderConfig{ ChunkSize: r.ChunkSize, } @@ -167,8 +168,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - registryKey := key.Namespace + "/" + key.Name - if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil { + if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { cancel() return err } @@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // stopDiscovery stops and removes a running discovery pipeline // for the given TargetSource key -func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { +func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { r.mu.Lock() running, ok := r.running[key] if ok { @@ -204,8 +204,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { r.mu.Unlock() if ok { - registryKey := key.Namespace + "/" + key.Name - r.DiscoveryRegistry.Unregister(registryKey) + r.DiscoveryRegistry.Unregister(key) } } From 9d305601d18ae0f9f4d9f0168ec799b15e8b4a2a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 08:10:41 +0000 Subject: [PATCH 071/246] fix error message and add a word of caution for key comparables --- internal/controller/discovery/registry/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 1892d28e..093bd2c 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -7,6 +7,7 @@ import ( // Registry is a thread-safe key -> channel registry // K must be comparable so it can be used as a map key +// DO NOT USE a pointer type as K type Registry[K comparable, V any] struct { mu sync.RWMutex m map[K]chan<- V @@ -20,7 +21,7 @@ func (r *Registry[K, V]) Register(key K, ch chan<- V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { - return fmt.Errorf("already registered: %s", key) + return fmt.Errorf("already registered: %v", key) } r.m[key] = ch return nil From dafa82bb1fd1fbbb5369d14ff82594be38b19ddb Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 08:11:17 +0000 Subject: [PATCH 072/246] consistently use namespaced name as refference to the targetsource --- .../discovery/core/loader_interface.go | 3 ++- internal/controller/discovery/loader.go | 8 ++++---- .../controller/discovery/loaders/http/loader.go | 11 ++++++----- internal/controller/targetsource_controller.go | 17 ++++++++--------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 17cd5f4..8964be8 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -4,6 +4,7 @@ import ( "context" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" ) // Loader defines a pluggable TargetSource loader interface @@ -16,7 +17,7 @@ type Loader interface { // The loader must stop cleanly when ctx is cancelled Start( ctx context.Context, - targetsourceName string, + targetsourceName types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, out chan<- []DiscoveryMessage, ) error diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index 42ce8da..0d8ddd3 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -6,19 +6,19 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" + "k8s.io/apimachinery/pkg/types" ) // NewLoader creates a loader by name -func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { - loaderName := namespace + "/" + name +func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { switch { case spec.Provider.HTTP != nil: return http.New(cfg), nil case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index f014a2f..09bb7d6 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -27,14 +28,14 @@ func (l *Loader) Name() string { func (l *Loader) Start( ctx context.Context, - targetsourceName string, + targetsourceNN types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, out chan<- []core.DiscoveryMessage, ) error { logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", targetsourceName, + "targetsource", targetsourceNN, ) logger.Info("HTTP loader started") @@ -51,17 +52,17 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"TargetSource": targetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"TargetSource": targetsourceNN.String()}, }, } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index d97b3a6..62b057d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -46,7 +46,7 @@ type TargetSourceReconciler struct { Scheme *runtime.Scheme mu sync.Mutex - running map[client.ObjectKey]runningSource + running map[types.NamespacedName]runningSource BufferSize int ChunkSize int @@ -96,7 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } // getTargetSource retrieves a TargetSource by name, handling cleanup if not found -func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) { +func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { // If the TargetSource no longer exists, ensure runtime cleanup @@ -109,9 +109,9 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client } // handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer -func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { +func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) + logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) r.stopDiscovery(key) @@ -141,7 +141,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour } // isPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { +func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool { r.mu.Lock() defer r.mu.Unlock() @@ -156,8 +156,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } loader, err := discovery.NewLoader( - targetSource.ObjectMeta.Name, - targetSource.ObjectMeta.Namespace, + key, targetSource.Spec, cfg, ) @@ -174,7 +173,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Start loader - go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) + go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel) // Start target applier manager := discovery.NewTargetApplier( @@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.running = make(map[client.ObjectKey]runningSource) + r.running = make(map[types.NamespacedName]runningSource) return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). From 2973c03a665beeb3b53ef7ff71d55921c21053e1 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 09:05:21 +0000 Subject: [PATCH 073/246] improve context cancling and error handling --- .../controller/discovery/target_applier.go | 102 ++++++++++++------ 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 3babebf..7fed5c9 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -37,49 +37,83 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues("targetSource", m.targetSource) - logger.Info("target applier started") - for { + queue := make([]core.DiscoveryMessage, 0, 265) + + for ctx.Err() == nil { select { + case batch, ok := <-m.in: + if !ok { + // Channel closed, pipeline is shutting down + logger.Info("input channel closed, stopping target applier") + return nil + } + queue = append(queue, batch...) + case <-ctx.Done(): - logger.Info("target applier stopped") + logger.Info("context canceled, stopping target applier") return nil + } - case messages := <-m.in: - for _, message := range messages { - // Type assert to determine if this is a snapshot or event - switch msg := message.(type) { - case core.DiscoverySnapshot: - // Collect snapshot chunks - logger.Info( - "received snapshot chunk", - "snapshotID", msg.SnapshotID, - "targetCount", len(msg.Targets), - ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) - if msg.IsLastChunk { - m.processSnapshot(msg.SnapshotID, logger) - } - - case core.DiscoveryEvent: - // Process individual event-driven update - logger.Info( - "received discovery event", - "target", msg.Target.Name, - ) - switch msg.Event { - case core.CREATE: - logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.UPDATE: - logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.DELETE: - logger.Info("Would delete target", "name", msg.Target.Name) - } - } + for len(queue) > 0 { + if ctx.Err() != nil { + break } + + msg := queue[0] + queue = queue[1:] + + if err := m.handleMessage(ctx, msg, logger); err != nil { + // Returning error lets the supervisor (controller) + // tear down and restart the pipeline via reconciliation + // Q: when to return an error vs just log and continue? + return err + } + } } + + logger.Info("target applier stopped") + return nil +} + +func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { + if err := ctx.Err(); err != nil { + return err + } + + // Type assert to determine if this is a snapshot or event + switch msg := message.(type) { + case core.DiscoverySnapshot: + // Collect snapshot chunks + logger.Info( + "received snapshot chunk", + "snapshotID", msg.SnapshotID, + "targetCount", len(msg.Targets), + ) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) + if msg.IsLastChunk { + m.processSnapshot(msg.SnapshotID, logger) + } + + case core.DiscoveryEvent: + // Process individual event-driven update + logger.Info( + "received discovery event", + "target", msg.Target.Name, + ) + switch msg.Event { + case core.CREATE: + logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", msg.Target.Name) + } + } + + return nil } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly From c95bdaf389038386a0b0b98759c98d4c10cb3f31 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 10:01:31 +0000 Subject: [PATCH 074/246] add supervised goroutines --- .../controller/targetsource_controller.go | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 62b057d..9fad373 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "sync" "k8s.io/apimachinery/pkg/runtime" @@ -172,17 +173,45 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } + // goroutines use done channel to report termination (nil or error) back to supervisor + // Buffer size = supervised goroutines = 2 (loader + applier) + done := make(chan error, 2) + // Start loader - go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel) + go runWithRecovery( + runtimeCtx, + "loader", + func(ctx context.Context) error { + return loader.Start(ctx, key, targetSource.Spec, targetChannel) + }, + done, + ) // Start target applier - manager := discovery.NewTargetApplier( + applier := discovery.NewTargetApplier( r.Client, r.Scheme, targetSource, targetChannel, ) - go manager.Run(runtimeCtx) + go runWithRecovery( + runtimeCtx, + "target-applier", + applier.Run, + done, + ) + + // Supervision goroutine to handle pipeline termination + go func() { + err := <-done + logger := log.FromContext(context.Background()).WithValues("targetSource", key) + if err != nil { + logger.Error(err, "Discovery pipeline terminated with error") + } + + // Ensure cleanup on termination + r.stopDiscovery(key) + }() r.mu.Lock() r.running[key] = runningSource{cancel: cancel} @@ -207,6 +236,25 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { } } +// runWithRecovery executes a worker function under panic protection +// and reports termination (nil or error) through done. +func runWithRecovery( + ctx context.Context, + name string, + run func(context.Context) error, + done chan<- error, +) { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic in %s: %v", name, r) + } + }() + + // Normal exit path + err := run(ctx) + done <- err +} + // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { r.running = make(map[types.NamespacedName]runningSource) From 0aa883d98c940ebf374c6b9492522e63a601ac6d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 12:52:08 +0000 Subject: [PATCH 075/246] refactor target applier --- internal/controller/discovery/target_applier.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 7fed5c9..c60f2b8 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -36,10 +36,14 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // and reconciles Target CRs accordingly func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). - WithValues("targetSource", m.targetSource) + WithValues( + "name", m.targetSource.Name, + "namespace", m.targetSource.Namespace, + ) + logger.Info("target applier started") - queue := make([]core.DiscoveryMessage, 0, 265) + queue := []core.DiscoveryMessage{} for ctx.Err() == nil { select { @@ -58,7 +62,7 @@ func (m *TargetApplier) Run(ctx context.Context) error { for len(queue) > 0 { if ctx.Err() != nil { - break + return ctx.Err() } msg := queue[0] From 27b2b1f711a4f60edd2609d8e2822adbfaf07991 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 14:15:34 +0000 Subject: [PATCH 076/246] add supervisor for the discovery pipelines --- internal/controller/discovery/supervisor.go | 123 ++++++++++++++++++ .../controller/targetsource_controller.go | 117 +++++++---------- 2 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 internal/controller/discovery/supervisor.go diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go new file mode 100644 index 0000000..ff19604 --- /dev/null +++ b/internal/controller/discovery/supervisor.go @@ -0,0 +1,123 @@ +package discovery + +import ( + "context" + "sync" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type ComponentExit struct { + Name string + Err error +} + +type RestartPolicy struct { + MaxRestarts int + Backoff time.Duration +} + +type Supervisor struct { + ctx context.Context + cancel context.CancelFunc + policy RestartPolicy + failures int + exits chan ComponentExit + wg sync.WaitGroup + stopped bool + stopMu sync.Mutex +} + +func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor { + ctx, cancel := context.WithCancel(parentCtx) + return &Supervisor{ + ctx: ctx, + cancel: cancel, + policy: policy, + exits: make(chan ComponentExit, 4), + failures: 0, + } +} + +func (s *Supervisor) Context() context.Context { + return s.ctx +} + +func (s *Supervisor) Stop() { + s.stopMu.Lock() + defer s.stopMu.Unlock() + + if s.stopped { + return + } + + s.stopped = true + s.cancel() +} + +func (s *Supervisor) Run( + start func(ctx context.Context, exits chan<- ComponentExit), +) error { + logger := log.FromContext(s.ctx).WithName("discovery-supervisor") + + for { + if s.failures > 0 { + logger.Info("Restarting pipeline", + "attempt", s.failures, + "maxAttempts", s.policy.MaxRestarts, + ) + + runtimeCtx, cancel := context.WithCancel(s.ctx) + s.wg = sync.WaitGroup{} + start(runtimeCtx, s.exits) + exit := <-s.exits // first failure wins + + logger.Error(exit.Err, + "Pipeline component crashed", + "component", exit.Name, + ) + + cancel() + s.wg.Wait() + + s.failures++ + if s.failures >= s.policy.MaxRestarts { + logger.Error(exit.Err, + "Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart", + "restarts", s.failures, + ) + s.Stop() + return exit.Err + } + + select { + case <-time.After(s.policy.Backoff): + // continue to restart + case <-s.ctx.Done(): + // Supervisor context canceled during backoff + return s.ctx.Err() + } + } + } +} + +func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) { + s.wg.Add(1) + + go func() { + defer s.wg.Done() + + err := fn(s.ctx) + if err == nil { + err = context.Canceled // treat normal exit as cancellation + } + + select { + case s.exits <- ComponentExit{Name: name, Err: err}: + // exit reported successfully + case <-s.ctx.Done(): + // Supervisor context canceled before reporting exit + } + }() +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9fad373..5d83db9 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,8 +18,8 @@ package controller import ( "context" - "fmt" "sync" + "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -33,9 +33,14 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/go-logr/logr" ) -const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" +const ( + targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" + pipelineMaxRestarts = 5 + pipelineBackoff = 3 * time.Second +) type runningSource struct { cancel context.CancelFunc @@ -63,8 +68,8 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx).WithValues( - "Name", req.NamespacedName, + logger := log.FromContext(ctx).WithName("targetsource controller").WithValues( + "targetsource", req.NamespacedName, ) targetSource, err := r.getTargetSource(ctx, req.NamespacedName) @@ -88,7 +93,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } // Start discovery pipeline - if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil { + if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } @@ -151,70 +156,63 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo } // startDiscoveryPipeline creates and starts the loader and target manager -func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error { - cfg := core.LoaderConfig{ - ChunkSize: r.ChunkSize, - } - - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - cfg, +func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { + supervisor := discovery.NewSupervisor( + context.Background(), + discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, ) - if err != nil { - return err - } - runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { - cancel() return err } - // goroutines use done channel to report termination (nil or error) back to supervisor - // Buffer size = supervised goroutines = 2 (loader + applier) - done := make(chan error, 2) + start := func(ctx context.Context, exits chan<- discovery.ComponentExit) { + // Create loader instance + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ + ChunkSize: r.ChunkSize, + }, + ) + if err != nil { + return + } + + // Create target applier instance + applier := discovery.NewTargetApplier( + r.Client, + r.Scheme, + targetSource, + targetChannel, + ) - // Start loader - go runWithRecovery( - runtimeCtx, - "loader", - func(ctx context.Context) error { + // Start loader + supervisor.Go("loader", func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - done, - ) + }) + // Start target applier + supervisor.Go("target-applier", applier.Run) - // Start target applier - applier := discovery.NewTargetApplier( - r.Client, - r.Scheme, - targetSource, - targetChannel, - ) - go runWithRecovery( - runtimeCtx, - "target-applier", - applier.Run, - done, - ) + } - // Supervision goroutine to handle pipeline termination go func() { - err := <-done - logger := log.FromContext(context.Background()).WithValues("targetSource", key) + err := supervisor.Run(start) if err != nil { - logger.Error(err, "Discovery pipeline terminated with error") + logger.Error(err, "Discovery pipeline stopped permanently") } - // Ensure cleanup on termination + close(targetChannel) + r.DiscoveryRegistry.Unregister(key) r.stopDiscovery(key) }() r.mu.Lock() - r.running[key] = runningSource{cancel: cancel} + r.running[key] = runningSource{cancel: supervisor.Stop} r.mu.Unlock() return nil @@ -236,25 +234,6 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { } } -// runWithRecovery executes a worker function under panic protection -// and reports termination (nil or error) through done. -func runWithRecovery( - ctx context.Context, - name string, - run func(context.Context) error, - done chan<- error, -) { - defer func() { - if r := recover(); r != nil { - done <- fmt.Errorf("panic in %s: %v", name, r) - } - }() - - // Normal exit path - err := run(ctx) - done <- err -} - // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { r.running = make(map[types.NamespacedName]runningSource) From 22fe2d894e2109c817a11b3153f298ba0fb8eb06 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 14:55:21 +0000 Subject: [PATCH 077/246] improve readability --- internal/controller/targetsource_controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 5d83db9..db60520 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -38,8 +38,9 @@ import ( const ( targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" - pipelineMaxRestarts = 5 - pipelineBackoff = 3 * time.Second + + pipelineMaxRestarts = 5 + pipelineBackoff = 3 * time.Second ) type runningSource struct { From 58538c76c0583e031b56031f72e639450c918910 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 15:02:48 +0000 Subject: [PATCH 078/246] remove side-effects from getter getTargetSource --- internal/controller/targetsource_controller.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index db60520..33342b4 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -75,6 +75,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.getTargetSource(ctx, req.NamespacedName) if err != nil { + // If the TargetSource no longer exists, ensure runtime cleanup + if client.IgnoreNotFound(err) == nil { + logger.Info("TargetSource not found, ensuring cleanup") + r.stopDiscovery(req.NamespacedName) + return ctrl.Result{}, nil + } return ctrl.Result{}, err } @@ -106,11 +112,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { - // If the TargetSource no longer exists, ensure runtime cleanup - if client.IgnoreNotFound(err) == nil { - r.stopDiscovery(key) - } - return nil, client.IgnoreNotFound(err) + return nil, err } return &targetSource, nil } From 4f0457ec86f4ed5df64a4216aadc8e3fc3551391 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 17:38:41 +0000 Subject: [PATCH 079/246] redesign supervisor --- internal/controller/discovery/supervisor.go | 145 ++++++++---------- .../controller/targetsource_controller.go | 85 +++++----- 2 files changed, 106 insertions(+), 124 deletions(-) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index ff19604..c716965 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -8,116 +8,93 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -type ComponentExit struct { - Name string - Err error -} - type RestartPolicy struct { MaxRestarts int Backoff time.Duration } +type Component struct { + Name string + Run func(ctx context.Context) error + Policy RestartPolicy +} + type Supervisor struct { - ctx context.Context - cancel context.CancelFunc - policy RestartPolicy - failures int - exits chan ComponentExit - wg sync.WaitGroup - stopped bool - stopMu sync.Mutex + ctx context.Context + cancel context.CancelFunc + + stopped bool + mu sync.Mutex + + components []Component } -func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor { - ctx, cancel := context.WithCancel(parentCtx) +func NewSupervisor(parent context.Context) *Supervisor { + ctx, cancel := context.WithCancel(parent) return &Supervisor{ - ctx: ctx, - cancel: cancel, - policy: policy, - exits: make(chan ComponentExit, 4), - failures: 0, + ctx: ctx, + cancel: cancel, } } -func (s *Supervisor) Context() context.Context { - return s.ctx +func (s *Supervisor) AddComponent(c Component) { + s.components = append(s.components, c) } -func (s *Supervisor) Stop() { - s.stopMu.Lock() - defer s.stopMu.Unlock() +func (s *Supervisor) runComponent(c Component) { + logger := log.FromContext(s.ctx).WithValues( + "component", c.Name, + ) - if s.stopped { - return - } - - s.stopped = true - s.cancel() -} - -func (s *Supervisor) Run( - start func(ctx context.Context, exits chan<- ComponentExit), -) error { - logger := log.FromContext(s.ctx).WithName("discovery-supervisor") + failures := 0 for { - if s.failures > 0 { - logger.Info("Restarting pipeline", - "attempt", s.failures, - "maxAttempts", s.policy.MaxRestarts, - ) + err := c.Run(s.ctx) + if s.ctx.Err() != nil { + return + } - runtimeCtx, cancel := context.WithCancel(s.ctx) - s.wg = sync.WaitGroup{} - start(runtimeCtx, s.exits) - exit := <-s.exits // first failure wins + failures++ + logger.Error(err, + "Component failed", + "attempt", failures, + ) - logger.Error(exit.Err, - "Pipeline component crashed", - "component", exit.Name, + if failures >= c.Policy.MaxRestarts { + logger.Error(err, + "Component exceeded restart limit; stopping discovery pipeline", + "restarts", failures, ) + s.Stop() + return + } - cancel() - s.wg.Wait() - - s.failures++ - if s.failures >= s.policy.MaxRestarts { - logger.Error(exit.Err, - "Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart", - "restarts", s.failures, - ) - s.Stop() - return exit.Err - } - - select { - case <-time.After(s.policy.Backoff): - // continue to restart - case <-s.ctx.Done(): - // Supervisor context canceled during backoff - return s.ctx.Err() - } + select { + case <-time.After(c.Policy.Backoff): + case <-s.ctx.Done(): + return } } } -func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) { - s.wg.Add(1) +func (s *Supervisor) Run() { + for _, c := range s.components { + component := c + go s.runComponent(component) + } +} - go func() { - defer s.wg.Done() +func (s *Supervisor) Stop() { + s.mu.Lock() + defer s.mu.Unlock() - err := fn(s.ctx) - if err == nil { - err = context.Canceled // treat normal exit as cancellation - } + if s.stopped { + return + } + s.stopped = true + s.cancel() +} - select { - case s.exits <- ComponentExit{Name: name, Err: err}: - // exit reported successfully - case <-s.ctx.Done(): - // Supervisor context canceled before reporting exit - } - }() +func (s *Supervisor) Done() <-chan struct{} { + return s.ctx.Done() } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 33342b4..68f47eb 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -160,54 +160,57 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo // startDiscoveryPipeline creates and starts the loader and target manager func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { - supervisor := discovery.NewSupervisor( - context.Background(), - discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, - ) + supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { return err } - start := func(ctx context.Context, exits chan<- discovery.ComponentExit) { - // Create loader instance - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - core.LoaderConfig{ - ChunkSize: r.ChunkSize, - }, - ) - if err != nil { - return - } + // Create loader instance + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ChunkSize: r.ChunkSize}, + ) + if err != nil { + return err + } - // Create target applier instance - applier := discovery.NewTargetApplier( - r.Client, - r.Scheme, - targetSource, - targetChannel, - ) + // Create target applier instance + applier := discovery.NewTargetApplier( + r.Client, + r.Scheme, + targetSource, + targetChannel, + ) - // Start loader - supervisor.Go("loader", func(ctx context.Context) error { + supervisor.AddComponent(discovery.Component{ + Name: "loader", + Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }) - // Start target applier - supervisor.Go("target-applier", applier.Run) + }, + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + }) - } + supervisor.AddComponent(discovery.Component{ + Name: "target-applier", + Run: applier.Run, + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + }) + + supervisor.Run() go func() { - err := supervisor.Run(start) - if err != nil { - logger.Error(err, "Discovery pipeline stopped permanently") - } + <-supervisor.Done() + + logger.Info("Pipeline stopped; performing final cleanup") close(targetChannel) r.DiscoveryRegistry.Unregister(key) @@ -215,25 +218,27 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName }() r.mu.Lock() - r.running[key] = runningSource{cancel: supervisor.Stop} + r.running[key] = runningSource{ + cancel: func() { + supervisor.Stop() + }, + } r.mu.Unlock() return nil } // stopDiscovery stops and removes a running discovery pipeline -// for the given TargetSource key func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { r.mu.Lock() running, ok := r.running[key] if ok { - running.cancel() delete(r.running, key) } r.mu.Unlock() if ok { - r.DiscoveryRegistry.Unregister(key) + running.cancel() } } From 2b728c4ff0287a1a9dd7137eee1b1c54f9d72dca Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 15:15:56 -0600 Subject: [PATCH 080/246] added const file for common labels --- internal/controller/discovery/core/const.go | 6 ++++++ internal/controller/discovery/target_manager.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 internal/controller/discovery/core/const.go diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/core/const.go new file mode 100644 index 0000000..82a5962 --- /dev/null +++ b/internal/controller/discovery/core/const.go @@ -0,0 +1,6 @@ +package core + +const ( + // Labels + LabelTargetSourceName = "operator.gnmic.dev/targetsource" +) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index c9ac079..a9cd463 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -175,7 +175,7 @@ func (m *TargetManager) applyTarget(ctx context.Context, name string, address st _, err := controllerutil.CreateOrUpdate(ctx, m.client, target, func() error { labels := map[string]string{ - "gnmic.io/source": m.targetSource.Name, + core.LabelTargetSourceName: m.targetSource.Name, } maps.Copy(labels, m.targetSource.Spec.TargetLabels) From 5abbd63ed664e58047785b445ae2b64c54ea7a9f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 15:17:09 -0600 Subject: [PATCH 081/246] simplified name and namespace calls for objects --- internal/controller/discovery/target_manager.go | 4 ++-- internal/controller/targetsource_controller.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index a9cd463..be6b1fe 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -42,8 +42,8 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ func (m *TargetManager) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues( - "targetSource", m.targetSource.ObjectMeta.Name, - "namespace", m.targetSource.ObjectMeta.Namespace, + "targetSource", m.targetSource.Name, + "namespace", m.targetSource.Namespace, ) logger.Info("target manager started") diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index da80030..ad5fdee 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -161,8 +161,8 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { // startDiscoveryPipeline creates and starts the loader and target manager func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { loader, err := discovery.NewLoader( - targetSource.ObjectMeta.Name, - targetSource.ObjectMeta.Namespace, + targetSource.Name, + targetSource.Namespace, targetSource.Spec, ) if err != nil { From 3d7ff3851710baed90bd66e00b266a919cf6f328 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 16:32:57 -0600 Subject: [PATCH 082/246] changed label handling and target object creation --- internal/controller/discovery/core/const.go | 9 ++++- .../discovery/loaders/http_pull/loader.go | 10 ++--- internal/controller/discovery/mapper.go | 39 ++++++++++++++++++- .../controller/discovery/target_manager.go | 34 +++++++--------- 4 files changed, 64 insertions(+), 28 deletions(-) diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/core/const.go index 82a5962..9548d94 100644 --- a/internal/controller/discovery/core/const.go +++ b/internal/controller/discovery/core/const.go @@ -1,6 +1,13 @@ package core const ( - // Labels + // Kubernetes Side Labels LabelTargetSourceName = "operator.gnmic.dev/targetsource" ) + +const ( + // Prefix and Labels for external systems + ExternalLabelPrefix = "gnmic_operator_" + + ExternalLabelTargetProfile = ExternalLabelPrefix + "target_profile" +) diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 2d8a9e9..e4325d0 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -62,12 +62,12 @@ func (l *Loader) Start( { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"gnmic_operator_target_profile": "default1"}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"gnmic_operator_target_profile": "default1"}, }, } @@ -81,12 +81,12 @@ func (l *Loader) Start( { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"gnmic_operator_target_profile": "default1"}, }, { Name: "leaf2", Address: "clab-3-nodes-leaf2:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"gnmic_operator_target_profile": "default1"}, }, } @@ -100,7 +100,7 @@ func (l *Loader) Start( { Name: "ceos1", Address: "clab-3-nodes-ceos2:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"gnmic_operator_target_profile": "default2"}, }, } diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index bee897a..765e5aa 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -4,11 +4,48 @@ package discovery // file decides which targets to create/update/delete import ( + "maps" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" ) -func GenerateEvents(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { +func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSource) *gnmicv1alpha1.Target { + t := &gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: d.Name, + Namespace: ts.Namespace, + Labels: make(map[string]string), + }, + } + + t.Spec.Address = d.Address + t.Spec.Profile = ts.Spec.TargetProfile + + maps.Copy(t.Labels, ts.Spec.TargetLabels) + + for k, v := range d.Labels { + if strings.HasPrefix(k, core.ExternalLabelPrefix) { + switch k { + case core.ExternalLabelTargetProfile: + t.Spec.Profile = v + default: + // handle unknown label + } + } else { + t.Labels[k] = v + } + } + + t.Labels[core.LabelTargetSourceName] = ts.Name + + return t +} + +func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { var events []core.DiscoveryEvent discoveredMap := make(map[string]core.DiscoveredTarget) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index be6b1fe..22d8012 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -2,7 +2,6 @@ package discovery import ( "context" - "maps" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -114,7 +113,7 @@ func (m *TargetManager) processSnapshot(ctx context.Context, snapshotID string, ) } - events := GenerateEvents(existing, targets) + events := generateEvents(existing, targets) nApply := 0 nDelete := 0 @@ -153,7 +152,9 @@ func (m *TargetManager) processEvent(ctx context.Context, event core.DiscoveryEv ) } case core.APPLY: - if err := m.applyTarget(ctx, event.Target.Name, event.Target.Address); err != nil { + target := generateTargetResource(event.Target, m.targetSource) + + if err := m.applyTarget(ctx, target); err != nil { logger.Error(err, "error applying target", "targetName", event.Target.Name, ) @@ -165,29 +166,19 @@ func (m *TargetManager) processEvent(ctx context.Context, event core.DiscoveryEv } } -func (m *TargetManager) applyTarget(ctx context.Context, name string, address string) error { - target := &gnmicv1alpha1.Target{ +func (m *TargetManager) applyTarget(ctx context.Context, desired *gnmicv1alpha1.Target) error { + existing := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: m.targetSource.Namespace, + Name: desired.Name, + Namespace: desired.Namespace, }, } - _, err := controllerutil.CreateOrUpdate(ctx, m.client, target, func() error { - labels := map[string]string{ - core.LabelTargetSourceName: m.targetSource.Name, - } - - maps.Copy(labels, m.targetSource.Spec.TargetLabels) - - target.Labels = labels + _, err := controllerutil.CreateOrUpdate(ctx, m.client, existing, func() error { + existing.Spec = desired.Spec + existing.Labels = desired.Labels - target.Spec = gnmicv1alpha1.TargetSpec{ - Address: address, - Profile: m.targetSource.Spec.TargetProfile, - } - - return controllerutil.SetControllerReference(m.targetSource, target, m.scheme) + return controllerutil.SetControllerReference(m.targetSource, existing, m.scheme) }) return err @@ -195,6 +186,7 @@ func (m *TargetManager) applyTarget(ctx context.Context, name string, address st func (m *TargetManager) deleteTarget(ctx context.Context, name string) error { existing := &gnmicv1alpha1.Target{} + err := m.client.Get(ctx, types.NamespacedName{ Name: name, Namespace: m.targetSource.Namespace, From 4d0a93740db7f3fc09e793175e20478710280e72 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 16:43:34 -0600 Subject: [PATCH 083/246] fixed label filtering for existing targets --- internal/controller/discovery/client.go | 3 ++- internal/controller/discovery/loaders/http_pull/loader.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index 3bc7ef7..d23c043 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" ) func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { @@ -16,7 +17,7 @@ func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1 err := c.List(ctx, &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ - "gnmic.io/source": ts.Name, + core.LabelTargetSourceName: ts.Name, }, ) if err != nil { diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index e4325d0..24ae094 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -62,7 +62,7 @@ func (l *Loader) Start( { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"gnmic_operator_target_profile": "default1"}, + Labels: map[string]string{}, }, { Name: "leaf1", From 60491be6b980c081f46955c59a8dc995db26c2e0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 09:06:20 +0000 Subject: [PATCH 084/246] add dependency handling of discovery pipeline components --- api/v1alpha1/targetsource_types.go | 7 ++ api/v1alpha1/zz_generated.deepcopy.go | 21 ++++ .../operator.gnmic.dev_targetsources.yaml | 5 + internal/controller/discovery/supervisor.go | 112 +++++++++--------- .../controller/targetsource_controller.go | 67 +++++++---- 5 files changed, 133 insertions(+), 79 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index feea000..a936e66 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -24,6 +24,8 @@ import ( // +kubebuilder:validation:Required type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` + // +kubebuilder:validation:Optional + Webhook WebhookSpec `json:"webhook,omitempty"` // TargetLabels map[string]string `json:"targetLabels,omitempty"` @@ -37,6 +39,11 @@ type ProviderSpec struct { Consul *ConsulConfig `json:"consul,omitempty"` } +type WebhookSpec struct { + // +kubebuilder:validation:Optional + Enabled *bool `json:"enabled,omitempty"` +} + type HTTPConfig struct { // +kubebuilder:validation:MinLength=1 URL string `json:"url"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 61e81fd..608d47e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1292,6 +1292,7 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { *out = new(ProviderSpec) (*in).DeepCopyInto(*out) } + in.Webhook.DeepCopyInto(&out.Webhook) if in.TargetLabels != nil { in, out := &in.TargetLabels, &out.TargetLabels *out = make(map[string]string, len(*in)) @@ -1477,3 +1478,23 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. +func (in *WebhookSpec) DeepCopy() *WebhookSpec { + if in == nil { + return nil + } + out := new(WebhookSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index f373822..b385d8e 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -67,6 +67,11 @@ spec: targetProfile: minLength: 1 type: string + webhook: + properties: + enabled: + type: boolean + type: object required: - provider - targetProfile diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index c716965..128305a 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -14,19 +14,20 @@ type RestartPolicy struct { } type Component struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy + Name string + Run func(ctx context.Context) error + Policy RestartPolicy + DegradeOnFailure bool } type Supervisor struct { ctx context.Context cancel context.CancelFunc - stopped bool - mu sync.Mutex + wg sync.WaitGroup - components []Component + mu sync.Mutex + stopped bool } func NewSupervisor(parent context.Context) *Supervisor { @@ -37,53 +38,6 @@ func NewSupervisor(parent context.Context) *Supervisor { } } -func (s *Supervisor) AddComponent(c Component) { - s.components = append(s.components, c) -} - -func (s *Supervisor) runComponent(c Component) { - logger := log.FromContext(s.ctx).WithValues( - "component", c.Name, - ) - - failures := 0 - - for { - err := c.Run(s.ctx) - if s.ctx.Err() != nil { - return - } - - failures++ - logger.Error(err, - "Component failed", - "attempt", failures, - ) - - if failures >= c.Policy.MaxRestarts { - logger.Error(err, - "Component exceeded restart limit; stopping discovery pipeline", - "restarts", failures, - ) - s.Stop() - return - } - - select { - case <-time.After(c.Policy.Backoff): - case <-s.ctx.Done(): - return - } - } -} - -func (s *Supervisor) Run() { - for _, c := range s.components { - component := c - go s.runComponent(component) - } -} - func (s *Supervisor) Stop() { s.mu.Lock() defer s.mu.Unlock() @@ -98,3 +52,55 @@ func (s *Supervisor) Stop() { func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } + +func (s *Supervisor) Wait() { + s.wg.Wait() +} + +func (s *Supervisor) RunComponent(component Component) { + s.wg.Add(1) + + go func() { + defer s.wg.Done() + + logger := log.FromContext(s.ctx).WithValues("component", component.Name) + failures := 0 + + for { + logger.Info("starting component") + err := component.Run(s.ctx) + + if s.ctx.Err() != nil { + logger.Info("component stopped due to pipeline shutdown") + return + } + + failures++ + logger.Error(err, + "component failed to run", + "attempt", failures, + "max", component.Policy.MaxRestarts, + ) + + if failures >= component.Policy.MaxRestarts { + if component.DegradeOnFailure { + logger.Error(err, + "component permanently failed; shutting down pipeline", + ) + s.Stop() + } else { + logger.Info( + "optional component permanently failed; continuing without it", + ) + } + return + } + + select { + case <-time.After(component.Policy.Backoff): + case <-s.ctx.Done(): + return + } + } + }() +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 68f47eb..a687e80 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -167,16 +167,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create loader instance - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - core.LoaderConfig{ChunkSize: r.ChunkSize}, - ) - if err != nil { - return err - } - // Create target applier instance applier := discovery.NewTargetApplier( r.Client, @@ -184,34 +174,59 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetSource, targetChannel, ) - - supervisor.AddComponent(discovery.Component{ - Name: "loader", - Run: func(ctx context.Context) error { - return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, - }) - - supervisor.AddComponent(discovery.Component{ + // Start target applier + applierReady := make(chan struct{}) + supervisor.RunComponent(discovery.Component{ Name: "target-applier", - Run: applier.Run, Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, + DegradeOnFailure: true, + Run: func(ctx context.Context) error { + close(applierReady) + return applier.Run(ctx) + }, }) + // Wait for applier to be ready before starting loader + select { + case <-applierReady: + case <-supervisor.Done(): + return nil + } - supervisor.Run() + // Create loader instance + loaderConfigured := targetSource.Spec.Provider != nil + webhookConfigured := targetSource.Spec.Webhook.Enabled != nil + if loaderConfigured { + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ChunkSize: r.ChunkSize}, + ) + if err != nil { + supervisor.Stop() + return err + } + + supervisor.RunComponent(discovery.Component{ + Name: "loader", + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + DegradeOnFailure: !webhookConfigured, + Run: func(ctx context.Context) error { + return loader.Start(ctx, key, targetSource.Spec, targetChannel) + }, + }) + } go func() { <-supervisor.Done() + supervisor.Wait() // Wait for components to exit logger.Info("Pipeline stopped; performing final cleanup") - close(targetChannel) r.DiscoveryRegistry.Unregister(key) r.stopDiscovery(key) From b8a6d272d479a97f05b5adeb6f9081520a236f8e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 09:31:29 +0000 Subject: [PATCH 085/246] refactor code --- internal/controller/discovery/supervisor.go | 54 +++++----- .../controller/targetsource_controller.go | 99 ++++++++++--------- 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index 128305a..710381e 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -8,18 +8,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -type RestartPolicy struct { - MaxRestarts int - Backoff time.Duration -} - -type Component struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy - DegradeOnFailure bool -} - +// Supervisor coordinates the runtime lifecycle of pipeline components +// +// Guarantees: +// - Each component is restarted independently +// - Permanent failure escalates according to policy +// - Stop() cancels all components +// - Wait() blocks until all goroutines exit type Supervisor struct { ctx context.Context cancel context.CancelFunc @@ -30,14 +25,30 @@ type Supervisor struct { stopped bool } -func NewSupervisor(parent context.Context) *Supervisor { - ctx, cancel := context.WithCancel(parent) +// RestartPolicy defines the restart behavior for a component +type RestartPolicy struct { + MaxRestarts int + Backoff time.Duration +} + +type ComponentSpec struct { + Name string + Run func(ctx context.Context) error + Policy RestartPolicy + // EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline + EscalatesOnFailure bool +} + +// NewSupervisor creates a new Supervisor with a cancellable context +func NewSupervisor(parentCtx context.Context) *Supervisor { + ctx, cancel := context.WithCancel(parentCtx) return &Supervisor{ ctx: ctx, cancel: cancel, } } +// Stop signals all supervised components to stop by canceling the context func (s *Supervisor) Stop() { s.mu.Lock() defer s.mu.Unlock() @@ -49,15 +60,14 @@ func (s *Supervisor) Stop() { s.cancel() } -func (s *Supervisor) Done() <-chan struct{} { - return s.ctx.Done() -} +// Done returns a channel that is closed when the pipeline is stopped +func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } -func (s *Supervisor) Wait() { - s.wg.Wait() -} +// Wait blocks until all supervised components have exited +func (s *Supervisor) Wait() { s.wg.Wait() } -func (s *Supervisor) RunComponent(component Component) { +// StartSupervisedComponent starts and supervises a component +func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { s.wg.Add(1) go func() { @@ -83,7 +93,7 @@ func (s *Supervisor) RunComponent(component Component) { ) if failures >= component.Policy.MaxRestarts { - if component.DegradeOnFailure { + if component.EscalatesOnFailure { logger.Error(err, "component permanently failed; shutting down pipeline", ) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index a687e80..f04eced 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -43,17 +43,26 @@ const ( pipelineBackoff = 3 * time.Second ) -type runningSource struct { +// pipelineHandle represents a controller-owned handle to a running pipeline +// The controller never manipulates internals; it only invokes cancel() +type pipelineHandle struct { cancel context.CancelFunc } // TargetSourceReconciler reconciles a TargetSource object +// +// Responsibilities: +// - Ensure at most one pipeline per TargetSource +// - Start pipelines on reconcile +// - Stop pipelines on deletion or NotFound +// - Delegate runtime failure handling to the Supervisor type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme - mu sync.Mutex - running map[types.NamespacedName]runningSource + mu sync.Mutex + // runningPipelines tracks currently active pipelines by NamespacedName + runningPipelines map[types.NamespacedName]pipelineHandle BufferSize int ChunkSize int @@ -69,47 +78,43 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx).WithName("targetsource controller").WithValues( - "targetsource", req.NamespacedName, - ) + logger := log.FromContext(ctx). + WithName("targetsource controller"). + WithValues("targetsource", req.NamespacedName) - targetSource, err := r.getTargetSource(ctx, req.NamespacedName) + targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) if err != nil { // If the TargetSource no longer exists, ensure runtime cleanup if client.IgnoreNotFound(err) == nil { - logger.Info("TargetSource not found, ensuring cleanup") - r.stopDiscovery(req.NamespacedName) + logger.Info("TargetSource not found; stopping discovery pipeline") + r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } return ctrl.Result{}, err } - // Handle deletion with finalizer if !targetSource.DeletionTimestamp.IsZero() { - return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource) + return r.reconcileDeletion(ctx, req.NamespacedName, targetSource) } - // Ensure finalizer is set if err := r.ensureFinalizer(ctx, targetSource); err != nil { return ctrl.Result{}, err } - // Check if pipeline is already running - if r.isPipelineRunning(req.NamespacedName) { + if r.hasPipelineRunning(req.NamespacedName) { return ctrl.Result{}, nil } - // Start discovery pipeline if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } - logger.Info("TargetSource pipeline started") + logger.Info("Discover pipeline started") return ctrl.Result{}, nil } -// getTargetSource retrieves a TargetSource by name, handling cleanup if not found -func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { +// fetchTargetSource retrieves a TargetSource by name, handling cleanup if not found +func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { return nil, err @@ -117,12 +122,20 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types. return &targetSource, nil } -// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer -func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { +// hasPipelineRunning checks if a discovery pipeline is already running for the given key +func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, exists := r.runningPipelines[key] + return exists +} + +// reconcileDeletion stops the discovery pipeline and removes the finalizer +func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) - r.stopDiscovery(key) + r.stopDiscoveryPipeline(key) // Remove finalizer if exists if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { @@ -149,16 +162,13 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } -// isPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool { - r.mu.Lock() - defer r.mu.Unlock() - - _, exists := r.running[key] - return exists -} - -// startDiscoveryPipeline creates and starts the loader and target manager +// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource +// +// Pipeline semantics: +// 1. target-applier is mandatory and must start first +// 2. loader is optional and conditional on spec +// 3. Permanent failure of required components shuts down the pipeline +// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { supervisor := discovery.NewSupervisor(context.Background()) @@ -176,15 +186,15 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target applier applierReady := make(chan struct{}) - supervisor.RunComponent(discovery.Component{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "target-applier", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - DegradeOnFailure: true, + EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(applierReady) + close(applierReady) // Signals that applier started successfully return applier.Run(ctx) }, }) @@ -209,31 +219,32 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.RunComponent(discovery.Component{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "loader", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - DegradeOnFailure: !webhookConfigured, + EscalatesOnFailure: !webhookConfigured, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) }, }) } + // Monitor supervisor in a separate goroutine to handle shutdown and cleanup go func() { <-supervisor.Done() supervisor.Wait() // Wait for components to exit - logger.Info("Pipeline stopped; performing final cleanup") + logger.Info("Pipeline stopped; cleaning up") close(targetChannel) r.DiscoveryRegistry.Unregister(key) - r.stopDiscovery(key) + r.stopDiscoveryPipeline(key) }() r.mu.Lock() - r.running[key] = runningSource{ + r.runningPipelines[key] = pipelineHandle{ cancel: func() { supervisor.Stop() }, @@ -243,12 +254,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return nil } -// stopDiscovery stops and removes a running discovery pipeline -func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { +// stopDiscoveryPipeline stops and removes a running discovery pipeline +func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) { r.mu.Lock() - running, ok := r.running[key] + running, ok := r.runningPipelines[key] if ok { - delete(r.running, key) + delete(r.runningPipelines, key) } r.mu.Unlock() @@ -259,7 +270,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.running = make(map[types.NamespacedName]runningSource) + r.runningPipelines = make(map[types.NamespacedName]pipelineHandle) return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). From eedfedf930d6f78ef9ca430115bb121ee9db129c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 12:55:52 +0000 Subject: [PATCH 086/246] improve context handling of and target applier semantics --- internal/controller/discovery/core/helpers.go | 14 +- internal/controller/discovery/core/types.go | 5 +- .../controller/discovery/target_applier.go | 209 ++++++++++++++---- 3 files changed, 184 insertions(+), 44 deletions(-) diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/helpers.go index 843f30e..f24b50c 100644 --- a/internal/controller/discovery/core/helpers.go +++ b/internal/controller/discovery/core/helpers.go @@ -2,6 +2,7 @@ package core import ( "context" + "fmt" ) // sendMessages sends discovery messages over a channel in a context-aware manner @@ -32,13 +33,15 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { var snapshots []DiscoverySnapshot totalTargets := len(targets) + totalChunks := (totalTargets + chunkSize - 1) / chunkSize _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] snapshots = append(snapshots, DiscoverySnapshot{ Targets: chunk, SnapshotID: snapshotID, - IsLastChunk: (end == totalTargets), + ChunkIndex: i / chunkSize, + TotalChunks: totalChunks, }) return nil }) @@ -48,8 +51,11 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu // SendSnapshot sends discovered targets as a snapshot over a channel in chunks func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { - snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) + if len(targets) == 0 { + return fmt.Errorf("no targets in Snapshot") + } + snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { // Convert DiscoverySnapshot to DiscoveryMessage messages := make([]DiscoveryMessage, 1) @@ -73,6 +79,10 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { + if len(events) == 0 { + return fmt.Errorf("no events to process") + } + messages := eventsToMessages(events) total := len(messages) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 61209fd..3f6957a 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -26,7 +26,8 @@ type DiscoveryEvent struct { } type DiscoverySnapshot struct { - Targets []DiscoveredTarget SnapshotID string - IsLastChunk bool + ChunkIndex int + TotalChunks int + Targets []DiscoveredTarget } diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index c60f2b8..ee127c5 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -2,6 +2,7 @@ package discovery import ( "context" + "fmt" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -12,13 +13,23 @@ import ( "github.com/go-logr/logr" ) +type snapshotBuffer struct { + snapshotID string + totalChunks int + received map[int][]core.DiscoveredTarget + complete bool +} + // TargetApplier consumes discovered targets and applies them to Kubernetes type TargetApplier struct { - client client.Client - scheme *runtime.Scheme - targetSource *gnmicv1alpha1.TargetSource - in <-chan []core.DiscoveryMessage - collected map[string][]core.DiscoveredTarget + client client.Client + scheme *runtime.Scheme + targetSource *gnmicv1alpha1.TargetSource + in <-chan []core.DiscoveryMessage + queue []core.DiscoveryMessage + activeSnapshot *snapshotBuffer + // Events are deferred while snapshot is in progress + defferedEvents []core.DiscoveryEvent } // NewTargetApplier wires a TargetApplier instance @@ -28,47 +39,43 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ scheme: s, targetSource: ts, in: in, - collected: make(map[string][]core.DiscoveredTarget), } } // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *TargetApplier) Run(ctx context.Context) error { +func (a *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues( - "name", m.targetSource.Name, - "namespace", m.targetSource.Namespace, + "name", a.targetSource.Name, + "namespace", a.targetSource.Namespace, ) - logger.Info("target applier started") - queue := []core.DiscoveryMessage{} - for ctx.Err() == nil { select { - case batch, ok := <-m.in: + case batch, ok := <-a.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target applier") return nil } - queue = append(queue, batch...) + a.queue = append(a.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target applier") return nil } - for len(queue) > 0 { + for len(a.queue) > 0 { if ctx.Err() != nil { - return ctx.Err() + return nil // why return nil? } - msg := queue[0] - queue = queue[1:] + msg := a.queue[0] + a.queue = a.queue[1:] - if err := m.handleMessage(ctx, msg, logger); err != nil { + if err := a.processMessage(ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -82,7 +89,7 @@ func (m *TargetApplier) Run(ctx context.Context) error { return nil } -func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -94,12 +101,10 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover logger.Info( "received snapshot chunk", "snapshotID", msg.SnapshotID, + "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) - if msg.IsLastChunk { - m.processSnapshot(msg.SnapshotID, logger) - } + return a.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -107,31 +112,155 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover "received discovery event", "target", msg.Target.Name, ) - switch msg.Event { - case core.CREATE: - logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.UPDATE: - logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.DELETE: - logger.Info("Would delete target", "name", msg.Target.Name) + return a.processEvent(ctx, msg, logger) + + default: + return fmt.Errorf("unknonw discovery message type %T", msg) + } +} + +// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly +func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if a.activeSnapshot == nil { + a.startNewSnapshot(chunk, logger) + return nil + } + + snapshot := a.activeSnapshot + // Check if a new snapshot arrived + if snapshot.snapshotID != chunk.SnapshotID { + // If current snapshot is complete apply it first + if snapshot.complete { + if err := a.applySnapshot(ctx, snapshot, logger); err != nil { + return err + } + } else { + // If a new snapshot is started before the old one completed + // the old one can be discarded + logger.Info( + "discarding incomplete snapshot", + "snapshotID", snapshot.snapshotID, + ) } + + // Start collecting the new snapshot + a.startNewSnapshot(chunk, logger) + return nil + } + + return a.collectSnapshot(chunk, logger) +} + +func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + a.activeSnapshot = &snapshotBuffer{ + snapshotID: chunk.SnapshotID, + totalChunks: chunk.TotalChunks, + received: make(map[int][]core.DiscoveredTarget), + complete: false, + } + // Delete buffered events that will be current with new snapshot + a.defferedEvents = nil + + a.collectSnapshot(chunk, logger) +} + +func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := a.activeSnapshot + + if chunk.TotalChunks != snapshot.totalChunks { + logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) + } + if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { + logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) + a.activeSnapshot = nil + return nil + } + if _, exists := snapshot.received[chunk.ChunkIndex]; exists { + logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) + a.activeSnapshot = nil + return nil + } + + snapshot.received[chunk.ChunkIndex] = chunk.Targets + + if len(snapshot.received) == snapshot.totalChunks { + snapshot.complete = true } return nil } -// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) { - targets := m.collected[snapshotID] - delete(m.collected, snapshotID) +func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { + select { + case <-ctx.Done(): + a.activeSnapshot = nil + return nil + default: + } - logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets)) + var allTargets []core.DiscoveredTarget + for i := 0; i < snapshot.totalChunks; i++ { + select { + case <-ctx.Done(): + a.activeSnapshot = nil + return nil + default: + } - if m.targetSource.Spec.Provider.HTTP != nil { - logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name) + chunk, ok := snapshot.received[i] + if !ok { + logger.Error(nil, "missing snapshot chunk", "index", i) + a.activeSnapshot = nil + return nil + } + allTargets = append(allTargets, chunk...) } - for _, target := range targets { - logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels) + logger.Info( + "applying snapshot", + "snapshotID", snapshot.snapshotID, + "targetCount", len(allTargets), + ) + + // apply all targets + // a.applyTargets + + // Replay deffered events + for _, event := range a.defferedEvents { + select { + case <-ctx.Done(): + return nil + default: + } + if err := a.applyEvent(ctx, event, logger); err != nil { + return err + } } + + a.activeSnapshot = nil + a.defferedEvents = nil + return nil +} + +func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { + // If snapshot collecting is active defer events + if a.activeSnapshot != nil { + a.defferedEvents = append(a.defferedEvents, event) + return nil + } + + // Apply events + return a.applyEvent(ctx, event, logger) +} + +func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { + switch event.Event { + case core.CREATE: + logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", event.Target.Name) + } + return nil } From ff4f2bbfe7ffe9f7c3a4715268e692964b9e1ad8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 09:44:15 +0000 Subject: [PATCH 087/246] fix import --- internal/apiserver/apiserver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 7f125cf..0e65872 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,7 +9,6 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/gin-gonic/gin/binding" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" From ffbfd47953a6940057f76f7eaf8e6ebfde241807 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 14:29:34 +0000 Subject: [PATCH 088/246] update api contract --- cmd/main.go | 2 +- .../operator.gnmic.dev_targetsources.yaml | 4 ++-- internal/apiserver/apiserver.go | 2 +- internal/apiserver/openapi.yaml | 19 +++++++++---------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 6ab354f..5db5f6e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -238,7 +238,7 @@ func main() { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - api.ChunkSize = discoveryChunkSize + // api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 95f5e15..5c96244 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -57,8 +57,8 @@ spec: type: object pull: properties: - url: - type: string + enabled: + type: boolean type: object type: object x-kubernetes-validations: diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index ed92e1b..ccc8f15 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,6 +1,6 @@ package apiserver -//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml // or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index bd66bb8..d84f265 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -21,23 +21,22 @@ paths: application/json: schema: type: object - required: [TargetSource, Target] properties: - TargetSource: - $ref: '#/components/schemas/TargetSource' - Target: - type: array - items: - $ref: '#/components/schemas/Target' + Targets: + $ref: '#/components/schemas/Targets' components: schemas: - TargetSource: + Targets: type: object properties: - namespace: + targetSourceNameSpace: type: string - name: + TargetSourceName: type: string + TargetList: + type: array + items: + $ref: '#/components/schemas/Target' Target: type: object properties: From 8d7ee19f7c9a2f7fa2c01bc32bbd24a7f46a17e8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 15:43:22 +0000 Subject: [PATCH 089/246] CreateTargets works for complete POST request --- internal/apiserver/apiserver.go | 61 +++++++++++++++------------------ internal/apiserver/gen.go | 24 ++++++------- internal/apiserver/temp.md | 6 ++++ 3 files changed, 45 insertions(+), 46 deletions(-) create mode 100644 internal/apiserver/temp.md diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index ccc8f15..b24c7d7 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,7 +1,8 @@ package apiserver -// go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml -// or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) +//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) +// Then use: go generate ./internal/apiserver import ( "context" @@ -50,49 +51,41 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes func (a *APIServer) CreateTargets(c *gin.Context) { - // logger.Info("Create Targets called") - var payloadTarget []Target - var payloadTargetSource TargetSource + var payloadTargets Targets fmt.Println("Binding Target to PayloadTarget") - // https://gin-gonic.com/en/docs/binding/bind-body-into-different-structs/ - if err := c.ShouldBindBodyWithJSON(&payloadTarget); err != nil { + if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) fmt.Printf("err: %s", err.Error) return } - fmt.Printf("payloadTarget: %s", payloadTarget) - if err := c.ShouldBindBodyWithJSON(&payloadTargetSource); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - // error {"error":"json: cannot unmarshal object into Go value of type []apiserver.Target"} targets := []core.DiscoveryEvent{} - for _, target := range payloadTarget { - event := core.CREATE - switch *target.Operation { - case Create: - event = core.CREATE - case Delete: - event = core.DELETE + if len(*payloadTargets.TargetList) > 0 { // doesn't work on empty TargetList + for _, target := range *payloadTargets.TargetList { + event := core.CREATE + switch *target.Operation { + case Create: + event = core.CREATE + case Delete: + event = core.DELETE + } + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, + }) } - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, - }, - Event: event, - }) } key := types.NamespacedName{ - Namespace: *payloadTargetSource.Namespace, - Name: *payloadTargetSource.Name, + Namespace: *payloadTargets.TargetSourceNameSpace, + Name: *payloadTargets.TargetSourceName, } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { @@ -100,7 +93,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) return } - + fmt.Println("Preparing SendEvents") core.SendEvents(context.Background(), ch, targets, 10) // make number constant - c.JSON(http.StatusOK, payloadTarget) + c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index acc383b..84321bd 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -46,10 +46,11 @@ type Target struct { // TargetOperation defines model for Target.Operation. type TargetOperation string -// TargetSource defines model for TargetSource. -type TargetSource struct { - Name *string `json:"name,omitempty"` - Namespace *string `json:"namespace,omitempty"` +// Targets defines model for Targets. +type Targets struct { + TargetList *[]Target `json:"TargetList,omitempty"` + TargetSourceName *string `json:"TargetSourceName,omitempty"` + TargetSourceNameSpace *string `json:"targetSourceNameSpace,omitempty"` } // ServerInterface represents all server handlers. @@ -131,14 +132,13 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4RSTW/bMAz9KwK3oxG73c23oRiKHLYVa2/DDprMJCpsSSOpAUHg/z7oY0k8Z+hJBPlI", - "ke+9Exg/Be/QCUN/AjYHnHQOXzTtUVIUyAcksZjzehgIOYdyDAg9sJB1e5gbcHrCm4U0QIv1LlXRxQn6", - "72AItSA0MOCIgvCjWTcG8js73h4qep/XsILT7X1qQhPpI8yXhP/5ikYSolz57CMZXN/633tSgYM2t6rr", - "b1LKup3PYCvpHNh/+bw16mvmxZP69un5RX182kIDv5E4MwXdptvcVfqcDhZ6+LC533TQQNByyCu2hcVy", - "R84Ez1m2M+fbAXp4WMAaIOTgHZc777u79BjvBF1u1iGM1uT29pWLbsUca5YuTjkr8Z5wBz28ay/2aqu3", - "2gpfybMW4+0pFZsIJvwVLeGQnLUo/p175a9rYQZkQzYUc1Yoq8LqoDgag8y7OI7FQRynSdPxzKiS2mGd", - "kgOqpa65pQ2jzgRWlpbCPKI8jJEF6SnBVsp06VkueYVXhBLJ4fDPco8oyhSYyt/P8zz/CQAA//+Shfmd", - "7gMAAA==", + "H4sIAAAAAAAC/4yTz2vdMAzH/xWj7RiStLvlNsooD7au7PU2dvAcJc8lsY2kDB4l//uwnf5Ikwc9xchf", + "Wfp+pDyB8WPwDp0wNE/A5oSjTscHTT1KPAXyAUksprhuW0JORzkHhAZYyLoe5gKcHnH3Ij6gxXoXb9FN", + "IzS/wRBqQSigxQEF4U+xTQzkOzvsPyq6T21YwXG/nyWgifQZ5teA//uIRqIiu+StzXzx3bKsSnwm7KCB", + "T9Urt2qBVi3ENnWfqxz9RAbvLjGSd6Jj0GZPubURQ9Z1PomtRFzQ3/04GPUzcfekfn07Pqiv9wco4B8S", + "p0lAXdbl1TIep4OFBr6U12UNBQQtp2S4ylN6y8lnJi8zPbTQwM1KVgAhB+84w7yur+LHeCfoUrIOYbAm", + "pVePnPcic7w0ig/i50uAWmRDNuQlfH5TZXet4skYZO6mYcibwtM4ajq/OFOyZFin5IRqzTelVGHQycjy", + "36wB3aLcDBML0n2UbQjV8bNu8o1eEcpEDtt3zd2iKJNlKpWf53n+HwAA////AVrz1gMAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md new file mode 100644 index 0000000..90f3fb0 --- /dev/null +++ b/internal/apiserver/temp.md @@ -0,0 +1,6 @@ +## CURL request +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' + + +## Empty TargetList +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' \ No newline at end of file From a66accbbcac43a0cdbefa4f59231ca57fca1635f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 18:23:19 -0600 Subject: [PATCH 090/246] moved finalizer label into const file --- internal/controller/const.go | 2 ++ internal/controller/targetsource_controller.go | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/const.go b/internal/controller/const.go index b5196b8..5ef2e8f 100644 --- a/internal/controller/const.go +++ b/internal/controller/const.go @@ -21,6 +21,8 @@ const ( LabelCertType = "operator.gnmic.dev/cert-type" LabelValueCertTypeClient = "client" LabelValueCertTypeTunnel = "tunnel" + + LabelTargetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" ) const ( diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index f04eced..232c624 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -37,8 +37,6 @@ import ( ) const ( - targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" - pipelineMaxRestarts = 5 pipelineBackoff = 3 * time.Second ) @@ -138,8 +136,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type r.stopDiscoveryPipeline(key) // Remove finalizer if exists - if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { - controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer) + if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { + controllerutil.RemoveFinalizer(targetSource, LabelTargetSourceFinalizer) if err := r.Update(ctx, targetSource); err != nil { return ctrl.Result{}, err } @@ -150,11 +148,11 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type // ensureFinalizer adds the finalizer if not present and updates the TargetSource func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error { - if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { return nil } - controllerutil.AddFinalizer(targetSource, targetSourceFinalizer) + controllerutil.AddFinalizer(targetSource, LabelTargetSourceFinalizer) if err := r.Update(ctx, targetSource); err != nil { return err } From 3b2d9258a06116738be182e567ee6f275c9ad0e4 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 18:29:05 -0600 Subject: [PATCH 091/246] fixed typo --- internal/controller/discovery/target_applier.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index ee127c5..3c714bd 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -29,7 +29,7 @@ type TargetApplier struct { queue []core.DiscoveryMessage activeSnapshot *snapshotBuffer // Events are deferred while snapshot is in progress - defferedEvents []core.DiscoveryEvent + deferredEvents []core.DiscoveryEvent } // NewTargetApplier wires a TargetApplier instance @@ -159,7 +159,7 @@ func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger lo complete: false, } // Delete buffered events that will be current with new snapshot - a.defferedEvents = nil + a.deferredEvents = nil a.collectSnapshot(chunk, logger) } @@ -225,8 +225,8 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf // apply all targets // a.applyTargets - // Replay deffered events - for _, event := range a.defferedEvents { + // Replay deferred events + for _, event := range a.deferredEvents { select { case <-ctx.Done(): return nil @@ -238,14 +238,14 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf } a.activeSnapshot = nil - a.defferedEvents = nil + a.deferredEvents = nil return nil } func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events if a.activeSnapshot != nil { - a.defferedEvents = append(a.defferedEvents, event) + a.deferredEvents = append(a.deferredEvents, event) return nil } From 3ba86cb63c45a7f042a2051faca5f8ddfdc5b2ad Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:21:01 -0600 Subject: [PATCH 092/246] restructured loaders package --- .../controller/discovery/loaders/http/{loader.go => http.go} | 0 .../discovery/loaders/http/{loader_test.go => http_test.go} | 0 .../controller/discovery/{loader.go => loaders/loaders.go} | 2 +- internal/controller/targetsource_controller.go | 3 ++- 4 files changed, 3 insertions(+), 2 deletions(-) rename internal/controller/discovery/loaders/http/{loader.go => http.go} (100%) rename internal/controller/discovery/loaders/http/{loader_test.go => http_test.go} (100%) rename internal/controller/discovery/{loader.go => loaders/loaders.go} (97%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/http.go similarity index 100% rename from internal/controller/discovery/loaders/http/loader.go rename to internal/controller/discovery/loaders/http/http.go diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/http_test.go similarity index 100% rename from internal/controller/discovery/loaders/http/loader_test.go rename to internal/controller/discovery/loaders/http/http_test.go diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loaders/loaders.go similarity index 97% rename from internal/controller/discovery/loader.go rename to internal/controller/discovery/loaders/loaders.go index 0d8ddd3..45bf9c1 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loaders/loaders.go @@ -1,4 +1,4 @@ -package discovery +package loaders import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 232c624..77a3a35 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -31,6 +31,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" @@ -207,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loaderConfigured := targetSource.Spec.Provider != nil webhookConfigured := targetSource.Spec.Webhook.Enabled != nil if loaderConfigured { - loader, err := discovery.NewLoader( + loader, err := loaders.NewLoader( key, targetSource.Spec, core.LoaderConfig{ChunkSize: r.ChunkSize}, From d0ac86be2e389e91ef833bf5c278324af2df59bb Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:21:13 -0600 Subject: [PATCH 093/246] restructured target handler --- internal/controller/discovery/client.go | 27 ---- .../{target_applier.go => target_handler.go} | 121 ++++++++++-------- .../controller/targetsource_controller.go | 20 +-- 3 files changed, 80 insertions(+), 88 deletions(-) delete mode 100644 internal/controller/discovery/client.go rename internal/controller/discovery/{target_applier.go => target_handler.go} (66%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go deleted file mode 100644 index 3bc7ef7..0000000 --- a/internal/controller/discovery/client.go +++ /dev/null @@ -1,27 +0,0 @@ -package discovery - -// File may become obsolete, depends on how the logic to compare desired vs. existing state will get implemented - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/client" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" -) - -func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { - var targetList gnmicv1alpha1.TargetList - - err := c.List(ctx, &targetList, - client.InNamespace(ts.Namespace), - client.MatchingLabels{ - "gnmic.io/source": ts.Name, - }, - ) - if err != nil { - return nil, err - } - - return targetList.Items, nil -} diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_handler.go similarity index 66% rename from internal/controller/discovery/target_applier.go rename to internal/controller/discovery/target_handler.go index 3c714bd..e8c0308 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_handler.go @@ -20,8 +20,9 @@ type snapshotBuffer struct { complete bool } -// TargetApplier consumes discovered targets and applies them to Kubernetes -type TargetApplier struct { +// TargetHandler consumes discovered targets and applies them to Kubernetes +type TargetHandler struct { + ctx context.Context client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource @@ -32,9 +33,9 @@ type TargetApplier struct { deferredEvents []core.DiscoveryEvent } -// NewTargetApplier wires a TargetApplier instance -func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier { - return &TargetApplier{ +// NewTargetHandler wires a TargetHandler instance +func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler { + return &TargetHandler{ client: c, scheme: s, targetSource: ts, @@ -44,38 +45,40 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (a *TargetApplier) Run(ctx context.Context) error { - logger := log.FromContext(ctx). +func (c *TargetHandler) Run(ctx context.Context) error { + c.ctx = ctx + + logger := log.FromContext(c.ctx). WithValues( - "name", a.targetSource.Name, - "namespace", a.targetSource.Namespace, + "name", c.targetSource.Name, + "namespace", c.targetSource.Namespace, ) - logger.Info("target applier started") + logger.Info("target handler started") - for ctx.Err() == nil { + for c.ctx.Err() == nil { select { - case batch, ok := <-a.in: + case batch, ok := <-c.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target applier") + logger.Info("input channel closed, stopping target handler") return nil } - a.queue = append(a.queue, batch...) + c.queue = append(c.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target applier") + logger.Info("context canceled, stopping target handler") return nil } - for len(a.queue) > 0 { + for len(c.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := a.queue[0] - a.queue = a.queue[1:] + msg := c.queue[0] + c.queue = c.queue[1:] - if err := a.processMessage(ctx, msg, logger); err != nil { + if err := c.processMessage(c.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -85,11 +88,11 @@ func (a *TargetApplier) Run(ctx context.Context) error { } } - logger.Info("target applier stopped") + logger.Info("target handler stopped") return nil } -func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -104,7 +107,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return a.processSnapshot(ctx, msg, logger) + return c.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -112,7 +115,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove "received discovery event", "target", msg.Target.Name, ) - return a.processEvent(ctx, msg, logger) + return c.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -120,18 +123,18 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if a.activeSnapshot == nil { - a.startNewSnapshot(chunk, logger) +func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if c.activeSnapshot == nil { + c.startNewSnapshot(chunk, logger) return nil } - snapshot := a.activeSnapshot + snapshot := c.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := a.applySnapshot(ctx, snapshot, logger); err != nil { + if err := c.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -144,40 +147,40 @@ func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.Discover } // Start collecting the new snapshot - a.startNewSnapshot(chunk, logger) + c.startNewSnapshot(chunk, logger) return nil } - return a.collectSnapshot(chunk, logger) + return c.collectSnapshot(chunk, logger) } -func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - a.activeSnapshot = &snapshotBuffer{ +func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + c.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - a.deferredEvents = nil + c.deferredEvents = nil - a.collectSnapshot(chunk, logger) + c.collectSnapshot(chunk, logger) } -func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := a.activeSnapshot +func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := c.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } @@ -190,10 +193,10 @@ func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger log return nil } -func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - a.activeSnapshot = nil + c.activeSnapshot = nil return nil default: } @@ -202,7 +205,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - a.activeSnapshot = nil + c.activeSnapshot = nil return nil default: } @@ -210,7 +213,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -226,34 +229,34 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf // a.applyTargets // Replay deferred events - for _, event := range a.deferredEvents { + for _, event := range c.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := a.applyEvent(ctx, event, logger); err != nil { + if err := c.applyEvent(ctx, event, logger); err != nil { return err } } - a.activeSnapshot = nil - a.deferredEvents = nil + c.activeSnapshot = nil + c.deferredEvents = nil return nil } -func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if a.activeSnapshot != nil { - a.deferredEvents = append(a.deferredEvents, event) + if c.activeSnapshot != nil { + c.deferredEvents = append(c.deferredEvents, event) return nil } // Apply events - return a.applyEvent(ctx, event, logger) + return c.applyEvent(ctx, event, logger) } -func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.CREATE: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) @@ -264,3 +267,19 @@ func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEven } return nil } + +func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList + + err := c.client.List(c.ctx, &targetList, + client.InNamespace(c.targetSource.Namespace), + client.MatchingLabels{ + "gnmic.io/source": c.targetSource.Name, + }, + ) + if err != nil { + return nil, err + } + + return targetList.Items, nil +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 77a3a35..4d5f400 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource // // Pipeline semantics: -// 1. target-applier is mandatory and must start first +// 1. target-handler is mandatory and must start first // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister @@ -176,30 +176,30 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create target applier instance - applier := discovery.NewTargetApplier( + // Create target targetHandler instance + targetHandler := discovery.NewTargetHandler( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target applier - applierReady := make(chan struct{}) + // Start target handler + handlerReady := make(chan struct{}) supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-applier", + Name: "target-handler", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(applierReady) // Signals that applier started successfully - return applier.Run(ctx) + close(handlerReady) // Signals that handler started successfully + return targetHandler.Run(ctx) }, }) - // Wait for applier to be ready before starting loader + // Wait for handler to be ready before starting loader select { - case <-applierReady: + case <-handlerReady: case <-supervisor.Done(): return nil } From 240a2bc382c5133829d327bde1cebfb4fd1530e9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:59:29 -0600 Subject: [PATCH 094/246] ran go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f236ded..827da2a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 github.com/go-logr/logr v1.4.3 + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.27.3 github.com/onsi/gomega v1.38.3 github.com/openconfig/gnmic/pkg/api v0.1.10 @@ -47,7 +48,6 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect - github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect From 80d321129925fa53d4374a789c821e76bbe82d95 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 27 Apr 2026 08:55:33 +0000 Subject: [PATCH 095/246] manual implementation of required fields --- go.mod | 2 +- internal/apiserver/apiserver.go | 35 +++++++++++++++++++++++---------- internal/apiserver/gen.go | 31 +++++++++++++++-------------- internal/apiserver/openapi.yaml | 10 ++++++++++ internal/apiserver/temp.md | 9 +++++++-- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 0a1d66c..8e56473 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/gnmic/operator go 1.25.5 require ( + github.com/bytedance/gopkg v0.1.3 github.com/cert-manager/cert-manager v1.19.3 github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 @@ -19,7 +20,6 @@ require ( ) require ( - github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b24c7d7..5865b86 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" + "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" @@ -53,20 +54,35 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes func (a *APIServer) CreateTargets(c *gin.Context) { - var payloadTargets Targets fmt.Println("Binding Target to PayloadTarget") if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - fmt.Printf("err: %s", err.Error) return } + // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware + // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation + // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. + if payloadTargets.TargetSourceNameSpace == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) + return + } + if payloadTargets.TargetSourceName == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) + return + } + + targets := []core.DiscoveryEvent{} - if len(*payloadTargets.TargetList) > 0 { // doesn't work on empty TargetList - for _, target := range *payloadTargets.TargetList { + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Address == "" || target.Name == "" || target.Operation == "" { + logger.Warn("Target receieved at index %s by pull interface does not contain Address, Name or Operation and is skipped.", i) + break + } event := core.CREATE - switch *target.Operation { + switch target.Operation { case Create: event = core.CREATE case Delete: @@ -74,8 +90,8 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, + Name: target.Name, + Address: target.Address, Labels: map[string]string{"key": "Is this a tag?"}, }, Event: event, @@ -84,8 +100,8 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } key := types.NamespacedName{ - Namespace: *payloadTargets.TargetSourceNameSpace, - Name: *payloadTargets.TargetSourceName, + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { @@ -93,7 +109,6 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) return } - fmt.Println("Preparing SendEvents") core.SendEvents(context.Background(), ch, targets, 10) // make number constant c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 84321bd..ae7704c 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -36,11 +36,11 @@ func (e TargetOperation) Valid() bool { // Target defines model for Target. type Target struct { - Address *string `json:"address,omitempty"` - Name *string `json:"name,omitempty"` - Operation *TargetOperation `json:"operation,omitempty"` - Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` + Address string `json:"address"` + Name string `json:"name"` + Operation TargetOperation `json:"operation"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` } // TargetOperation defines model for Target.Operation. @@ -48,9 +48,9 @@ type TargetOperation string // Targets defines model for Targets. type Targets struct { - TargetList *[]Target `json:"TargetList,omitempty"` - TargetSourceName *string `json:"TargetSourceName,omitempty"` - TargetSourceNameSpace *string `json:"targetSourceNameSpace,omitempty"` + TargetList []Target `json:"TargetList"` + TargetSourceName string `json:"TargetSourceName"` + TargetSourceNameSpace string `json:"targetSourceNameSpace"` } // ServerInterface represents all server handlers. @@ -132,13 +132,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTz2vdMAzH/xWj7RiStLvlNsooD7au7PU2dvAcJc8lsY2kDB4l//uwnf5Ikwc9xchf", - "Wfp+pDyB8WPwDp0wNE/A5oSjTscHTT1KPAXyAUksprhuW0JORzkHhAZYyLoe5gKcHnH3Ij6gxXoXb9FN", - "IzS/wRBqQSigxQEF4U+xTQzkOzvsPyq6T21YwXG/nyWgifQZ5teA//uIRqIiu+StzXzx3bKsSnwm7KCB", - "T9Urt2qBVi3ENnWfqxz9RAbvLjGSd6Jj0GZPubURQ9Z1PomtRFzQ3/04GPUzcfekfn07Pqiv9wco4B8S", - "p0lAXdbl1TIep4OFBr6U12UNBQQtp2S4ylN6y8lnJi8zPbTQwM1KVgAhB+84w7yur+LHeCfoUrIOYbAm", - "pVePnPcic7w0ig/i50uAWmRDNuQlfH5TZXet4skYZO6mYcibwtM4ajq/OFOyZFin5IRqzTelVGHQycjy", - "36wB3aLcDBML0n2UbQjV8bNu8o1eEcpEDtt3zd2iKJNlKpWf53n+HwAA////AVrz1gMAAA==", + "H4sIAAAAAAAC/4yTz4rbMBDGX0VMezSxd3vzrSxlCbTbpdlb6UGVJ4kWW1JnxoWw+N2LJCdrxw70ZCHN", + "n29+3/gNjO+Cd+iEoX4DNkfsdDq+aDqgxFMgH5DEYrrXTUPI6SingFADC1l3gKEApztcfYgFtFjv4iu6", + "voP6JxhCLQgFNNiiIPwqlomB/N6260VFH5IMK9it6xkvNJE+wTAUQPint4RNbJ/EFpd5piLflfjfr2gk", + "1so8eAkkP3y1LDMxHwn3UMOH8p1wOeItR7YLhecuO9+TwadbNOUqaBe0WYu8Gng9baVnMZ1pySLWtW7v", + "U0cr0R04PH3bGvU9EfSkfnzZvajPz1so4C8SJ+Oh2lSbu3EbnA4Wavi0ud9UUEDQckzUyrwUU9g+g724", + "s22ghodZWJyTg3ecHbmv7uLHeCfoUrIOobUmpZevnNcwm3HLz//0kBeUz/fr3BpkQzbkX+Ecq/LQjeLe", + "GGTe922b95X7rtN0ugysZMywTskR1Rx7SilDq9N849875/aI8tD2LEjPMWwBroqfuchJvCKUnhw2V+Ie", + "UZTJYSq1H4Zh+BcAAP//nMyhxFwEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index d84f265..38917ee 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -21,6 +21,8 @@ paths: application/json: schema: type: object + required: + - Targets properties: Targets: $ref: '#/components/schemas/Targets' @@ -28,6 +30,10 @@ components: schemas: Targets: type: object + required: + - targetSourceNameSpace + - TargetSourceName + - TargetList properties: targetSourceNameSpace: type: string @@ -39,6 +45,10 @@ components: $ref: '#/components/schemas/Target' Target: type: object + required: + - name + - address + - operation properties: name: type: string diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 90f3fb0..7bca5d8 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,6 +1,11 @@ ## CURL request curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' - ## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' \ No newline at end of file +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' + +## Empty Target in Target List +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' + +## Empty TargetSourceName +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' \ No newline at end of file From b2d3c189efa2a8cab570708f63c0588085481673 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 27 Apr 2026 14:26:32 +0000 Subject: [PATCH 096/246] add URL templating --- cmd/main.go | 3 +- internal/apiserver/apiserver.go | 50 +++++++++++++++++++++++++-------- internal/apiserver/temp.md | 27 +++++++++++++++--- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5db5f6e..125e32d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -233,12 +233,11 @@ func main() { } if apiAddr != "" { - api, err := apiserver.New(apiAddr, clusterReconciler) + api, err := apiserver.New(apiAddr, clusterReconciler, discoveryChunkSize) if err != nil { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - // api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 5865b86..1e75a20 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,6 @@ package apiserver import ( "context" - "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -22,9 +21,15 @@ type APIServer struct { router *gin.Engine clusterReconciler *controller.ClusterReconciler DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + chunkSize int } -func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { +type urlStruct struct { + namespace string `uri:"namespace" binding:"required"` + gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` +} + +func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -33,9 +38,9 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ }, router: router, clusterReconciler: clusterReconciler, + chunkSize: chunkSize, } - - apiBaseURL := "/api/v1/namespaceCluster/namegNMIcCluster" + apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } @@ -44,7 +49,8 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { - plan, err := a.clusterReconciler.GetClusterPlan("temp", "temp") + url := parseURI(c) + plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcClusterName) if err != nil { c.String(404, err.Error()) return @@ -52,10 +58,15 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { + // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template + // But I don't think it is needed in the CreateTargets function + // url := parseURI(c) + // fmt.Printf("namespace: %s", url.namespace) + // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) + var payloadTargets Targets - fmt.Println("Binding Target to PayloadTarget") if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return @@ -73,14 +84,18 @@ func (a *APIServer) CreateTargets(c *gin.Context) { return } - targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index %s by pull interface does not contain Address, Name or Operation and is skipped.", i) + logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") + break + } + if target.Operation.Valid() != true { + logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") break } + event := core.CREATE switch target.Operation { case Create: @@ -88,6 +103,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { case Delete: event = core.DELETE } + targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, @@ -105,10 +121,20 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { - // Error message to be udpated!! - c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) + logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) return } - core.SendEvents(context.Background(), ch, targets, 10) // make number constant + core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } + +// parseURI parses URI to urlStruct. +func parseURI(c *gin.Context) (url urlStruct) { + var u urlStruct + if err := c.ShouldBindUri(&u); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + return u +} diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 7bca5d8..fae3f49 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,11 +1,30 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' ## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' ## Empty Target in Target List -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' ## Empty TargetSourceName -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' \ No newline at end of file +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' + +## Wrong operation +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' + + +http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets +{ + "TargetSourceName": "netbox", + "TargetSourceNameSpace": "netbox", + "TargetList": [ + { + "name": "{{ data.name }}", + "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", + "profile": "{{ data.custom_fields.profile | default('') }}", + "tags": {{ data.tags | map(attribute='name') | list | tojson }}, + "operation":"create" + } + ] +} From 7ef1281a7b37bd8b9a845501f7011c615710429b Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 10:10:08 -0600 Subject: [PATCH 097/246] renamed target applier to message processor & created client.go for generic functions --- internal/controller/discovery/client.go | 25 ++++ ...target_handler.go => message_processor.go} | 112 ++++++++---------- .../controller/targetsource_controller.go | 2 +- 3 files changed, 74 insertions(+), 65 deletions(-) create mode 100644 internal/controller/discovery/client.go rename internal/controller/discovery/{target_handler.go => message_processor.go} (63%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go new file mode 100644 index 0000000..72147b7 --- /dev/null +++ b/internal/controller/discovery/client.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList + + err := c.List(ctx, &targetList, + client.InNamespace(ts.Namespace), + client.MatchingLabels{ + "gnmic.io/source": ts.Name, + }, + ) + if err != nil { + return nil, err + } + + return targetList.Items, nil +} diff --git a/internal/controller/discovery/target_handler.go b/internal/controller/discovery/message_processor.go similarity index 63% rename from internal/controller/discovery/target_handler.go rename to internal/controller/discovery/message_processor.go index e8c0308..65c8b44 100644 --- a/internal/controller/discovery/target_handler.go +++ b/internal/controller/discovery/message_processor.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// TargetHandler consumes discovered targets and applies them to Kubernetes -type TargetHandler struct { +// MessageProcessor consumes discovered targets and applies them to Kubernetes +type MessageProcessor struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type TargetHandler struct { deferredEvents []core.DiscoveryEvent } -// NewTargetHandler wires a TargetHandler instance -func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler { - return &TargetHandler{ +// NewMessageProcessor wires a MessageProcessor instance +func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { + return &MessageProcessor{ client: c, scheme: s, targetSource: ts, @@ -45,40 +45,40 @@ func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (c *TargetHandler) Run(ctx context.Context) error { - c.ctx = ctx +func (m *MessageProcessor) Run(ctx context.Context) error { + m.ctx = ctx - logger := log.FromContext(c.ctx). + logger := log.FromContext(m.ctx). WithValues( - "name", c.targetSource.Name, - "namespace", c.targetSource.Namespace, + "name", m.targetSource.Name, + "namespace", m.targetSource.Namespace, ) logger.Info("target handler started") - for c.ctx.Err() == nil { + for m.ctx.Err() == nil { select { - case batch, ok := <-c.in: + case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target handler") return nil } - c.queue = append(c.queue, batch...) + m.queue = append(m.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target handler") return nil } - for len(c.queue) > 0 { + for len(m.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := c.queue[0] - c.queue = c.queue[1:] + msg := m.queue[0] + m.queue = m.queue[1:] - if err := c.processMessage(c.ctx, msg, logger); err != nil { + if err := m.processMessage(m.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -92,7 +92,7 @@ func (c *TargetHandler) Run(ctx context.Context) error { return nil } -func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -107,7 +107,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return c.processSnapshot(ctx, msg, logger) + return m.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -115,7 +115,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove "received discovery event", "target", msg.Target.Name, ) - return c.processEvent(ctx, msg, logger) + return m.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -123,18 +123,18 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if c.activeSnapshot == nil { - c.startNewSnapshot(chunk, logger) +func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if m.activeSnapshot == nil { + m.startNewSnapshot(chunk, logger) return nil } - snapshot := c.activeSnapshot + snapshot := m.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := c.applySnapshot(ctx, snapshot, logger); err != nil { + if err := m.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -147,40 +147,40 @@ func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.Discover } // Start collecting the new snapshot - c.startNewSnapshot(chunk, logger) + m.startNewSnapshot(chunk, logger) return nil } - return c.collectSnapshot(chunk, logger) + return m.collectSnapshot(chunk, logger) } -func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - c.activeSnapshot = &snapshotBuffer{ +func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + m.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - c.deferredEvents = nil + m.deferredEvents = nil - c.collectSnapshot(chunk, logger) + m.collectSnapshot(chunk, logger) } -func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := c.activeSnapshot +func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := m.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } @@ -193,10 +193,10 @@ func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger log return nil } -func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - c.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -205,7 +205,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - c.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -213,7 +213,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -229,34 +229,34 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf // a.applyTargets // Replay deferred events - for _, event := range c.deferredEvents { + for _, event := range m.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := c.applyEvent(ctx, event, logger); err != nil { + if err := m.applyEvent(ctx, event, logger); err != nil { return err } } - c.activeSnapshot = nil - c.deferredEvents = nil + m.activeSnapshot = nil + m.deferredEvents = nil return nil } -func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if c.activeSnapshot != nil { - c.deferredEvents = append(c.deferredEvents, event) + if m.activeSnapshot != nil { + m.deferredEvents = append(m.deferredEvents, event) return nil } // Apply events - return c.applyEvent(ctx, event, logger) + return m.applyEvent(ctx, event, logger) } -func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.CREATE: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) @@ -267,19 +267,3 @@ func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEven } return nil } - -func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) { - var targetList gnmicv1alpha1.TargetList - - err := c.client.List(c.ctx, &targetList, - client.InNamespace(c.targetSource.Namespace), - client.MatchingLabels{ - "gnmic.io/source": c.targetSource.Name, - }, - ) - if err != nil { - return nil, err - } - - return targetList.Items, nil -} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 4d5f400..8070a3a 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -177,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target targetHandler instance - targetHandler := discovery.NewTargetHandler( + targetHandler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From 7bcbcc023ff39e36a565e9235f503a98375f3327 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 15:15:56 -0600 Subject: [PATCH 098/246] added const file for common labels --- internal/controller/discovery/client.go | 3 ++- internal/controller/discovery/core/const.go | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 internal/controller/discovery/core/const.go diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index 3bc7ef7..d23c043 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" ) func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { @@ -16,7 +17,7 @@ func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1 err := c.List(ctx, &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ - "gnmic.io/source": ts.Name, + core.LabelTargetSourceName: ts.Name, }, ) if err != nil { diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/core/const.go new file mode 100644 index 0000000..82a5962 --- /dev/null +++ b/internal/controller/discovery/core/const.go @@ -0,0 +1,6 @@ +package core + +const ( + // Labels + LabelTargetSourceName = "operator.gnmic.dev/targetsource" +) From d10fc9ac868d50be64c123cbc619b2f4eb189682 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 11:26:10 -0600 Subject: [PATCH 099/246] removed all package --- internal/controller/discovery/loaders/all/all.go | 5 ----- internal/controller/targetsource_controller.go | 1 - 2 files changed, 6 deletions(-) delete mode 100644 internal/controller/discovery/loaders/all/all.go diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go deleted file mode 100644 index 3590cda..0000000 --- a/internal/controller/discovery/loaders/all/all.go +++ /dev/null @@ -1,5 +0,0 @@ -package all - -import ( - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http" -) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8070a3a..49f9683 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -32,7 +32,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) From 108bd2dc3f58b2193535c8eadf6c30ee1d6d0dad Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 11:34:11 -0600 Subject: [PATCH 100/246] changed error lookup to apierrors --- internal/controller/targetsource_controller.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 232c624..2f198a6 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -21,6 +21,7 @@ import ( "sync" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -81,13 +82,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request WithValues("targetsource", req.NamespacedName) targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) - if err != nil { - // If the TargetSource no longer exists, ensure runtime cleanup - if client.IgnoreNotFound(err) == nil { - logger.Info("TargetSource not found; stopping discovery pipeline") - r.stopDiscoveryPipeline(req.NamespacedName) - return ctrl.Result{}, nil - } + // If the TargetSource no longer exists, ensure runtime cleanup + if apierrors.IsNotFound(err) { + logger.Info("TargetSource not found; stopping discovery pipeline") + r.stopDiscoveryPipeline(req.NamespacedName) + return ctrl.Result{}, nil + } else if err != nil { return ctrl.Result{}, err } From b7dd0367e99a0c5435db00092c83e1bc01ab439b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 08:53:30 +0000 Subject: [PATCH 101/246] remove unused fiels --- internal/controller/discovery/mapper.go | 4 ---- internal/controller/discovery/mapper_test.go | 1 - 2 files changed, 5 deletions(-) delete mode 100644 internal/controller/discovery/mapper.go delete mode 100644 internal/controller/discovery/mapper_test.go diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go deleted file mode 100644 index 18470b2..0000000 --- a/internal/controller/discovery/mapper.go +++ /dev/null @@ -1,4 +0,0 @@ -package discovery - -// This file makes diff between existing and new targets -// file decides which targets to create/update/delete diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go deleted file mode 100644 index 5844159..0000000 --- a/internal/controller/discovery/mapper_test.go +++ /dev/null @@ -1 +0,0 @@ -package discovery From d3a9b5ca3021c9f0485698c1d1c54bbd3562bb9b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 11:56:56 +0000 Subject: [PATCH 102/246] rename files and restructure packages --- .../core/{loader_interface.go => loader.go} | 0 .../core/{message_interface.go => message.go} | 0 .../discovery/core/{helpers.go => send.go} | 0 internal/controller/discovery/core/types.go | 4 ++-- internal/controller/discovery/discovery.go | 17 +++++++++++++++++ .../loaders/{loaders.go => factory.go} | 0 .../loaders/http/{http.go => loader.go} | 0 .../http/{http_test.go => loader_test.go} | 0 .../discovery/{ => pipeline}/supervisor.go | 5 +++-- .../discovery/{ => reconciler}/client.go | 13 ++++++++++--- .../{ => reconciler}/message_processor.go | 2 +- internal/controller/targetsource_controller.go | 15 ++++++++------- 12 files changed, 41 insertions(+), 15 deletions(-) rename internal/controller/discovery/core/{loader_interface.go => loader.go} (100%) rename internal/controller/discovery/core/{message_interface.go => message.go} (100%) rename internal/controller/discovery/core/{helpers.go => send.go} (100%) create mode 100644 internal/controller/discovery/discovery.go rename internal/controller/discovery/loaders/{loaders.go => factory.go} (100%) rename internal/controller/discovery/loaders/http/{http.go => loader.go} (100%) rename internal/controller/discovery/loaders/http/{http_test.go => loader_test.go} (100%) rename internal/controller/discovery/{ => pipeline}/supervisor.go (95%) rename internal/controller/discovery/{ => reconciler}/client.go (68%) rename internal/controller/discovery/{ => reconciler}/message_processor.go (99%) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader.go similarity index 100% rename from internal/controller/discovery/core/loader_interface.go rename to internal/controller/discovery/core/loader.go diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message.go similarity index 100% rename from internal/controller/discovery/core/message_interface.go rename to internal/controller/discovery/core/message.go diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/send.go similarity index 100% rename from internal/controller/discovery/core/helpers.go rename to internal/controller/discovery/core/send.go diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 3f6957a..28ec503 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -12,14 +12,14 @@ type DiscoveredTarget struct { Labels map[string]string } +type EventAction int + const ( DELETE EventAction = 0 CREATE EventAction = 1 UPDATE EventAction = 2 ) -type EventAction int - type DiscoveryEvent struct { Target DiscoveredTarget Event EventAction diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go new file mode 100644 index 0000000..3dc51bd --- /dev/null +++ b/internal/controller/discovery/discovery.go @@ -0,0 +1,17 @@ +package discovery + +// Package discovery implements the discovery runtime subsystem. +// +// The discovery subsystem is responsible for: +// - Receiving discovery data from external providers (loaders, webhooks). +// - Supervising discovery pipelines and restart semantics. +// - Applying discovered state to Kubernetes Targets. +// +// The package is structured into the following subpackages: +// - core: message contracts, snapshot/event types, and transport helpers. +// - pipeline: supervision, restart policies, and lifecycle control. +// - reconciler: snapshot + event target state application logic. +// - loaders: target discovery providers (HTTP, webhook, etc.). +// - registry: key -> channel registry. +// +// At the moment, the targetsource controller imports specific subpackages explicitly. diff --git a/internal/controller/discovery/loaders/loaders.go b/internal/controller/discovery/loaders/factory.go similarity index 100% rename from internal/controller/discovery/loaders/loaders.go rename to internal/controller/discovery/loaders/factory.go diff --git a/internal/controller/discovery/loaders/http/http.go b/internal/controller/discovery/loaders/http/loader.go similarity index 100% rename from internal/controller/discovery/loaders/http/http.go rename to internal/controller/discovery/loaders/http/loader.go diff --git a/internal/controller/discovery/loaders/http/http_test.go b/internal/controller/discovery/loaders/http/loader_test.go similarity index 100% rename from internal/controller/discovery/loaders/http/http_test.go rename to internal/controller/discovery/loaders/http/loader_test.go diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/pipeline/supervisor.go similarity index 95% rename from internal/controller/discovery/supervisor.go rename to internal/controller/discovery/pipeline/supervisor.go index 710381e..042d305 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/pipeline/supervisor.go @@ -1,4 +1,4 @@ -package discovery +package pipeline import ( "context" @@ -25,12 +25,13 @@ type Supervisor struct { stopped bool } -// RestartPolicy defines the restart behavior for a component +// RestartPolicy defines restart behavior of a component type RestartPolicy struct { MaxRestarts int Backoff time.Duration } +// ComponentSpec defines a supervised component type ComponentSpec struct { Name string Run func(ctx context.Context) error diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/reconciler/client.go similarity index 68% rename from internal/controller/discovery/client.go rename to internal/controller/discovery/reconciler/client.go index 25100bd..4bbbbc1 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/reconciler/client.go @@ -1,4 +1,4 @@ -package discovery +package reconciler import ( "context" @@ -9,10 +9,17 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) -func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { +func fetchExistingTargets( + ctx context.Context, + c client.Client, + ts *gnmicv1alpha1.TargetSource, +) ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList - err := c.List(ctx, &targetList, + err := c.List( + ctx, + &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ core.LabelTargetSourceName: ts.Name, diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go similarity index 99% rename from internal/controller/discovery/message_processor.go rename to internal/controller/discovery/reconciler/message_processor.go index 65c8b44..0c205bd 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -1,4 +1,4 @@ -package discovery +package reconciler import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 49f9683..35946d2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -29,9 +29,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" + "github.com/gnmic/operator/internal/controller/discovery/pipeline" + "github.com/gnmic/operator/internal/controller/discovery/reconciler" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -168,7 +169,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { - supervisor := discovery.NewSupervisor(context.Background()) + supervisor := pipeline.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { @@ -176,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target targetHandler instance - targetHandler := discovery.NewMessageProcessor( + targetHandler := reconciler.NewMessageProcessor( r.Client, r.Scheme, targetSource, @@ -184,9 +185,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target handler handlerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ + supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "target-handler", - Policy: discovery.RestartPolicy{ + Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, @@ -217,9 +218,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ + supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "loader", - Policy: discovery.RestartPolicy{ + Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, From 0c80394ab358c662fe519b872ed7219c2f7e384c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:01:40 +0000 Subject: [PATCH 103/246] rename target handler to target reconciler --- .../discovery/reconciler/message_processor.go | 8 ++++---- internal/controller/targetsource_controller.go | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go index 0c205bd..2c4632c 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -53,20 +53,20 @@ func (m *MessageProcessor) Run(ctx context.Context) error { "name", m.targetSource.Name, "namespace", m.targetSource.Namespace, ) - logger.Info("target handler started") + logger.Info("target reconciler started") for m.ctx.Err() == nil { select { case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target handler") + logger.Info("input channel closed, stopping target reconciler") return nil } m.queue = append(m.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target handler") + logger.Info("context canceled, stopping target reconciler") return nil } @@ -88,7 +88,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { } } - logger.Info("target handler stopped") + logger.Info("target reconciler stopped") return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 35946d2..6c9ad31 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource // // Pipeline semantics: -// 1. target-handler is mandatory and must start first +// 1. target reconciler is mandatory and must start first // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister @@ -176,14 +176,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create target targetHandler instance - targetHandler := reconciler.NewMessageProcessor( + // Create target reconciler instance + targetReconciler := reconciler.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target handler + // Start target reconciler handlerReady := make(chan struct{}) supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "target-handler", @@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName EscalatesOnFailure: true, Run: func(ctx context.Context) error { close(handlerReady) // Signals that handler started successfully - return targetHandler.Run(ctx) + return targetReconciler.Run(ctx) }, }) // Wait for handler to be ready before starting loader From 04208bf078b170160a6ef72eda6b6ddaa3630070 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:20:58 +0000 Subject: [PATCH 104/246] rename handler to reconciler --- internal/controller/targetsource_controller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 6c9ad31..9078af2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -184,22 +184,22 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetChannel, ) // Start target reconciler - handlerReady := make(chan struct{}) + reconcilerReady := make(chan struct{}) supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ - Name: "target-handler", + Name: "target-reconciler", Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(handlerReady) // Signals that handler started successfully + close(reconcilerReady) // Signals that reconciler started successfully return targetReconciler.Run(ctx) }, }) - // Wait for handler to be ready before starting loader + // Wait for reconciler to be ready before starting loader select { - case <-handlerReady: + case <-reconcilerReady: case <-supervisor.Done(): return nil } From c3818ce6f7693360496866d7ba1694f7ce702f32 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:21:46 +0000 Subject: [PATCH 105/246] clarify interface files --- .../discovery/core/{loader.go => loader_interface.go} | 2 +- internal/controller/discovery/core/message.go | 4 ---- internal/controller/discovery/core/message_interface.go | 5 +++++ 3 files changed, 6 insertions(+), 5 deletions(-) rename internal/controller/discovery/core/{loader.go => loader_interface.go} (91%) create mode 100644 internal/controller/discovery/core/message_interface.go diff --git a/internal/controller/discovery/core/loader.go b/internal/controller/discovery/core/loader_interface.go similarity index 91% rename from internal/controller/discovery/core/loader.go rename to internal/controller/discovery/core/loader_interface.go index 8964be8..72f1898 100644 --- a/internal/controller/discovery/core/loader.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -14,7 +14,7 @@ type Loader interface { Name() string // Start begins discovery and pushes target snapshots or events into the out channel - // The loader must stop cleanly when ctx is cancelled + // The loader must stop cleanly when ctx is canceled Start( ctx context.Context, targetsourceName types.NamespacedName, diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go index 0836bc6..af4f6c1 100644 --- a/internal/controller/discovery/core/message.go +++ b/internal/controller/discovery/core/message.go @@ -1,8 +1,4 @@ package core -type DiscoveryMessage interface { - isDiscoveryMessage() -} - func (DiscoveryEvent) isDiscoveryMessage() {} func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go new file mode 100644 index 0000000..07b819e --- /dev/null +++ b/internal/controller/discovery/core/message_interface.go @@ -0,0 +1,5 @@ +package core + +type DiscoveryMessage interface { + isDiscoveryMessage() +} From ab09c7c6c76ea36b27d89049e9d45953fd905801 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 29 Apr 2026 12:23:07 +0000 Subject: [PATCH 106/246] small code refactor --- internal/apiserver/apiserver.go | 57 +++++++++++++++++---------------- internal/apiserver/gen.go | 27 +++++++++------- internal/apiserver/openapi.yaml | 5 +-- internal/apiserver/temp.md | 14 ++++---- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 1e75a20..b6f04e8 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,6 +6,7 @@ package apiserver import ( "context" + "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -25,8 +26,8 @@ type APIServer struct { } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` - gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` + namespace string `uri:"namespace" binding:"required"` + gNMIcControllerName string `uri:"gNMIcControllerName" binding:"required"` } func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { @@ -40,7 +41,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize clusterReconciler: clusterReconciler, chunkSize: chunkSize, } - apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" + apiBaseURL := "/api/v1/:namespace/:gNMIcControllerName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } @@ -50,7 +51,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) - plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcClusterName) + plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcControllerName) if err != nil { c.String(404, err.Error()) return @@ -60,12 +61,7 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { - // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template - // But I don't think it is needed in the CreateTargets function - // url := parseURI(c) - // fmt.Printf("namespace: %s", url.namespace) - // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) - + logger.Info("received POST request for CreateTargets.") var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) @@ -76,32 +72,50 @@ func (a *APIServer) CreateTargets(c *gin.Context) { // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. if payloadTargets.TargetSourceNameSpace == "" { + logger.Error("POST request does not contain value targetSourceNameSpace.") c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) return } if payloadTargets.TargetSourceName == "" { + logger.Error("POST request does not contain value targetSourceName.") c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) return } + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + ch, ok := a.DiscoveryRegistry.Get(key) + if !ok { + logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) + return + } + + fmt.Printf("payloadTargets %+v\n", payloadTargets) targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") + logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") break } if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") + logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") break } event := core.CREATE switch target.Operation { - case Create: - event = core.CREATE - case Delete: + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: event = core.DELETE + default: + logger.Warn("Received invalid Operation flag") } targets = append(targets, core.DiscoveryEvent{ @@ -114,17 +128,6 @@ func (a *APIServer) CreateTargets(c *gin.Context) { }) } } - - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) - if !ok { - logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) - return - } core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } @@ -137,4 +140,4 @@ func parseURI(c *gin.Context) (url urlStruct) { return } return u -} +} \ No newline at end of file diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index ae7704c..11e23a2 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -18,16 +18,19 @@ import ( // Defines values for TargetOperation. const ( - Create TargetOperation = "create" - Delete TargetOperation = "delete" + Created TargetOperation = "created" + Deleted TargetOperation = "deleted" + Updated TargetOperation = "updated" ) // Valid indicates whether the value is a known member of the TargetOperation enum. func (e TargetOperation) Valid() bool { switch e { - case Create: + case Created: return true - case Delete: + case Deleted: + return true + case Updated: return true default: return false @@ -132,14 +135,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTz4rbMBDGX0VMezSxd3vzrSxlCbTbpdlb6UGVJ4kWW1JnxoWw+N2LJCdrxw70ZCHN", - "n29+3/gNjO+Cd+iEoX4DNkfsdDq+aDqgxFMgH5DEYrrXTUPI6SingFADC1l3gKEApztcfYgFtFjv4iu6", - "voP6JxhCLQgFNNiiIPwqlomB/N6260VFH5IMK9it6xkvNJE+wTAUQPint4RNbJ/EFpd5piLflfjfr2gk", - "1so8eAkkP3y1LDMxHwn3UMOH8p1wOeItR7YLhecuO9+TwadbNOUqaBe0WYu8Gng9baVnMZ1pySLWtW7v", - "U0cr0R04PH3bGvU9EfSkfnzZvajPz1so4C8SJ+Oh2lSbu3EbnA4Wavi0ud9UUEDQckzUyrwUU9g+g724", - "s22ghodZWJyTg3ecHbmv7uLHeCfoUrIOobUmpZevnNcwm3HLz//0kBeUz/fr3BpkQzbkX+Ecq/LQjeLe", - "GGTe922b95X7rtN0ugysZMywTskR1Rx7SilDq9N849875/aI8tD2LEjPMWwBroqfuchJvCKUnhw2V+Ie", - "UZTJYSq1H4Zh+BcAAP//nMyhxFwEAAA=", + "H4sIAAAAAAAC/4yTzYrbMBDHX0VMezSxd3vzrSxlCbTbpdlb6UGVJokWW1JHo0JY/O5FkpO1Ywd60iDN", + "x39+M3oD5XrvLFoO0L5BUEfsZTZfJB2Qk+XJeSQ2mO+l1oQhm3zyCC0EJmMPMFRgZY+rDymBZONsekUb", + "e2h/giKUjBoqiF6PlsYOk/WrWibx5PamWy/A8pAlGcZ+Xdt4IYnkCYahAsI/0RDqJCULry69TQW/K3G/", + "X1FxylXYhCWc8vDVBJ6J+Ui4hxY+1O+06xF1PXJeKDxX2blICp9ukeUrp52Xas3zquH1sJWa1bSnJYuU", + "19i9yxUNp+nA4enbVonvmaAj8ePL7kV8ft5CBX+RQl4CaDbN5m7cDCu9gRY+be43DVTgJR8ztbosyBS2", + "K2Av09lqaOFh5pb6DN7ZUCZy39ylQznLaHOw9L4zKofXr6GsZBnGrXn+5wzDgvL5fp2bxqDI+PItzr5i", + "/BUiRKUwhH3surKvIfa9pNOlYcFjhLGCjyjm2HNI7TuZ+xt/8pzbI/JDFwMjPSe3BbgmHXORE39ByJEs", + "6itxj8hCFTeRyw/DMPwLAAD//2Qu5BBoBAAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 38917ee..5accf2b 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -63,5 +63,6 @@ components: operation: type: string enum: - - create - - delete \ No newline at end of file + - created + - updated + - deleted \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index fae3f49..715ef09 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' @@ -14,17 +14,17 @@ curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' -http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets + { - "TargetSourceName": "netbox", - "TargetSourceNameSpace": "netbox", + "TargetSourceName": "webhook-test", + "TargetSourceNameSpace": "default", "TargetList": [ { "name": "{{ data.name }}", "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", "profile": "{{ data.custom_fields.profile | default('') }}", - "tags": {{ data.tags | map(attribute='name') | list | tojson }}, - "operation":"create" + "tags": [{% for tag in data.tags %}"{{ tag.name }}"{% if not loop.last %}, {% endif %}{% endfor %}], + "operation": "{{ event }}" } ] -} +} \ No newline at end of file From e4df0d4a6245d71d48539414b0f3ab45136de874 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:35:14 +0000 Subject: [PATCH 107/246] define EventAction to be go idomatic --- internal/controller/discovery/core/types.go | 20 +++++++++++-------- .../discovery/reconciler/message_processor.go | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 28ec503..1ae2f7a 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -4,6 +4,18 @@ type LoaderConfig struct { ChunkSize int } +// EventAction represents the type of a discovery event +type EventAction int + +const ( + // EventDelete indicates that a target should be removed + EventDelete EventAction = iota + // EventCreate indicates that a target should be created + EventCreate + // EventUpdate indicates that a target should be updated + EventUpdate +) + // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { @@ -12,14 +24,6 @@ type DiscoveredTarget struct { Labels map[string]string } -type EventAction int - -const ( - DELETE EventAction = 0 - CREATE EventAction = 1 - UPDATE EventAction = 2 -) - type DiscoveryEvent struct { Target DiscoveredTarget Event EventAction diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go index 2c4632c..a0e91e5 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -258,11 +258,11 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { - case core.CREATE: + case core.EventCreate: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.UPDATE: + case core.EventUpdate: logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.DELETE: + case core.EventDelete: logger.Info("Would delete target", "name", event.Target.Name) } return nil From 86c0af066faef2af3e75d68d3285c16dc6978bbe Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 13:49:19 +0000 Subject: [PATCH 108/246] add webhook activation info to metadata of DiscoveryRegistry --- cmd/main.go | 2 +- internal/apiserver/apiserver.go | 2 +- internal/controller/discovery/core/types.go | 5 +++++ .../controller/discovery/registry/registry.go | 14 +++++++------- internal/controller/targetsource_controller.go | 18 +++++++++++------- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e4bad31..4cf6e94 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -86,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]() + discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 17e5c82..a7ca16a 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -14,7 +14,7 @@ type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 1ae2f7a..68c9c7e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,5 +1,10 @@ package core +type DiscoveryRegistryValue struct { + Channel chan<- []DiscoveryMessage + WebhookEnabled bool +} + type LoaderConfig struct { ChunkSize int } diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 093bd2c..f2630e8 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -10,20 +10,20 @@ import ( // DO NOT USE a pointer type as K type Registry[K comparable, V any] struct { mu sync.RWMutex - m map[K]chan<- V + m map[K]V } func NewRegistry[K comparable, V any]() *Registry[K, V] { - return &Registry[K, V]{m: make(map[K]chan<- V)} + return &Registry[K, V]{m: make(map[K]V)} } -func (r *Registry[K, V]) Register(key K, ch chan<- V) error { +func (r *Registry[K, V]) Register(key K, value V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { return fmt.Errorf("already registered: %v", key) } - r.m[key] = ch + r.m[key] = value return nil } @@ -33,9 +33,9 @@ func (r *Registry[K, V]) Unregister(key K) { r.mu.Unlock() } -func (r *Registry[K, V]) Get(key K) (chan<- V, bool) { +func (r *Registry[K, V]) Get(key K) (V, bool) { r.mu.RLock() - ch, ok := r.m[key] + value, ok := r.m[key] r.mu.RUnlock() - return ch, ok + return value, ok } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9078af2..c7e6460 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -66,7 +66,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -108,7 +108,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info("Discover pipeline started") + logger.Info("Discovery pipeline started") return ctrl.Result{}, nil } @@ -161,7 +161,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } -// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource +// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource // // Pipeline semantics: // 1. target reconciler is mandatory and must start first @@ -169,10 +169,16 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { + loaderConfigured := targetSource.Spec.Provider != nil + webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + supervisor := pipeline.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { + if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ + Channel: targetChannel, + WebhookEnabled: webhookActivated, + }); err != nil { return err } @@ -205,8 +211,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create loader instance - loaderConfigured := targetSource.Spec.Provider != nil - webhookConfigured := targetSource.Spec.Webhook.Enabled != nil if loaderConfigured { loader, err := loaders.NewLoader( key, @@ -224,7 +228,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - EscalatesOnFailure: !webhookConfigured, + EscalatesOnFailure: !webhookActivated, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) }, From 284b1f290bd7f1c33f6213bba5399fb16ac0dae9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:33:40 -0600 Subject: [PATCH 109/246] moved reconciler files to discovery --- internal/controller/discovery/{reconciler => }/client.go | 2 +- .../discovery/{reconciler => }/message_processor.go | 2 +- internal/controller/targetsource_controller.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename internal/controller/discovery/{reconciler => }/client.go (96%) rename internal/controller/discovery/{reconciler => }/message_processor.go (99%) diff --git a/internal/controller/discovery/reconciler/client.go b/internal/controller/discovery/client.go similarity index 96% rename from internal/controller/discovery/reconciler/client.go rename to internal/controller/discovery/client.go index 4bbbbc1..2deb477 100644 --- a/internal/controller/discovery/reconciler/client.go +++ b/internal/controller/discovery/client.go @@ -1,4 +1,4 @@ -package reconciler +package discovery import ( "context" diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/message_processor.go similarity index 99% rename from internal/controller/discovery/reconciler/message_processor.go rename to internal/controller/discovery/message_processor.go index a0e91e5..6e69c99 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -1,4 +1,4 @@ -package reconciler +package discovery import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c7e6460..84f9a6f 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -29,10 +29,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/gnmic/operator/internal/controller/discovery/pipeline" - "github.com/gnmic/operator/internal/controller/discovery/reconciler" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target reconciler instance - targetReconciler := reconciler.NewMessageProcessor( + targetReconciler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From b59897c253b5db8858a03026ae187ac6c8959d19 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:34:55 -0600 Subject: [PATCH 110/246] renamed messageProcessor to targetReconciler --- ...sage_processor.go => target_reconciler.go} | 96 +++++++++---------- .../controller/targetsource_controller.go | 2 +- 2 files changed, 49 insertions(+), 49 deletions(-) rename internal/controller/discovery/{message_processor.go => target_reconciler.go} (72%) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/target_reconciler.go similarity index 72% rename from internal/controller/discovery/message_processor.go rename to internal/controller/discovery/target_reconciler.go index 6e69c99..4f3711c 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/target_reconciler.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// MessageProcessor consumes discovered targets and applies them to Kubernetes -type MessageProcessor struct { +// TargetReconciler consumes discovered targets and applies them to Kubernetes +type TargetReconciler struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type MessageProcessor struct { deferredEvents []core.DiscoveryEvent } -// NewMessageProcessor wires a MessageProcessor instance -func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { - return &MessageProcessor{ +// NewTargetReconciler wires a TargetReconciler instance +func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler { + return &TargetReconciler{ client: c, scheme: s, targetSource: ts, @@ -45,40 +45,40 @@ func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *MessageProcessor) Run(ctx context.Context) error { - m.ctx = ctx +func (r *TargetReconciler) Run(ctx context.Context) error { + r.ctx = ctx - logger := log.FromContext(m.ctx). + logger := log.FromContext(r.ctx). WithValues( - "name", m.targetSource.Name, - "namespace", m.targetSource.Namespace, + "name", r.targetSource.Name, + "namespace", r.targetSource.Namespace, ) logger.Info("target reconciler started") - for m.ctx.Err() == nil { + for r.ctx.Err() == nil { select { - case batch, ok := <-m.in: + case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target reconciler") return nil } - m.queue = append(m.queue, batch...) + r.queue = append(r.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target reconciler") return nil } - for len(m.queue) > 0 { + for len(r.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := m.queue[0] - m.queue = m.queue[1:] + msg := r.queue[0] + r.queue = r.queue[1:] - if err := m.processMessage(m.ctx, msg, logger); err != nil { + if err := r.processMessage(r.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -92,7 +92,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { return nil } -func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -107,7 +107,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return m.processSnapshot(ctx, msg, logger) + return r.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -115,7 +115,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc "received discovery event", "target", msg.Target.Name, ) - return m.processEvent(ctx, msg, logger) + return r.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -123,18 +123,18 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if m.activeSnapshot == nil { - m.startNewSnapshot(chunk, logger) +func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if r.activeSnapshot == nil { + r.startNewSnapshot(chunk, logger) return nil } - snapshot := m.activeSnapshot + snapshot := r.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := m.applySnapshot(ctx, snapshot, logger); err != nil { + if err := r.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -147,40 +147,40 @@ func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.Disco } // Start collecting the new snapshot - m.startNewSnapshot(chunk, logger) + r.startNewSnapshot(chunk, logger) return nil } - return m.collectSnapshot(chunk, logger) + return r.collectSnapshot(chunk, logger) } -func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - m.activeSnapshot = &snapshotBuffer{ +func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + r.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - m.deferredEvents = nil + r.deferredEvents = nil - m.collectSnapshot(chunk, logger) + r.collectSnapshot(chunk, logger) } -func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := m.activeSnapshot +func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := r.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } @@ -193,10 +193,10 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } -func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - m.activeSnapshot = nil + r.activeSnapshot = nil return nil default: } @@ -205,7 +205,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - m.activeSnapshot = nil + r.activeSnapshot = nil return nil default: } @@ -213,7 +213,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -229,34 +229,34 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot // a.applyTargets // Replay deferred events - for _, event := range m.deferredEvents { + for _, event := range r.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := m.applyEvent(ctx, event, logger); err != nil { + if err := r.applyEvent(ctx, event, logger); err != nil { return err } } - m.activeSnapshot = nil - m.deferredEvents = nil + r.activeSnapshot = nil + r.deferredEvents = nil return nil } -func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if m.activeSnapshot != nil { - m.deferredEvents = append(m.deferredEvents, event) + if r.activeSnapshot != nil { + r.deferredEvents = append(r.deferredEvents, event) return nil } // Apply events - return m.applyEvent(ctx, event, logger) + return r.applyEvent(ctx, event, logger) } -func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventCreate: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 84f9a6f..65a4cf9 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target reconciler instance - targetReconciler := discovery.NewMessageProcessor( + targetReconciler := discovery.NewTargetReconciler( r.Client, r.Scheme, targetSource, From c268808d67eb8df1d7328c0658b36bd369eda489 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:38:23 -0600 Subject: [PATCH 111/246] moved registry.go to discovery --- internal/controller/discovery/{registry => }/registry.go | 2 +- internal/controller/targetsource_controller.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{registry => }/registry.go (97%) diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry.go similarity index 97% rename from internal/controller/discovery/registry/registry.go rename to internal/controller/discovery/registry.go index f2630e8..0afa2b2 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry.go @@ -1,4 +1,4 @@ -package registry +package discovery import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 65a4cf9..3b62b6d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -33,7 +33,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/gnmic/operator/internal/controller/discovery/pipeline" - "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -66,7 +65,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete From 02958966b77f80ee3fc1f0e447b98967e54e9c2a Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:39:32 -0600 Subject: [PATCH 112/246] moved supervisor to discovery --- .../controller/discovery/{pipeline => }/supervisor.go | 2 +- internal/controller/targetsource_controller.go | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) rename internal/controller/discovery/{pipeline => }/supervisor.go (99%) diff --git a/internal/controller/discovery/pipeline/supervisor.go b/internal/controller/discovery/supervisor.go similarity index 99% rename from internal/controller/discovery/pipeline/supervisor.go rename to internal/controller/discovery/supervisor.go index 042d305..56fa687 100644 --- a/internal/controller/discovery/pipeline/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -1,4 +1,4 @@ -package pipeline +package discovery import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 3b62b6d..301e421 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -32,7 +32,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" - "github.com/gnmic/operator/internal/controller/discovery/pipeline" "github.com/go-logr/logr" ) @@ -171,7 +170,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - supervisor := pipeline.NewSupervisor(context.Background()) + supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ @@ -190,9 +189,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target reconciler reconcilerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "target-reconciler", - Policy: pipeline.RestartPolicy{ + Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, @@ -221,9 +220,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "loader", - Policy: pipeline.RestartPolicy{ + Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, From 4d32c40fb2e319fa2ff77a9c05f576ba6e0dba4d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:40:26 -0600 Subject: [PATCH 113/246] moved factory.go to discovery/loaders.go --- .../controller/discovery/{loaders/factory.go => loaders.go} | 2 +- internal/controller/targetsource_controller.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{loaders/factory.go => loaders.go} (97%) diff --git a/internal/controller/discovery/loaders/factory.go b/internal/controller/discovery/loaders.go similarity index 97% rename from internal/controller/discovery/loaders/factory.go rename to internal/controller/discovery/loaders.go index 45bf9c1..0d8ddd3 100644 --- a/internal/controller/discovery/loaders/factory.go +++ b/internal/controller/discovery/loaders.go @@ -1,4 +1,4 @@ -package loaders +package discovery import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 301e421..9ba2c94 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -31,7 +31,6 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/go-logr/logr" ) @@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName // Create loader instance if loaderConfigured { - loader, err := loaders.NewLoader( + loader, err := discovery.NewLoader( key, targetSource.Spec, core.LoaderConfig{ChunkSize: r.ChunkSize}, From 7671c1a20aa7a48a26cf306c55ef0698c1ec448f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:54:58 -0600 Subject: [PATCH 114/246] moved send.go to loaders package --- .../discovery/loaders/http/loader.go | 3 ++- .../discovery/{core => loaders}/send.go | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) rename internal/controller/discovery/{core => loaders}/send.go (67%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 09bb7d6..1e5fc37 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -10,6 +10,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/google/uuid" ) @@ -66,7 +67,7 @@ func (l *Loader) Start( }, } - if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/core/send.go b/internal/controller/discovery/loaders/send.go similarity index 67% rename from internal/controller/discovery/core/send.go rename to internal/controller/discovery/loaders/send.go index f24b50c..1377432 100644 --- a/internal/controller/discovery/core/send.go +++ b/internal/controller/discovery/loaders/send.go @@ -1,12 +1,14 @@ -package core +package loaders import ( "context" "fmt" + + "github.com/gnmic/operator/internal/controller/discovery/core" ) // sendMessages sends discovery messages over a channel in a context-aware manner -func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error { +func sendMessages(ctx context.Context, out chan<- []core.DiscoveryMessage, messages []core.DiscoveryMessage) error { select { case <-ctx.Done(): return ctx.Err() @@ -30,14 +32,14 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { } // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots -func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { - var snapshots []DiscoverySnapshot +func createDiscoverySnapshots(targets []core.DiscoveredTarget, snapshotID string, chunkSize int) []core.DiscoverySnapshot { + var snapshots []core.DiscoverySnapshot totalTargets := len(targets) totalChunks := (totalTargets + chunkSize - 1) / chunkSize _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] - snapshots = append(snapshots, DiscoverySnapshot{ + snapshots = append(snapshots, core.DiscoverySnapshot{ Targets: chunk, SnapshotID: snapshotID, ChunkIndex: i / chunkSize, @@ -50,7 +52,7 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu } // SendSnapshot sends discovered targets as a snapshot over a channel in chunks -func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { +func SendSnapshot(ctx context.Context, out chan<- []core.DiscoveryMessage, targets []core.DiscoveredTarget, snapshotID string, chunkSize int) error { if len(targets) == 0 { return fmt.Errorf("no targets in Snapshot") } @@ -58,7 +60,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { // Convert DiscoverySnapshot to DiscoveryMessage - messages := make([]DiscoveryMessage, 1) + messages := make([]core.DiscoveryMessage, 1) messages[0] = snapshot if err := sendMessages(ctx, out, messages); err != nil { @@ -69,8 +71,8 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] return nil } -func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { - message := make([]DiscoveryMessage, len(events)) +func eventsToMessages(events []core.DiscoveryEvent) []core.DiscoveryMessage { + message := make([]core.DiscoveryMessage, len(events)) for i, event := range events { message[i] = event } @@ -78,7 +80,7 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { } // SendEvents sends discovery messages over channel in a context-aware manner -func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { +func SendEvents(ctx context.Context, out chan<- []core.DiscoveryMessage, events []core.DiscoveryEvent, chunkSize int) error { if len(events) == 0 { return fmt.Errorf("no events to process") } From 5f1e9cbe91d28e837ff7fbfae4029df45f27c001 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:55:59 -0600 Subject: [PATCH 115/246] eliminated message.go --- internal/controller/discovery/core/message.go | 4 ---- internal/controller/discovery/core/message_interface.go | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 internal/controller/discovery/core/message.go diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go deleted file mode 100644 index af4f6c1..0000000 --- a/internal/controller/discovery/core/message.go +++ /dev/null @@ -1,4 +0,0 @@ -package core - -func (DiscoveryEvent) isDiscoveryMessage() {} -func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go index 07b819e..0836bc6 100644 --- a/internal/controller/discovery/core/message_interface.go +++ b/internal/controller/discovery/core/message_interface.go @@ -3,3 +3,6 @@ package core type DiscoveryMessage interface { isDiscoveryMessage() } + +func (DiscoveryEvent) isDiscoveryMessage() {} +func (DiscoverySnapshot) isDiscoveryMessage() {} From 6d6753731ca36cdafa5a251e164ed1b70eafd3dc Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:56:39 -0600 Subject: [PATCH 116/246] moved const.go to discovery.go --- internal/controller/discovery/client.go | 3 +-- internal/controller/discovery/{core => }/const.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{core => }/const.go (81%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index 2deb477..cb02161 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -6,7 +6,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" ) func fetchExistingTargets( @@ -22,7 +21,7 @@ func fetchExistingTargets( &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ - core.LabelTargetSourceName: ts.Name, + LabelTargetSourceName: ts.Name, }, ) if err != nil { diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/const.go similarity index 81% rename from internal/controller/discovery/core/const.go rename to internal/controller/discovery/const.go index 82a5962..ac7a57f 100644 --- a/internal/controller/discovery/core/const.go +++ b/internal/controller/discovery/const.go @@ -1,4 +1,4 @@ -package core +package discovery const ( // Labels From 391463097c6caab4b89c72de9789efe8b346e8bf Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:28:29 -0600 Subject: [PATCH 117/246] renamed core package within targetsource controller --- internal/controller/targetsource_controller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9ba2c94..e52b02b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -30,7 +30,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" - "github.com/gnmic/operator/internal/controller/discovery/core" + discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/go-logr/logr" ) @@ -63,7 +63,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -171,8 +171,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName supervisor := discovery.NewSupervisor(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ + targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) + if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ Channel: targetChannel, WebhookEnabled: webhookActivated, }); err != nil { @@ -212,7 +212,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loader, err := discovery.NewLoader( key, targetSource.Spec, - core.LoaderConfig{ChunkSize: r.ChunkSize}, + discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize}, ) if err != nil { supervisor.Stop() From 46a201fc1d9f0dc9cc73825477f789fc3cb3e860 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:34:42 -0600 Subject: [PATCH 118/246] changed events to delete / apply --- internal/controller/discovery/core/types.go | 6 ++---- internal/controller/discovery/target_reconciler.go | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 68c9c7e..2c37fc7 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -15,10 +15,8 @@ type EventAction int const ( // EventDelete indicates that a target should be removed EventDelete EventAction = iota - // EventCreate indicates that a target should be created - EventCreate - // EventUpdate indicates that a target should be updated - EventUpdate + // EventApply indicates that a target should be applied (created or updated) + EventApply ) // DiscoveredTarget represents a target discovered from an external source diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 4f3711c..86470c6 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -258,12 +258,10 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { - case core.EventCreate: - logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.EventUpdate: - logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) case core.EventDelete: logger.Info("Would delete target", "name", event.Target.Name) + case core.EventApply: + logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) } return nil } From 7b17f7e77644abff70f5796704e36b10bf03da15 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:37:39 -0600 Subject: [PATCH 119/246] moved send.go into separate utils for loaders --- internal/controller/discovery/loaders/http/loader.go | 4 ++-- internal/controller/discovery/loaders/{ => utils}/send.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename internal/controller/discovery/loaders/{ => utils}/send.go (99%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 1e5fc37..d7d5961 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -10,7 +10,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders" + loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" "github.com/google/uuid" ) @@ -67,7 +67,7 @@ func (l *Loader) Start( }, } - if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/loaders/send.go b/internal/controller/discovery/loaders/utils/send.go similarity index 99% rename from internal/controller/discovery/loaders/send.go rename to internal/controller/discovery/loaders/utils/send.go index 1377432..3cfba8d 100644 --- a/internal/controller/discovery/loaders/send.go +++ b/internal/controller/discovery/loaders/utils/send.go @@ -1,4 +1,4 @@ -package loaders +package utils import ( "context" From 4540163d4137a27a291846a5960ecf09844bf5f8 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:45:43 -0600 Subject: [PATCH 120/246] replaced legacy registry package --- cmd/main.go | 4 ++-- internal/apiserver/apiserver.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4cf6e94..aaf398a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,7 +42,7 @@ import ( "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/gnmic/operator/internal/controller/discovery" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -86,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() + discoveryRegistry := discovery.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index a7ca16a..705b277 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,7 @@ import ( "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/gnmic/operator/internal/controller/discovery" "k8s.io/apimachinery/pkg/types" ) @@ -14,7 +14,7 @@ type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { From 183abe226c6b3a7f39967a5c299ebc073b323094 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 14:30:09 -0600 Subject: [PATCH 121/246] moved client/mapper functions out of target_reconciler.go --- internal/controller/discovery/client.go | 51 +++++++++++++++-- internal/controller/discovery/mapper.go | 5 ++ .../controller/discovery/target_reconciler.go | 56 ++----------------- 3 files changed, 54 insertions(+), 58 deletions(-) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index cb02161..a9d790f 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -3,17 +3,17 @@ package discovery import ( "context" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" ) -func fetchExistingTargets( - ctx context.Context, - c client.Client, - ts *gnmicv1alpha1.TargetSource, -) ([]gnmicv1alpha1.Target, error) { - +func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { var targetList gnmicv1alpha1.TargetList err := c.List( @@ -30,3 +30,42 @@ func fetchExistingTargets( return targetList.Items, nil } + +func applyTarget(ctx context.Context, c client.Client, s *runtime.Scheme, desired *gnmicv1alpha1.Target, ts *gnmicv1alpha1.TargetSource) error { + existing := &gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: desired.Name, + Namespace: desired.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, c, existing, func() error { + existing.Spec = desired.Spec + existing.Labels = desired.Labels + + return controllerutil.SetControllerReference(ts, existing, s) + }) + + return err +} + +func deleteTarget(ctx context.Context, c client.Client, name string, namespace string) error { + existing := &gnmicv1alpha1.Target{} + + err := c.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, existing) + if apierrors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + + err = c.Delete(ctx, existing) + if apierrors.IsNotFound(err) { + return nil + } + + return err +} diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 89a081f..f34fe36 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -73,3 +73,8 @@ func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.Discovere return events } + +func normalizeTarget(t core.DiscoveredTarget, tsName string) core.DiscoveredTarget { + t.Name = tsName + "-" + t.Name + return t +} diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index a924132..e8e24b5 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -4,12 +4,8 @@ import ( "context" "fmt" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -113,7 +109,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc ) for i := range msg.Targets { - msg.Targets[i] = r.normalizeTarget(msg.Targets[i]) + msg.Targets[i] = normalizeTarget(msg.Targets[i], r.targetSource.Namespace) } return r.processSnapshot(ctx, msg, logger) @@ -125,7 +121,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc "target", msg.Target.Name, ) - msg.Target = r.normalizeTarget(msg.Target) + msg.Target = normalizeTarget(msg.Target, r.targetSource.Namespace) return r.processEvent(ctx, msg, logger) default: @@ -299,7 +295,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: - if err := r.deleteTarget(ctx, event.Target.Name); err != nil { + if err := deleteTarget(ctx, r.client, event.Target.Name, r.targetSource.Namespace); err != nil { logger.Error(err, "error deleting target", "targetName", event.Target.Name, ) @@ -311,7 +307,7 @@ func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryE case core.EventApply: target := generateTargetResource(event.Target, r.targetSource) - if err := r.applyTarget(ctx, target); err != nil { + if err := applyTarget(ctx, r.client, r.scheme, target, r.targetSource); err != nil { logger.Error(err, "error applying target", "targetName", event.Target.Name, ) @@ -324,47 +320,3 @@ func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryE return nil } - -func (r *TargetReconciler) applyTarget(ctx context.Context, desired *gnmicv1alpha1.Target) error { - existing := &gnmicv1alpha1.Target{ - ObjectMeta: metav1.ObjectMeta{ - Name: desired.Name, - Namespace: desired.Namespace, - }, - } - - _, err := controllerutil.CreateOrUpdate(ctx, r.client, existing, func() error { - existing.Spec = desired.Spec - existing.Labels = desired.Labels - - return controllerutil.SetControllerReference(r.targetSource, existing, r.scheme) - }) - - return err -} - -func (r *TargetReconciler) deleteTarget(ctx context.Context, name string) error { - existing := &gnmicv1alpha1.Target{} - - err := r.client.Get(ctx, types.NamespacedName{ - Name: name, - Namespace: r.targetSource.Namespace, - }, existing) - if apierrors.IsNotFound(err) { - return nil - } else if err != nil { - return err - } - - err = r.client.Delete(ctx, existing) - if apierrors.IsNotFound(err) { - return nil - } - - return err -} - -func (r *TargetReconciler) normalizeTarget(t core.DiscoveredTarget) core.DiscoveredTarget { - t.Name = r.targetSource.Name + "-" + t.Name - return t -} From 0811afd19a5da16c71962ab62be8b047a71f7f0d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 14:54:09 -0600 Subject: [PATCH 122/246] renamed functions --- .../controller/discovery/target_reconciler.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index e8e24b5..34158c7 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -112,7 +112,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc msg.Targets[i] = normalizeTarget(msg.Targets[i], r.targetSource.Namespace) } - return r.processSnapshot(ctx, msg, logger) + return r.handleSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -122,15 +122,15 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc ) msg.Target = normalizeTarget(msg.Target, r.targetSource.Namespace) - return r.processEvent(ctx, msg, logger) + return r.handleEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) } } -// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { +// handleSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly +func (r *TargetReconciler) handleSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { if r.activeSnapshot == nil { r.startNewSnapshot(chunk, logger) return nil @@ -141,7 +141,7 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := r.applySnapshot(ctx, snapshot, logger); err != nil { + if err := r.reconcileSnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -200,7 +200,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } -func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) handleEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events if r.activeSnapshot != nil { r.deferredEvents = append(r.deferredEvents, event) @@ -208,10 +208,10 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover } // Apply events - return r.applyEvent(ctx, event, logger) + return r.reconcileEvent(ctx, event, logger) } -func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (r *TargetReconciler) reconcileSnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): r.activeSnapshot = nil @@ -272,7 +272,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot ) for _, e := range events { - r.processEvent(ctx, e, logger) + r.handleEvent(ctx, e, logger) } // Replay deferred events @@ -282,7 +282,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot return nil default: } - if err := r.applyEvent(ctx, event, logger); err != nil { + if err := r.reconcileEvent(ctx, event, logger); err != nil { return err } } @@ -292,7 +292,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot return nil } -func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) reconcileEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: if err := deleteTarget(ctx, r.client, event.Target.Name, r.targetSource.Namespace); err != nil { From c728fa2f340066c1f261769ab379ba223e12d62c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:21:11 +0000 Subject: [PATCH 123/246] add supervisor restart policy to targetsource spec configuration --- api/v1alpha1/targetsource_types.go | 13 ++++- api/v1alpha1/zz_generated.deepcopy.go | 30 ++++++++++++ .../operator.gnmic.dev_targetsources.yaml | 7 +++ internal/controller/discovery/defaults.go | 12 +++++ .../controller/targetsource_controller.go | 49 ++++++++++++------- 5 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 internal/controller/discovery/defaults.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index a936e66..7c8f74c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -26,9 +26,12 @@ type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` // +kubebuilder:validation:Optional Webhook WebhookSpec `json:"webhook,omitempty"` - // + // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` + // +kubebuilder:validation:Optional + RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"` + // +kubebuilder:validation:MinLength=1 TargetProfile string `json:"targetProfile"` } @@ -54,6 +57,14 @@ type ConsulConfig struct { URL string `json:"url,omitempty"` } +type RestartPolicySpec struct { + // +kubebuilder:validation:Optional + MaxRestarts *int `json:"maxRestarts,omitempty"` + + // +kubebuilder:validation:Optional + BackoffSeconds *int `json:"backoffSeconds,omitempty"` +} + // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { Status string `json:"status"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..df08573 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -843,6 +843,31 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) { + *out = *in + if in.MaxRestarts != nil { + in, out := &in.MaxRestarts, &out.MaxRestarts + *out = new(int) + **out = **in + } + if in.BackoffSeconds != nil { + in, out := &in.BackoffSeconds, &out.BackoffSeconds + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec. +func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec { + if in == nil { + return nil + } + out := new(RestartPolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -1300,6 +1325,11 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { (*out)[key] = val } } + if in.RestartPolicy != nil { + in, out := &in.RestartPolicy, &out.RestartPolicy + *out = new(RestartPolicySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec. diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index b385d8e..6464ea2 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,6 +60,13 @@ spec: - message: exactly one of the fields in [http consul] must be set rule: '[has(self.http),has(self.consul)].filter(x,x==true).size() == 1' + restartPolicy: + properties: + backoffSeconds: + type: integer + maxRestarts: + type: integer + type: object targetLabels: additionalProperties: type: string diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go new file mode 100644 index 0000000..dc6f046 --- /dev/null +++ b/internal/controller/discovery/defaults.go @@ -0,0 +1,12 @@ +package discovery + +import "time" + +// DefaultRestartPolicy defines the default restart behavior +// for the discovery components +func DefaultRestartPolicy() RestartPolicy { + return RestartPolicy{ + MaxRestarts: 5, + Backoff: 3 * time.Second, + } +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 06b4fac..fddebda 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -35,11 +35,6 @@ import ( "github.com/go-logr/logr" ) -const ( - pipelineMaxRestarts = 5 - pipelineBackoff = 3 * time.Second -) - // pipelineHandle represents a controller-owned handle to a running pipeline // The controller never manipulates internals; it only invokes cancel() type pipelineHandle struct { @@ -158,6 +153,29 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } +// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy +func resolveRestartPolicy( + override *gnmicv1alpha1.RestartPolicySpec, +) discovery.RestartPolicy { + defaults := discovery.DefaultRestartPolicy() + + if override == nil { + return defaults + } + + resolved := defaults + + if override.MaxRestarts != nil { + resolved.MaxRestarts = *override.MaxRestarts + } + + if override.BackoffSeconds != nil { + resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second + } + + return resolved +} + // startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource // // Pipeline semantics: @@ -168,6 +186,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) supervisor := discovery.NewSupervisor(context.Background()) @@ -187,22 +206,19 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetChannel, ) // Start target reconciler - reconcilerReady := make(chan struct{}) + targetReconcilerReady := make(chan struct{}) supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-reconciler", - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, + Name: "target-reconciler", + Policy: restartPolicy, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(reconcilerReady) // Signals that reconciler started successfully + close(targetReconcilerReady) // Signals that reconciler started successfully return targetReconciler.Run(ctx) }, }) // Wait for reconciler to be ready before starting loader select { - case <-reconcilerReady: + case <-targetReconcilerReady: case <-supervisor.Done(): return nil } @@ -220,11 +236,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "loader", - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, + Name: "loader", + Policy: restartPolicy, EscalatesOnFailure: !webhookActivated, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) From 0ff9bdfac9de6e41f4203fc46c4b1ebf54308ce9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 30 Apr 2026 08:29:31 +0000 Subject: [PATCH 124/246] tags wrapped in labels --- internal/apiserver/apiserver.go | 16 ++++++++++++++-- internal/apiserver/gen.go | 24 +++++++++++++++--------- internal/apiserver/openapi.yaml | 13 ++++++++++--- internal/apiserver/temp.md | 8 ++++++-- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b6f04e8..34c09d5 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -118,16 +118,28 @@ func (a *APIServer) CreateTargets(c *gin.Context) { logger.Warn("Received invalid Operation flag") } + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, Address: target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, + Labels: labelToMap, }, Event: event, }) } } + + fmt.Printf("core.DiscoveryEvent was created: %v", targets) core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } @@ -140,4 +152,4 @@ func parseURI(c *gin.Context) (url urlStruct) { return } return u -} \ No newline at end of file +} diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 11e23a2..07a82c2 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -37,13 +37,19 @@ func (e TargetOperation) Valid() bool { } } +// Tag defines model for Tag. +type Tag struct { + Key *string `json:"key,omitempty"` + Value *string `json:"value,omitempty"` +} + // Target defines model for Target. type Target struct { Address string `json:"address"` + Labels *[]Tag `json:"labels,omitempty"` Name string `json:"name"` Operation TargetOperation `json:"operation"` Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` } // TargetOperation defines model for Target.Operation. @@ -135,14 +141,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTzYrbMBDHX0VMezSxd3vzrSxlCbTbpdlb6UGVJokWW1JHo0JY/O5FkpO1Ywd60iDN", - "x39+M3oD5XrvLFoO0L5BUEfsZTZfJB2Qk+XJeSQ2mO+l1oQhm3zyCC0EJmMPMFRgZY+rDymBZONsekUb", - "e2h/giKUjBoqiF6PlsYOk/WrWibx5PamWy/A8pAlGcZ+Xdt4IYnkCYahAsI/0RDqJCULry69TQW/K3G/", - "X1FxylXYhCWc8vDVBJ6J+Ui4hxY+1O+06xF1PXJeKDxX2blICp9ukeUrp52Xas3zquH1sJWa1bSnJYuU", - "19i9yxUNp+nA4enbVonvmaAj8ePL7kV8ft5CBX+RQl4CaDbN5m7cDCu9gRY+be43DVTgJR8ztbosyBS2", - "K2Av09lqaOFh5pb6DN7ZUCZy39ylQznLaHOw9L4zKofXr6GsZBnGrXn+5wzDgvL5fp2bxqDI+PItzr5i", - "/BUiRKUwhH3surKvIfa9pNOlYcFjhLGCjyjm2HNI7TuZ+xt/8pzbI/JDFwMjPSe3BbgmHXORE39ByJEs", - "6itxj8hCFTeRyw/DMPwLAAD//2Qu5BBoBAAA", + "H4sIAAAAAAAC/5STz2vbMBTH/xXxtqOJ3e7m2yijFLauLLmNHlT5JVEnS9rTU8EU/+9DkpMmtQPdSc96", + "vz9f+RWU672zaDlA+wpB7bGX2dzIXTo8OY/EGvPlHxzSwYNHaCEwabuDsYIXaSIueMbqcOOenlFxit1I", + "2iHPa8uuIwxhsb6RT2iySzP22fhMuIUWPtVvG9TT+HWa/a21JJJD+rayx8XyaQzJ2tnkRRt7aH+DIpSM", + "HVQQfTdZHRpM1mM1L+LJbbW5QIHwb9SEXSqcx6iO+562f7zIK8yBFcd3Hfg/wGT2C2yKZ+0iKby/xInf", + "Ba29VB9YeDltoWd1utOcRaqr7dbljpoTa9jd/7hT4mcm6Ej8+rbeiK8Pd1DBC1LIkkKzalZXk85Weg0t", + "fFldrxqowEveZ2p1kfsUtitgj+rcddDCzVlY2jN4Z0NR5Lq5SodyltHmZOm90Sqn18+hPLAixiU9P6hh", + "mFE+3C9z6zAo0r488kOsmN64CFEpDGEbjRly4RD7XtJwXFjwlKGt4D2Kc+w5pfZG5v2mv/uc2y3yjYmB", + "kR5S2Axck47zIU/iBSFHsti9G+4WWagSJnL7cRzHfwEAAP//JDP+7tUEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 5accf2b..093c1a4 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -56,13 +56,20 @@ components: type: string # in the format "IP:port" profile: type: string - tags: + labels: type: array items: - type: string + $ref: '#/components/schemas/Tag' operation: type: string enum: - created - updated - - deleted \ No newline at end of file + - deleted + Tag: + type: object + properties: + key: + type: string + value: + type: string \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 715ef09..c70f463 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' @@ -23,8 +23,12 @@ curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" "name": "{{ data.name }}", "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", "profile": "{{ data.custom_fields.profile | default('') }}", - "tags": [{% for tag in data.tags %}"{{ tag.name }}"{% if not loop.last %}, {% endif %}{% endfor %}], + "labels": [{ + "key": "tags", + "value": "{{ data.tags | map(attribute='name') | join(', ') }}" + }, "operation": "{{ event }}" + ] } ] } \ No newline at end of file From 589bc9f8cf0643af82f40c4e126ec2e72fc7e67e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:31:37 +0000 Subject: [PATCH 125/246] add targetsource example for lab --- lab/dev/resources/targetsources/cts1.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 lab/dev/resources/targetsources/cts1.yml diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml new file mode 100644 index 0000000..682930c --- /dev/null +++ b/lab/dev/resources/targetsources/cts1.yml @@ -0,0 +1,18 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: http-discovery +spec: + provider: + http: + url: http://srbsci-121:8081/api/dcim/devices/?export=test + webhook: + enabled: true + targetLabels: + source: inventory + site: siteA + tags: "inventory,siteA,http-discovery" + restartPolicy: + maxRestarts: 2 + backoffSeconds: 4 + targetProfile: eos \ No newline at end of file From a5dde06e8df6dafe7a72f5650e35584ef22b2662 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:37:04 +0000 Subject: [PATCH 126/246] remove targetsource example to not add unnecassary logging to main --- lab/dev/resources/targetsources/cts1.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 lab/dev/resources/targetsources/cts1.yml diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml deleted file mode 100644 index 682930c..0000000 --- a/lab/dev/resources/targetsources/cts1.yml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: http-discovery -spec: - provider: - http: - url: http://srbsci-121:8081/api/dcim/devices/?export=test - webhook: - enabled: true - targetLabels: - source: inventory - site: siteA - tags: "inventory,siteA,http-discovery" - restartPolicy: - maxRestarts: 2 - backoffSeconds: 4 - targetProfile: eos \ No newline at end of file From 4be9c27a8a547ded1d79f9d6a542da2ad148fe2b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:42:06 +0000 Subject: [PATCH 127/246] update gitignore to not push targetsources in order to prevent logging in main branch --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 29d31af..7515fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,9 @@ notes/ docs/public docs/resources/_gen/ docs/.hugo_build.lock -test/integration/clab-* \ No newline at end of file +test/integration/clab-* + +# Only for development and testing purposes +# To be removed after development of targetsource +# ignored in order to not add unnecassary logging messages +lab/dev/resources/targetsources \ No newline at end of file From 7962d775ad2e71dece21147e9b9b4dba25104b19 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 30 Apr 2026 09:36:46 +0000 Subject: [PATCH 128/246] refactor --- internal/apiserver/apiserver.go | 100 +++++++++++++++++--------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 34c09d5..8c01486 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,6 @@ package apiserver import ( "context" - "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -62,42 +61,33 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { logger.Info("received POST request for CreateTargets.") + var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware - // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation - // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. - if payloadTargets.TargetSourceNameSpace == "" { - logger.Error("POST request does not contain value targetSourceNameSpace.") - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) - return - } - if payloadTargets.TargetSourceName == "" { - logger.Error("POST request does not contain value targetSourceName.") - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) - return - } - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) + ch, ok := a.DiscoveryRegistry.Get(getKey(payloadTargets)) if !ok { logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) return } - fmt.Printf("payloadTargets %+v\n", payloadTargets) + targets := createDiscoveryEvent(payloadTargets) + // fmt.Printf("core.DiscoveryEvent was created: %v", targets) + core.SendEvents(context.Background(), ch, targets, a.chunkSize) + c.JSON(http.StatusOK, payloadTargets) +} +// createDiscoveryEvent creates object of type core.DiscoveryEvent +func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { + // no REST API return here as not all targets might logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") break } @@ -106,42 +96,56 @@ func (a *APIServer) CreateTargets(c *gin.Context) { break } - event := core.CREATE - switch target.Operation { - case Created: - event = core.UPDATE - case Updated: - event = core.UPDATE - case Deleted: - event = core.DELETE - default: - logger.Warn("Received invalid Operation flag") - } - - labelToMap := make(map[string]string) - if target.Labels != nil { - for _, tag := range *target.Labels { - if tag.Key == nil || tag.Value == nil || *tag.Key == "" { - continue - } - labelToMap[*tag.Key] = *tag.Value - } - } - targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, Address: target.Address, - Labels: labelToMap, + Labels: convertTargetLabelsToMap(target), }, - Event: event, + Event: getEvent(target), }) } } + return targets +} - fmt.Printf("core.DiscoveryEvent was created: %v", targets) - core.SendEvents(context.Background(), ch, targets, a.chunkSize) - c.JSON(http.StatusOK, payloadTargets) +// getKey returns key for used to identify correct channel in DiscoveryRegistry +func getKey(payloadTargets Targets) types.NamespacedName { + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + return key +} + +// convertTargetLabelsToMap converts target.Labels to map. +func convertTargetLabelsToMap(target Target) map[string]string { + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + return labelToMap +} + +// getEvent converts target.Operation to core.Operation. +func getEvent(target Target) core.EventAction { + event := core.CREATE + switch target.Operation { + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: + event = core.DELETE + default: + logger.Warn("Received invalid Operation flag") + } + return event } // parseURI parses URI to urlStruct. From 7337541e70e7bbf0867eb2a1e66a7c6ffacc3799 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 09:56:06 +0000 Subject: [PATCH 129/246] add component info to logging --- internal/controller/discovery/target_reconciler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 86470c6..3a9f327 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -50,6 +50,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error { logger := log.FromContext(r.ctx). WithValues( + "component", "target reconciler", "name", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) From 41d54987415e9dc7c31096b12eff8e9022680405 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 10:02:20 +0000 Subject: [PATCH 130/246] make snapshot id a bit smaller --- internal/controller/discovery/loaders/http/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index bc87855..84cb70b 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -77,7 +77,7 @@ func (l *Loader) Start( return } - snapshotID := fmt.Sprintf("snapshot-%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString()) + snapshotID := fmt.Sprintf("%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString()) if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { logger.Error(err, "failed to send discovery snapshot") return From 3ec3203efa7a6e484902ba17dd4eed9085c52277 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:21:19 +0000 Subject: [PATCH 131/246] if context is canceled return with ctx.Err() not a clean exit --- internal/controller/discovery/target_reconciler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 3a9f327..2f623c3 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -73,7 +73,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error { for len(r.queue) > 0 { if ctx.Err() != nil { - return nil // why return nil? + return ctx.Err() } msg := r.queue[0] From 0eaffdcfc63c32bd6d63e46f3081a12015bd76e4 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:34:18 +0000 Subject: [PATCH 132/246] applied kubebuilder best-practise logging --- .../discovery/loaders/http/loader.go | 6 +- internal/controller/discovery/supervisor.go | 22 ++++--- .../controller/discovery/target_reconciler.go | 59 ++++++++++++++----- .../controller/targetsource_controller.go | 31 ++++++++-- 4 files changed, 89 insertions(+), 29 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index d7d5961..67c61e1 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -39,7 +39,11 @@ func (l *Loader) Start( "targetsource", targetsourceNN, ) - logger.Info("HTTP loader started") + logger.Info( + "HTTP loader started", + "targetsource", targetsourceNN.Name, + "namespace", targetsourceNN.Namespace, + ) // Only for debugging: emit a static snapshot every 30 seconds ticker := time.NewTicker(30 * time.Second) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index 56fa687..22ec227 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -78,7 +78,10 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { failures := 0 for { - logger.Info("starting component") + logger.Info( + "Starting supervised component", + "component", component.Name, + ) err := component.Run(s.ctx) if s.ctx.Err() != nil { @@ -87,21 +90,26 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { } failures++ - logger.Error(err, - "component failed to run", + logger.Error( + err, + "Supervised component failed", + "component", component.Name, "attempt", failures, - "max", component.Policy.MaxRestarts, + "maxRestarts", component.Policy.MaxRestarts, ) if failures >= component.Policy.MaxRestarts { if component.EscalatesOnFailure { - logger.Error(err, - "component permanently failed; shutting down pipeline", + logger.Error( + err, + "Supervised component permanently failed; stopped discovery pipeline", + "component", component.Name, ) s.Stop() } else { logger.Info( - "optional component permanently failed; continuing without it", + "Optional component permanently failed; continuing without it", + "component", component.Name, ) } return diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 2f623c3..67d9611 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -54,20 +54,30 @@ func (r *TargetReconciler) Run(ctx context.Context) error { "name", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) - logger.Info("target reconciler started") + logger.Info( + "Target reconciler started", + "targetsource", r.targetSource.Name, + "namespace", r.targetSource.Namespace, + ) for r.ctx.Err() == nil { select { case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target reconciler") + logger.Info( + "Input channel closed; stopping target reconciler", + "targetsource", r.targetSource.Name, + ) return nil } r.queue = append(r.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target reconciler") + logger.Info( + "Context was canceled; stopping target reconciler", + "targetsource", r.targetSource.Name, + ) return nil } @@ -103,23 +113,24 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc case core.DiscoverySnapshot: // Collect snapshot chunks logger.Info( - "received snapshot chunk", + "Received discovery snapshot chunk", "snapshotID", msg.SnapshotID, - "index", msg.ChunkIndex, - "targetCount", len(msg.Targets), + "chunkIndex", msg.ChunkIndex, + "targets", len(msg.Targets), ) return r.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update logger.Info( - "received discovery event", + "Received discovery event", + "event", msg.Event, "target", msg.Target.Name, ) return r.processEvent(ctx, msg, logger) default: - return fmt.Errorf("unknonw discovery message type %T", msg) + return fmt.Errorf("Unknown discovery message type %T", msg) } } @@ -142,7 +153,7 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco // If a new snapshot is started before the old one completed // the old one can be discarded logger.Info( - "discarding incomplete snapshot", + "Discarded incomplete discovery snapshot", "snapshotID", snapshot.snapshotID, ) } @@ -172,7 +183,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger snapshot := r.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { - logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) + logger.Error( + nil, + "Snapshot totalChunks mismatch", + "snapshotID", snapshot.snapshotID, + ) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) @@ -180,7 +195,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { - logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) + logger.Error( + nil, + "Duplicate snapshot chunk received", + "chunkIndex", chunk.ChunkIndex, + ) r.activeSnapshot = nil return nil } @@ -221,9 +240,9 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot } logger.Info( - "applying snapshot", + "Applying discovery snapshot", "snapshotID", snapshot.snapshotID, - "targetCount", len(allTargets), + "targets", len(allTargets), ) // apply all targets @@ -260,9 +279,19 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: - logger.Info("Would delete target", "name", event.Target.Name) + logger.Info( + "Deleting Target", + "target", event.Target.Name, + "targetsource", r.targetSource.Name, + ) case core.EventApply: - logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + logger.Info( + "Applying Target", + "target", event.Target.Name, + "address", event.Target.Address, + "labels", event.Target.Labels, + "targetsource", r.targetSource.Name, + ) } return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index fddebda..c82ad08 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -71,13 +71,20 @@ type TargetSourceReconciler struct { // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx). - WithName("targetsource controller"). - WithValues("targetsource", req.NamespacedName) + WithName("targetsource-controller"). + WithValues( + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - logger.Info("TargetSource not found; stopping discovery pipeline") + logger.Info( + "TargetSource not found; stopped discovery pipeline", + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { @@ -100,7 +107,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info("Discovery pipeline started") + logger.Info( + "Started discovery pipeline", + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) return ctrl.Result{}, nil } @@ -124,7 +135,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) + logger.Info( + "TargetSource was marked for deletion; stopping discovery pipeline", + "targetsource", key.Name, + "namespace", key.Namespace, + ) r.stopDiscoveryPipeline(key) @@ -250,10 +265,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName <-supervisor.Done() supervisor.Wait() // Wait for components to exit - logger.Info("Pipeline stopped; cleaning up") close(targetChannel) r.DiscoveryRegistry.Unregister(key) r.stopDiscoveryPipeline(key) + logger.Info( + "Discovery pipeline stopped; cleaned up resources", + "targetsource", key.Name, + "namespace", key.Namespace, + ) }() r.mu.Lock() From e447b3bd18867bd9d5cd1df19262dec14d0785dd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:36:18 +0000 Subject: [PATCH 133/246] improved logging --- .../controller/discovery/loaders/http/loader.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index c95f1b3..4b58223 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -77,13 +77,20 @@ func (l *Loader) Start( spec.Provider.HTTP.Token, ) if err != nil { - logger.Error(err, "failed to fetch targets from HTTP endpoint") + logger.Error( + err, + "Failed to fetch targets from HTTP endpoint", + "url", spec.Provider.HTTP.URL, + ) return } snapshotID := fmt.Sprintf("%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString()) if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { - logger.Error(err, "failed to send discovery snapshot") + logger.Error( + err, + "Failed to send discovery snapshot", + ) return } } @@ -95,7 +102,11 @@ func (l *Loader) Start( for { select { case <-ctx.Done(): - logger.Info("HTTP loader stopped") + logger.Info( + "HTTP loader stopped", + "targetsource", targetsourceNN.Name, + "namespace", targetsourceNN.Namespace, + ) return nil case <-ticker.C: From fca37e0e87076946731fe0918846d75ab20d0356 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:39:48 +0000 Subject: [PATCH 134/246] improved logging --- .../discovery/loaders/http/loader.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 4b58223..95470fc 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -44,15 +44,12 @@ func (l *Loader) Start( logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", targetsourceNN, - ) - - logger.Info( - "HTTP loader started", "targetsource", targetsourceNN.Name, "namespace", targetsourceNN.Namespace, ) + logger.Info("HTTP loader started") + // Input Validation of spec if spec.Provider == nil || spec.Provider.HTTP == nil { return errors.New("HTTP loader requires spec.provider.http to be set") @@ -66,7 +63,11 @@ func (l *Loader) Start( ticker := time.NewTicker(interval) defer ticker.Stop() - logger.Info("HTTP pull loader started", "interval", interval.String()) + logger.Info( + "HTTP polling discovery started", + "interval", interval.String(), + "url", spec.Provider.HTTP.URL, + ) // helper function to fetch targets and emit discovery messages fetchAndEmit := func() { @@ -90,9 +91,17 @@ func (l *Loader) Start( logger.Error( err, "Failed to send discovery snapshot", + "snapshotID", snapshotID, + "targets", len(targets), ) return } + + logger.Info( + "Discovery snapshot sent", + "snapshotID", snapshotID, + "targets", len(targets), + ) } // Immediate fetch on startup @@ -102,11 +111,7 @@ func (l *Loader) Start( for { select { case <-ctx.Done(): - logger.Info( - "HTTP loader stopped", - "targetsource", targetsourceNN.Name, - "namespace", targetsourceNN.Namespace, - ) + logger.Info("HTTP loader stopped") return nil case <-ticker.C: From fd4abe7f086416c0bd4b52249a03625ac6d72124 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:55:58 +0000 Subject: [PATCH 135/246] improved logging --- .../controller/discovery/target_reconciler.go | 34 +++++++++---------- .../controller/targetsource_controller.go | 29 ++++++++-------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 67d9611..39382ab 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -48,36 +48,26 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T func (r *TargetReconciler) Run(ctx context.Context) error { r.ctx = ctx - logger := log.FromContext(r.ctx). - WithValues( - "component", "target reconciler", - "name", r.targetSource.Name, - "namespace", r.targetSource.Namespace, - ) - logger.Info( - "Target reconciler started", + logger := log.FromContext(ctx).WithValues( + "component", "target-reconciler", "targetsource", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) + logger.Info("Target reconciler started") + for r.ctx.Err() == nil { select { case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info( - "Input channel closed; stopping target reconciler", - "targetsource", r.targetSource.Name, - ) + logger.Info("Input channel closed; stopping target reconciler") return nil } r.queue = append(r.queue, batch...) case <-ctx.Done(): - logger.Info( - "Context was canceled; stopping target reconciler", - "targetsource", r.targetSource.Name, - ) + logger.Info("Context was canceled; stopping target reconciler") return nil } @@ -190,7 +180,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger ) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { - logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) + logger.Error( + nil, + "Snapshot chunk index out of range", + "chunkIndex", chunk.ChunkIndex, + ) r.activeSnapshot = nil return nil } @@ -232,7 +226,11 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot chunk, ok := snapshot.received[i] if !ok { - logger.Error(nil, "missing snapshot chunk", "index", i) + logger.Error( + nil, + "Missing snapshot chunk", + "chunkIndex", i, + ) r.activeSnapshot = nil return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c82ad08..f36d47d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -80,11 +80,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - logger.Info( - "TargetSource not found; stopped discovery pipeline", - "targetsource", req.NamespacedName.Name, - "namespace", req.NamespacedName.Namespace, - ) + logger.Info("TargetSource not found; stopped discovery pipeline") r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { @@ -100,6 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } if r.hasPipelineRunning(req.NamespacedName) { + logger.Info("Discovery pipeline already running; reconciliation completed") return ctrl.Result{}, nil } @@ -107,11 +104,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info( - "Started discovery pipeline", - "targetsource", req.NamespacedName.Name, - "namespace", req.NamespacedName.Namespace, - ) + logger.Info("Started discovery pipeline") return ctrl.Result{}, nil } @@ -134,13 +127,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { - logger := log.FromContext(ctx) - logger.Info( - "TargetSource was marked for deletion; stopping discovery pipeline", + logger := log.FromContext(ctx).WithValues( "targetsource", key.Name, "namespace", key.Namespace, ) - + logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") r.stopDiscoveryPipeline(key) // Remove finalizer if exists @@ -149,6 +140,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type if err := r.Update(ctx, targetSource); err != nil { return ctrl.Result{}, err } + + logger.Info("Removed TargetSource finalizer") } return ctrl.Result{}, nil @@ -165,6 +158,12 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return err } + log.FromContext(ctx).Info( + "Added TargetSource finalizer", + "targetsource", targetSource.Name, + "namespace", targetSource.Namespace, + ) + return nil } @@ -234,7 +233,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName // Wait for reconciler to be ready before starting loader select { case <-targetReconcilerReady: + logger.Info("Target reconciler started") case <-supervisor.Done(): + logger.Info("Supervisor stopped before target reconciler became ready") return nil } From a6bc11447919eac352164b959bcac482c2b0a115 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 13:23:47 +0000 Subject: [PATCH 136/246] simplified pipeline context handling --- internal/controller/discovery/core/types.go | 12 +++- internal/controller/discovery/registry.go | 8 +++ .../controller/targetsource_controller.go | 68 ++++++------------- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 2c37fc7..2f89fdf 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,8 +1,18 @@ package core +import "context" + +// DiscoveryRegistryValue represents the controller-owned runtime state +// of a discovery pipeline for a single TargetSource type DiscoveryRegistryValue struct { - Channel chan<- []DiscoveryMessage + // Channel is the outbound communication channel used by discovery + // components (loaders, webhooks, etc.) to emit discovery messages + Channel chan<- []DiscoveryMessage + // WebhookEnabled indicates whether webhook-based discovery is enabled + // for this TargetSource WebhookEnabled bool + // Stop cancels the discovery pipeline associated with this registry entry + Stop context.CancelFunc } type LoaderConfig struct { diff --git a/internal/controller/discovery/registry.go b/internal/controller/discovery/registry.go index 0afa2b2..2193665 100644 --- a/internal/controller/discovery/registry.go +++ b/internal/controller/discovery/registry.go @@ -39,3 +39,11 @@ func (r *Registry[K, V]) Get(key K) (V, bool) { r.mu.RUnlock() return value, ok } + +func (r *Registry[K, V]) Exists(key K) bool { + r.mu.RLock() + defer r.mu.RUnlock() + + _, exists := r.m[key] + return exists +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index f36d47d..dca0570 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,12 +34,6 @@ import ( "github.com/go-logr/logr" ) -// pipelineHandle represents a controller-owned handle to a running pipeline -// The controller never manipulates internals; it only invokes cancel() -type pipelineHandle struct { - cancel context.CancelFunc -} - // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: @@ -52,14 +45,13 @@ type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme - mu sync.Mutex - // runningPipelines tracks currently active pipelines by NamespacedName - runningPipelines map[types.NamespacedName]pipelineHandle - BufferSize int ChunkSize int - DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[ + types.NamespacedName, + discoveryTypes.DiscoveryRegistryValue, + ] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -80,8 +72,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { + if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { + pipeline.Stop() + r.DiscoveryRegistry.Unregister(req.NamespacedName) + } logger.Info("TargetSource not found; stopped discovery pipeline") - r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { return ctrl.Result{}, err @@ -95,7 +90,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if r.hasPipelineRunning(req.NamespacedName) { + if r.DiscoveryRegistry.Exists(req.NamespacedName) { logger.Info("Discovery pipeline already running; reconciliation completed") return ctrl.Result{}, nil } @@ -117,14 +112,6 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type return &targetSource, nil } -// hasPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool { - r.mu.Lock() - defer r.mu.Unlock() - _, exists := r.runningPipelines[key] - return exists -} - // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues( @@ -132,7 +119,10 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type "namespace", key.Namespace, ) logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") - r.stopDiscoveryPipeline(key) + if pipeline, ok := r.DiscoveryRegistry.Get(key); ok { + pipeline.Stop() + r.DiscoveryRegistry.Unregister(key) + } // Remove finalizer if exists if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { @@ -197,7 +187,11 @@ func resolveRestartPolicy( // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { +func (r *TargetSourceReconciler) startDiscoveryPipeline( + key types.NamespacedName, + targetSource *gnmicv1alpha1.TargetSource, + logger logr.Logger, +) error { loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) @@ -208,6 +202,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ Channel: targetChannel, WebhookEnabled: webhookActivated, + Stop: supervisor.Stop, }); err != nil { return err } @@ -268,7 +263,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName close(targetChannel) r.DiscoveryRegistry.Unregister(key) - r.stopDiscoveryPipeline(key) logger.Info( "Discovery pipeline stopped; cleaned up resources", "targetsource", key.Name, @@ -276,35 +270,11 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) }() - r.mu.Lock() - r.runningPipelines[key] = pipelineHandle{ - cancel: func() { - supervisor.Stop() - }, - } - r.mu.Unlock() - return nil } -// stopDiscoveryPipeline stops and removes a running discovery pipeline -func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) { - r.mu.Lock() - running, ok := r.runningPipelines[key] - if ok { - delete(r.runningPipelines, key) - } - r.mu.Unlock() - - if ok { - running.cancel() - } -} - // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.runningPipelines = make(map[types.NamespacedName]pipelineHandle) - return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). Named("targetsource"). From 54c41fdf1b68f5168dad66c7cb5c7adca29016ed Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 14:36:23 +0000 Subject: [PATCH 137/246] add timeout as a const --- internal/controller/discovery/loaders/http/loader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 95470fc..9a7a949 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -18,7 +18,8 @@ import ( ) const ( - defaultPollInterval = 30 * time.Second + defaultPollInterval = 30 * time.Second + defaultTimeoutSeconds = 30 ) // Loader implements the HTTP pull discovery mechanism @@ -56,9 +57,8 @@ func (l *Loader) Start( } client := &http.Client{ - Timeout: 30 * time.Second, + Timeout: defaultTimeoutSeconds * time.Second, } - interval := defaultPollInterval ticker := time.NewTicker(interval) defer ticker.Stop() From 535ee49438fb9f4a3b45449a8321d6ab92966e42 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 07:06:36 +0000 Subject: [PATCH 138/246] rename target reconciler to message processor --- ...get_reconciler.go => message_processor.go} | 108 +++++++++--------- .../controller/targetsource_controller.go | 2 +- 2 files changed, 55 insertions(+), 55 deletions(-) rename internal/controller/discovery/{target_reconciler.go => message_processor.go} (68%) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/message_processor.go similarity index 68% rename from internal/controller/discovery/target_reconciler.go rename to internal/controller/discovery/message_processor.go index 39382ab..ed66940 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/message_processor.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// TargetReconciler consumes discovered targets and applies them to Kubernetes -type TargetReconciler struct { +// MessageProcessor consumes discovery messages and applies them to Kubernetes +type MessageProcessor struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type TargetReconciler struct { deferredEvents []core.DiscoveryEvent } -// NewTargetReconciler wires a TargetReconciler instance -func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler { - return &TargetReconciler{ +// NewMessageProcessor wires a MessageProcessor instance +func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { + return &MessageProcessor{ client: c, scheme: s, targetSource: ts, @@ -45,41 +45,41 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (r *TargetReconciler) Run(ctx context.Context) error { - r.ctx = ctx +func (m *MessageProcessor) Run(ctx context.Context) error { + m.ctx = ctx logger := log.FromContext(ctx).WithValues( - "component", "target-reconciler", - "targetsource", r.targetSource.Name, - "namespace", r.targetSource.Namespace, + "component", "message-processor", + "targetsource", m.targetSource.Name, + "namespace", m.targetSource.Namespace, ) - logger.Info("Target reconciler started") + logger.Info("Message processor started") - for r.ctx.Err() == nil { + for m.ctx.Err() == nil { select { - case batch, ok := <-r.in: + case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("Input channel closed; stopping target reconciler") + logger.Info("Input channel closed; stopping message processor") return nil } - r.queue = append(r.queue, batch...) + m.queue = append(m.queue, batch...) case <-ctx.Done(): - logger.Info("Context was canceled; stopping target reconciler") + logger.Info("Context was canceled; stopping message processor") return nil } - for len(r.queue) > 0 { + for len(m.queue) > 0 { if ctx.Err() != nil { return ctx.Err() } - msg := r.queue[0] - r.queue = r.queue[1:] + msg := m.queue[0] + m.queue = m.queue[1:] - if err := r.processMessage(r.ctx, msg, logger); err != nil { + if err := m.processMessage(m.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -89,11 +89,11 @@ func (r *TargetReconciler) Run(ctx context.Context) error { } } - logger.Info("target reconciler stopped") + logger.Info("Message processor stopped") return nil } -func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -108,7 +108,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc "chunkIndex", msg.ChunkIndex, "targets", len(msg.Targets), ) - return r.processSnapshot(ctx, msg, logger) + return m.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -117,7 +117,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc "event", msg.Event, "target", msg.Target.Name, ) - return r.processEvent(ctx, msg, logger) + return m.processEvent(ctx, msg, logger) default: return fmt.Errorf("Unknown discovery message type %T", msg) @@ -125,18 +125,18 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if r.activeSnapshot == nil { - r.startNewSnapshot(chunk, logger) +func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if m.activeSnapshot == nil { + m.startNewSnapshot(chunk, logger) return nil } - snapshot := r.activeSnapshot + snapshot := m.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := r.applySnapshot(ctx, snapshot, logger); err != nil { + if err := m.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -149,28 +149,28 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco } // Start collecting the new snapshot - r.startNewSnapshot(chunk, logger) + m.startNewSnapshot(chunk, logger) return nil } - return r.collectSnapshot(chunk, logger) + return m.collectSnapshot(chunk, logger) } -func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - r.activeSnapshot = &snapshotBuffer{ +func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + m.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - r.deferredEvents = nil + m.deferredEvents = nil - r.collectSnapshot(chunk, logger) + m.collectSnapshot(chunk, logger) } -func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := r.activeSnapshot +func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := m.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error( @@ -185,7 +185,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger "Snapshot chunk index out of range", "chunkIndex", chunk.ChunkIndex, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { @@ -194,7 +194,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger "Duplicate snapshot chunk received", "chunkIndex", chunk.ChunkIndex, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } @@ -207,10 +207,10 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } -func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - r.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -219,7 +219,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - r.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -231,7 +231,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot "Missing snapshot chunk", "chunkIndex", i, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -247,40 +247,40 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot // a.applyTargets // Replay deferred events - for _, event := range r.deferredEvents { + for _, event := range m.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := r.applyEvent(ctx, event, logger); err != nil { + if err := m.applyEvent(ctx, event, logger); err != nil { return err } } - r.activeSnapshot = nil - r.deferredEvents = nil + m.activeSnapshot = nil + m.deferredEvents = nil return nil } -func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if r.activeSnapshot != nil { - r.deferredEvents = append(r.deferredEvents, event) + if m.activeSnapshot != nil { + m.deferredEvents = append(m.deferredEvents, event) return nil } // Apply events - return r.applyEvent(ctx, event, logger) + return m.applyEvent(ctx, event, logger) } -func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: logger.Info( "Deleting Target", "target", event.Target.Name, - "targetsource", r.targetSource.Name, + "targetsource", m.targetSource.Name, ) case core.EventApply: logger.Info( @@ -288,7 +288,7 @@ func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryE "target", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels, - "targetsource", r.targetSource.Name, + "targetsource", m.targetSource.Name, ) } return nil diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index dca0570..b1755b0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -208,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline( } // Create target reconciler instance - targetReconciler := discovery.NewTargetReconciler( + targetReconciler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From c09c68f5c0c8a0f042b3192458bd4a8f1fe671cf Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 07:16:44 +0000 Subject: [PATCH 139/246] rename pipeline to runtime --- .../controller/targetsource_controller.go | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index b1755b0..d4442cc 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -37,9 +36,9 @@ import ( // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: -// - Ensure at most one pipeline per TargetSource -// - Start pipelines on reconcile -// - Stop pipelines on deletion or NotFound +// - Ensure at most one runtime per TargetSource +// - Start runtimes on reconcile +// - Stop runtimes on deletion or NotFound // - Delegate runtime failure handling to the Supervisor type TargetSourceReconciler struct { client.Client @@ -72,11 +71,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { - pipeline.Stop() + if runtime, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { + runtime.Stop() r.DiscoveryRegistry.Unregister(req.NamespacedName) } - logger.Info("TargetSource not found; stopped discovery pipeline") + logger.Info("TargetSource not found; stopped discovery runtime") return ctrl.Result{}, nil } else if err != nil { return ctrl.Result{}, err @@ -91,15 +90,15 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } if r.DiscoveryRegistry.Exists(req.NamespacedName) { - logger.Info("Discovery pipeline already running; reconciliation completed") + logger.Info("Discovery runtime already running; reconciliation completed") return ctrl.Result{}, nil } - if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } - logger.Info("Started discovery pipeline") + logger.Info("Started discovery runtime") return ctrl.Result{}, nil } @@ -112,15 +111,15 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type return &targetSource, nil } -// reconcileDeletion stops the discovery pipeline and removes the finalizer +// reconcileDeletion stops the discovery runtime and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues( "targetsource", key.Name, "namespace", key.Namespace, ) - logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") - if pipeline, ok := r.DiscoveryRegistry.Get(key); ok { - pipeline.Stop() + logger.Info("TargetSource was marked for deletion; stopping discovery runtime") + if runtime, ok := r.DiscoveryRegistry.Get(key); ok { + runtime.Stop() r.DiscoveryRegistry.Unregister(key) } @@ -157,6 +156,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } + // resolveRestartPolicy merges an optional spec override with the controller’s default restart policy func resolveRestartPolicy( override *gnmicv1alpha1.RestartPolicySpec, @@ -179,24 +179,19 @@ func resolveRestartPolicy( return resolved } - -// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource +// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource // -// Pipeline semantics: +// Runtime semantics: // 1. target reconciler is mandatory and must start first // 2. loader is optional and conditional on spec -// 3. Permanent failure of required components shuts down the pipeline +// 3. Permanent failure of required components shuts down the runtime // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryPipeline( +func (r *TargetSourceReconciler) startDiscoveryRuntime( key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, ) error { - loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) - - supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ @@ -264,7 +259,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline( close(targetChannel) r.DiscoveryRegistry.Unregister(key) logger.Info( - "Discovery pipeline stopped; cleaned up resources", + "Discovery runtime stopped; cleaned up resources", "targetsource", key.Name, "namespace", key.Namespace, ) From e4c01bac6d1abcdec28af44c20a65b6d4af477e0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:00:34 +0000 Subject: [PATCH 140/246] removed supervisor --- api/v1alpha1/targetsource_types.go | 11 -- .../discovery/core/loader_interface.go | 4 +- internal/controller/discovery/core/types.go | 5 +- internal/controller/discovery/defaults.go | 12 -- .../discovery/loaders/http/loader.go | 2 +- internal/controller/discovery/supervisor.go | 125 --------------- .../controller/targetsource_controller.go | 145 +++++++----------- 7 files changed, 59 insertions(+), 245 deletions(-) delete mode 100644 internal/controller/discovery/defaults.go delete mode 100644 internal/controller/discovery/supervisor.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 7c8f74c..ae719c1 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -29,9 +29,6 @@ type TargetSourceSpec struct { // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` - // +kubebuilder:validation:Optional - RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"` - // +kubebuilder:validation:MinLength=1 TargetProfile string `json:"targetProfile"` } @@ -57,14 +54,6 @@ type ConsulConfig struct { URL string `json:"url,omitempty"` } -type RestartPolicySpec struct { - // +kubebuilder:validation:Optional - MaxRestarts *int `json:"maxRestarts,omitempty"` - - // +kubebuilder:validation:Optional - BackoffSeconds *int `json:"backoffSeconds,omitempty"` -} - // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { Status string `json:"status"` diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 72f1898..bebd725 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -13,9 +13,9 @@ type Loader interface { // Name returns the unique loader identifier e.g. "pull" Name() string - // Start begins discovery and pushes target snapshots or events into the out channel + // Run begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is canceled - Start( + Run( ctx context.Context, targetsourceName types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 2f89fdf..94b4e85 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -8,10 +8,7 @@ type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages Channel chan<- []DiscoveryMessage - // WebhookEnabled indicates whether webhook-based discovery is enabled - // for this TargetSource - WebhookEnabled bool - // Stop cancels the discovery pipeline associated with this registry entry + // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc } diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go deleted file mode 100644 index dc6f046..0000000 --- a/internal/controller/discovery/defaults.go +++ /dev/null @@ -1,12 +0,0 @@ -package discovery - -import "time" - -// DefaultRestartPolicy defines the default restart behavior -// for the discovery components -func DefaultRestartPolicy() RestartPolicy { - return RestartPolicy{ - MaxRestarts: 5, - Backoff: 3 * time.Second, - } -} diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 67c61e1..383e974 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -27,7 +27,7 @@ func (l *Loader) Name() string { return "http" } -func (l *Loader) Start( +func (l *Loader) Run( ctx context.Context, targetsourceNN types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go deleted file mode 100644 index 22ec227..0000000 --- a/internal/controller/discovery/supervisor.go +++ /dev/null @@ -1,125 +0,0 @@ -package discovery - -import ( - "context" - "sync" - "time" - - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// Supervisor coordinates the runtime lifecycle of pipeline components -// -// Guarantees: -// - Each component is restarted independently -// - Permanent failure escalates according to policy -// - Stop() cancels all components -// - Wait() blocks until all goroutines exit -type Supervisor struct { - ctx context.Context - cancel context.CancelFunc - - wg sync.WaitGroup - - mu sync.Mutex - stopped bool -} - -// RestartPolicy defines restart behavior of a component -type RestartPolicy struct { - MaxRestarts int - Backoff time.Duration -} - -// ComponentSpec defines a supervised component -type ComponentSpec struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy - // EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline - EscalatesOnFailure bool -} - -// NewSupervisor creates a new Supervisor with a cancellable context -func NewSupervisor(parentCtx context.Context) *Supervisor { - ctx, cancel := context.WithCancel(parentCtx) - return &Supervisor{ - ctx: ctx, - cancel: cancel, - } -} - -// Stop signals all supervised components to stop by canceling the context -func (s *Supervisor) Stop() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.stopped { - return - } - s.stopped = true - s.cancel() -} - -// Done returns a channel that is closed when the pipeline is stopped -func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } - -// Wait blocks until all supervised components have exited -func (s *Supervisor) Wait() { s.wg.Wait() } - -// StartSupervisedComponent starts and supervises a component -func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { - s.wg.Add(1) - - go func() { - defer s.wg.Done() - - logger := log.FromContext(s.ctx).WithValues("component", component.Name) - failures := 0 - - for { - logger.Info( - "Starting supervised component", - "component", component.Name, - ) - err := component.Run(s.ctx) - - if s.ctx.Err() != nil { - logger.Info("component stopped due to pipeline shutdown") - return - } - - failures++ - logger.Error( - err, - "Supervised component failed", - "component", component.Name, - "attempt", failures, - "maxRestarts", component.Policy.MaxRestarts, - ) - - if failures >= component.Policy.MaxRestarts { - if component.EscalatesOnFailure { - logger.Error( - err, - "Supervised component permanently failed; stopped discovery pipeline", - "component", component.Name, - ) - s.Stop() - } else { - logger.Info( - "Optional component permanently failed; continuing without it", - "component", component.Name, - ) - } - return - } - - select { - case <-time.After(component.Policy.Backoff): - case <-s.ctx.Done(): - return - } - } - }() -} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index d4442cc..afc088b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -36,10 +36,10 @@ import ( // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: -// - Ensure at most one runtime per TargetSource -// - Start runtimes on reconcile -// - Stop runtimes on deletion or NotFound -// - Delegate runtime failure handling to the Supervisor +// - Ensure at most one discovery runtime per TargetSource +// - Start runtime on reconcile if not already running +// - Restart runtime on reconcile if spec changed +// - Stop runtime on deletion or NotFound type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme @@ -94,7 +94,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } @@ -156,113 +156,78 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } - -// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy -func resolveRestartPolicy( - override *gnmicv1alpha1.RestartPolicySpec, -) discovery.RestartPolicy { - defaults := discovery.DefaultRestartPolicy() - - if override == nil { - return defaults - } - - resolved := defaults - - if override.MaxRestarts != nil { - resolved.MaxRestarts = *override.MaxRestarts - } - - if override.BackoffSeconds != nil { - resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second - } - - return resolved -} -// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource +// startDiscovery creates and starts a discovery runtime for a TargetSource // -// Runtime semantics: -// 1. target reconciler is mandatory and must start first -// 2. loader is optional and conditional on spec -// 3. Permanent failure of required components shuts down the runtime -// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryRuntime( +// Invariant: +// - MessageProcessor and Loader must run for the lifetime of the TargetSource +// - Any unexpected exit is treated as a bug and triggers full shutdown +func (r *TargetSourceReconciler) startDiscovery( key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, ) error { - webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) + ctx, cancel := context.WithCancel(context.Background()) + + // Register discovery runtime of targetsource if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - WebhookEnabled: webhookActivated, - Stop: supervisor.Stop, + Channel: targetChannel, + Stop: cancel, }); err != nil { return err } - // Create target reconciler instance - targetReconciler := discovery.NewMessageProcessor( + // Cleanup function to cleanup discovery runtime of targetsource + cleanup := func() { + cancel() + r.DiscoveryRegistry.Unregister(key) + close(targetChannel) + } + + // Start message processor + messageProcessor := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target reconciler - targetReconcilerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-reconciler", - Policy: restartPolicy, - EscalatesOnFailure: true, - Run: func(ctx context.Context) error { - close(targetReconcilerReady) // Signals that reconciler started successfully - return targetReconciler.Run(ctx) - }, - }) - // Wait for reconciler to be ready before starting loader - select { - case <-targetReconcilerReady: - logger.Info("Target reconciler started") - case <-supervisor.Done(): - logger.Info("Supervisor stopped before target reconciler became ready") - return nil - } + go func() { + logger.Info("Message processor started") - // Create loader instance - if loaderConfigured { - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize}, - ) - if err != nil { - supervisor.Stop() - return err + if err := messageProcessor.Run(ctx); err != nil { + logger.Error(err, "Message processor exited unecpectedly") + } else { + logger.Error(nil, "Message processor exited unexpectedly without error") } - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "loader", - Policy: restartPolicy, - EscalatesOnFailure: !webhookActivated, - Run: func(ctx context.Context) error { - return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - }) - } + // Any exit is considered a bug that should stop the discovery runtime + cleanup() + }() - // Monitor supervisor in a separate goroutine to handle shutdown and cleanup + // Start target loader + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + loaderConfig := discoveryTypes.LoaderConfig{ + ChunkSize: r.ChunkSize, + } + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + loaderConfig, + ) + if err != nil { + logger.Error(err, "Target loader could not be created") + cleanup() + return err + } go func() { - <-supervisor.Done() - supervisor.Wait() // Wait for components to exit + if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil { + logger.Error(err, "Target loader exited unexpectedly") + } else { + logger.Error(nil, "Target loader exited unexpectedly without error") + } - close(targetChannel) - r.DiscoveryRegistry.Unregister(key) - logger.Info( - "Discovery runtime stopped; cleaned up resources", - "targetsource", key.Name, - "namespace", key.Namespace, - ) + // Any exit is considered a bug that should stop the discovery runtime + cleanup() }() return nil From 77dbd7e12dd19d33a38113d280b522a7fdea1d99 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:13:35 +0000 Subject: [PATCH 141/246] tidy loader configuration abstraction --- .../discovery/core/loader_interface.go | 10 +-------- internal/controller/discovery/core/types.go | 11 ++++++++-- internal/controller/discovery/loaders.go | 12 +++++------ .../discovery/loaders/http/loader.go | 21 +++++++------------ .../controller/targetsource_controller.go | 12 +++++------ 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index bebd725..895258a 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -2,9 +2,6 @@ package core import ( "context" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "k8s.io/apimachinery/pkg/types" ) // Loader defines a pluggable TargetSource loader interface @@ -15,10 +12,5 @@ type Loader interface { // Run begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is canceled - Run( - ctx context.Context, - targetsourceName types.NamespacedName, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []DiscoveryMessage, - ) error + Run(ctx context.Context, out chan<- []DiscoveryMessage) error } diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 94b4e85..5028972 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,6 +1,11 @@ package core -import "context" +import ( + "context" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) // DiscoveryRegistryValue represents the controller-owned runtime state // of a discovery pipeline for a single TargetSource @@ -13,7 +18,9 @@ type DiscoveryRegistryValue struct { } type LoaderConfig struct { - ChunkSize int + TargetsourceNN types.NamespacedName + Spec *gnmicv1alpha1.TargetSourceSpec + ChunkSize int } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 0d8ddd3..6c3e133 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -3,22 +3,20 @@ package discovery import ( "fmt" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" - "k8s.io/apimachinery/pkg/types" ) // NewLoader creates a loader by name -func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { +func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { - case spec.Provider.HTTP != nil: + case cfg.Spec.Provider.HTTP != nil: return http.New(cfg), nil - case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) + case cfg.Spec.Provider.Consul != nil: + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 383e974..17812aa 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -5,10 +5,8 @@ import ( "fmt" "time" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" "github.com/google/uuid" @@ -27,22 +25,17 @@ func (l *Loader) Name() string { return "http" } -func (l *Loader) Run( - ctx context.Context, - targetsourceNN types.NamespacedName, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { +func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error { logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", targetsourceNN, + "targetsource", l.cfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", targetsourceNN.Name, - "namespace", targetsourceNN.Namespace, + "targetsource", l.cfg.TargetsourceNN.Name, + "namespace", l.cfg.TargetsourceNN.Namespace, ) // Only for debugging: emit a static snapshot every 30 seconds @@ -57,17 +50,17 @@ func (l *Loader) Run( case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString()) + snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, }, } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index afc088b..0064570 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -207,20 +207,18 @@ func (r *TargetSourceReconciler) startDiscovery( // Start target loader // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled loaderConfig := discoveryTypes.LoaderConfig{ - ChunkSize: r.ChunkSize, + TargetsourceNN: key, + Spec: &targetSource.Spec, + ChunkSize: r.ChunkSize, } - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - loaderConfig, - ) + loader, err := discovery.NewLoader(loaderConfig) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() return err } go func() { - if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil { + if err := loader.Run(ctx, targetChannel); err != nil { logger.Error(err, "Target loader exited unexpectedly") } else { logger.Error(nil, "Target loader exited unexpectedly without error") From fe900e38774051956054d70af6ee24da88beb71f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:14:40 +0000 Subject: [PATCH 142/246] regenearte manifests without restartPolicy --- api/v1alpha1/zz_generated.deepcopy.go | 30 ------------------- .../operator.gnmic.dev_targetsources.yaml | 7 ----- 2 files changed, 37 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index df08573..608d47e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -843,31 +843,6 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) { - *out = *in - if in.MaxRestarts != nil { - in, out := &in.MaxRestarts, &out.MaxRestarts - *out = new(int) - **out = **in - } - if in.BackoffSeconds != nil { - in, out := &in.BackoffSeconds, &out.BackoffSeconds - *out = new(int) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec. -func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec { - if in == nil { - return nil - } - out := new(RestartPolicySpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -1325,11 +1300,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { (*out)[key] = val } } - if in.RestartPolicy != nil { - in, out := &in.RestartPolicy, &out.RestartPolicy - *out = new(RestartPolicySpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec. diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 6464ea2..b385d8e 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,13 +60,6 @@ spec: - message: exactly one of the fields in [http consul] must be set rule: '[has(self.http),has(self.consul)].filter(x,x==true).size() == 1' - restartPolicy: - properties: - backoffSeconds: - type: integer - maxRestarts: - type: integer - type: object targetLabels: additionalProperties: type: string From c1d7a91bc7e79f87736454dc31e530d3cf89b7fd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:29:54 +0000 Subject: [PATCH 143/246] tidy up comments --- internal/controller/discovery/discovery.go | 4 +--- internal/controller/discovery/loaders.go | 1 + internal/controller/targetsource_controller.go | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go index 3dc51bd..491cdfb 100644 --- a/internal/controller/discovery/discovery.go +++ b/internal/controller/discovery/discovery.go @@ -4,13 +4,11 @@ package discovery // // The discovery subsystem is responsible for: // - Receiving discovery data from external providers (loaders, webhooks). -// - Supervising discovery pipelines and restart semantics. // - Applying discovered state to Kubernetes Targets. // // The package is structured into the following subpackages: // - core: message contracts, snapshot/event types, and transport helpers. -// - pipeline: supervision, restart policies, and lifecycle control. -// - reconciler: snapshot + event target state application logic. +// - message processor: snapshot + event target state application logic. // - loaders: target discovery providers (HTTP, webhook, etc.). // - registry: key -> channel registry. // diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 6c3e133..9704b16 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -12,6 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { case cfg.Spec.Provider.HTTP != nil: + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled return http.New(cfg), nil case cfg.Spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 0064570..2ba18a2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -205,7 +205,6 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled loaderConfig := discoveryTypes.LoaderConfig{ TargetsourceNN: key, Spec: &targetSource.Spec, From 5c37a2a68557bb3a0fa142946f8593edca2be9ac Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:14:22 +0000 Subject: [PATCH 144/246] add dynamic endpoints --- api/v1alpha1/zz_generated.deepcopy.go | 5 + cmd/main.go | 20 ++-- internal/apiserver/apiserver.go | 111 +++++------------- internal/apiserver/targets.go | 71 +++++++++++ internal/controller/discovery/core/types.go | 4 +- internal/controller/discovery/loaders.go | 10 +- .../discovery/loaders/utils/endpoint.go | 22 ++++ .../controller/targetsource_controller.go | 4 + 8 files changed, 150 insertions(+), 97 deletions(-) create mode 100644 internal/apiserver/targets.go create mode 100644 internal/controller/discovery/loaders/utils/endpoint.go diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..3656535 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -826,6 +826,11 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = new(HTTPConfig) **out = **in } + if in.PULL != nil { + in, out := &in.PULL, &out.PULL + *out = new(WebhookSpec) + (*in).DeepCopyInto(*out) + } if in.Consul != nil { in, out := &in.Consul, &out.Consul *out = new(ConsulConfig) diff --git a/cmd/main.go b/cmd/main.go index e239a88..eadc12d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -43,8 +43,8 @@ import ( operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -127,12 +127,22 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Pipeline") os.Exit(1) } + + var api *apiserver.APIServer + if apiAddr != "" { + api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize) + if err != nil { + setupLog.Error(err, "unable to initialize API server") + os.Exit(1) + } + } if err := (&controller.TargetSourceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), BufferSize: discoveryBufferSize, ChunkSize: discoveryChunkSize, DiscoveryRegistry: discoveryRegistry, + APIRouter: api.Router(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) @@ -232,13 +242,7 @@ func main() { os.Exit(1) } - if apiAddr != "" { - api, err := apiserver.New(apiAddr, clusterReconciler, discoveryChunkSize) - if err != nil { - setupLog.Error(err, "unable to intialize gin API server") - os.Exit(1) - } - api.DiscoveryRegistry = discoveryRegistry + if api != nil { err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 9e9131a..4593737 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,14 +5,12 @@ package apiserver // Then use: go generate ./internal/apiserver import ( - "context" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" ) @@ -20,16 +18,27 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler - - DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ] + ChunkSize int } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` + namespace string `uri:"namespace" binding:"required"` gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` } -func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { +func New( + addr string, + clusterReconciler *controller.ClusterReconciler, + discoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ], + discoveryChunksize int, +) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -38,15 +47,26 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize }, router: router, clusterReconciler: clusterReconciler, - chunkSize: chunkSize, + DiscoveryRegistry: discoveryRegistry, + ChunkSize: discoveryChunksize, } - apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" - RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + // apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" + // RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + a.routes() return a, nil } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +func (a *APIServer) Router() *gin.Engine { + return a.router +} + +func (a *APIServer) routes() { + a.router.GET("/clusters/:namespace/:name/plan", a.GetClusterPlan) + a.router.POST("/api/v1/:namespace/target-source/:name/createTargets", a.CreateTargets) +} + // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) @@ -58,77 +78,6 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. -func (a *APIServer) CreateTargets(c *gin.Context) { - // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template - // But I don't think it is needed in the CreateTargets function - // url := parseURI(c) - // fmt.Printf("namespace: %s", url.namespace) - // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) - - var payloadTargets Targets - if err := c.ShouldBind(&payloadTargets); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware - // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation - // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. - if payloadTargets.TargetSourceNameSpace == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) - return - } - if payloadTargets.TargetSourceName == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) - return - } - - targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { - if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") - break - } - if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") - break - } - - event := core.CREATE - switch target.Operation { - case Create: - event = core.CREATE - case Delete: - event = core.DELETE - } - - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: target.Name, - Address: target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, - }, - Event: event, - }) - } - } - - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) - if !ok { - logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) - return - } - core.SendEvents(context.Background(), ch, targets, a.chunkSize) - c.JSON(http.StatusOK, payloadTargets) -} - // parseURI parses URI to urlStruct. func parseURI(c *gin.Context) (url urlStruct) { var u urlStruct diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go new file mode 100644 index 0000000..0e2c48f --- /dev/null +++ b/internal/apiserver/targets.go @@ -0,0 +1,71 @@ +package apiserver + +import ( + "context" + "net/http" + + "github.com/bytedance/gopkg/util/logger" + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" + "k8s.io/apimachinery/pkg/types" +) + +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. +// Creates a []core.DiscoveryEvent sends it to the core package. +func (a *APIServer) CreateTargets(c *gin.Context) { + namespace := c.Param("namespace") + name := c.Param("name") + key := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + + registry, ok := a.DiscoveryRegistry.Get(key) + if !ok { + c.JSON(http.StatusNotFound, gin.H{ + "error": "TargetSource not active or does not exist", + }) + return + } + + var payloadTargets Targets + if err := c.ShouldBind(&payloadTargets); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + targets := []core.DiscoveryEvent{} + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Address == "" || target.Name == "" || target.Operation == "" { + logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") + break + } + if target.Operation.Valid() != true { + logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") + break + } + + event := core.EventApply + switch target.Operation { + case Create: + event = core.EventApply + case Delete: + event = core.EventDelete + } + + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: target.Name, + Address: target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, + }) + } + } + + utils.SendEvents(context.Background(), registry.Channel, targets, a.ChunkSize) + c.JSON(http.StatusOK, payloadTargets) +} diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 5028972..12490e9 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -3,12 +3,13 @@ package core import ( "context" + "github.com/gin-gonic/gin" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" ) // DiscoveryRegistryValue represents the controller-owned runtime state -// of a discovery pipeline for a single TargetSource +// and its configuration type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages @@ -21,6 +22,7 @@ type LoaderConfig struct { TargetsourceNN types.NamespacedName Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int + Router *gin.Engine } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 9704b16..02669cb 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -4,20 +4,16 @@ import ( "fmt" "github.com/gnmic/operator/internal/controller/discovery/core" - http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) -// NewLoader creates a loader by name func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { - switch { case cfg.Spec.Provider.HTTP != nil: - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled return http.New(cfg), nil - case cfg.Spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + default: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } - } diff --git a/internal/controller/discovery/loaders/utils/endpoint.go b/internal/controller/discovery/loaders/utils/endpoint.go new file mode 100644 index 0000000..ef83f18 --- /dev/null +++ b/internal/controller/discovery/loaders/utils/endpoint.go @@ -0,0 +1,22 @@ +package utils + +import ( + "fmt" + + "github.com/gin-gonic/gin" + "k8s.io/apimachinery/pkg/types" +) + +func CreateTargetsPath( + router *gin.Engine, + nn types.NamespacedName, + handler gin.HandlerFunc, +) { + path := fmt.Sprintf( + "/api/v1/%s/target-source/%s/createTargets", + nn.Namespace, + nn.Name, + ) + + router.POST(path, handler) +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 2ba18a2..935139c 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/gin-gonic/gin" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" @@ -51,6 +52,8 @@ type TargetSourceReconciler struct { types.NamespacedName, discoveryTypes.DiscoveryRegistryValue, ] + + APIRouter *gin.Engine } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -209,6 +212,7 @@ func (r *TargetSourceReconciler) startDiscovery( TargetsourceNN: key, Spec: &targetSource.Spec, ChunkSize: r.ChunkSize, + Router: r.APIRouter, } loader, err := discovery.NewLoader(loaderConfig) if err != nil { From 05c7538ce47a4b81fd245b11435cb13481c4c671 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:23:34 +0000 Subject: [PATCH 145/246] move webhook spec into provider and rename it to acceptPush --- api/v1alpha1/targetsource_types.go | 10 +++------- internal/controller/discovery/core/types.go | 5 ++++- internal/controller/discovery/loaders.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index ae719c1..3d69743 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -24,8 +24,7 @@ import ( // +kubebuilder:validation:Required type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` - // +kubebuilder:validation:Optional - Webhook WebhookSpec `json:"webhook,omitempty"` + // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` @@ -39,14 +38,11 @@ type ProviderSpec struct { Consul *ConsulConfig `json:"consul,omitempty"` } -type WebhookSpec struct { - // +kubebuilder:validation:Optional - Enabled *bool `json:"enabled,omitempty"` -} - type HTTPConfig struct { // +kubebuilder:validation:MinLength=1 URL string `json:"url"` + // +kubebuilder:validation:Optional + AcceptPush bool `json:"acceptPush,omitempty"` } type ConsulConfig struct { diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 5028972..1dfcc9f 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -8,19 +8,22 @@ import ( ) // DiscoveryRegistryValue represents the controller-owned runtime state -// of a discovery pipeline for a single TargetSource +// with its configuration for a single TargetSource type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages Channel chan<- []DiscoveryMessage // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc + + LoaderConfig *LoaderConfig } type LoaderConfig struct { TargetsourceNN types.NamespacedName Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int + AcceptPush bool } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 9704b16..487c76b 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -12,7 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { case cfg.Spec.Provider.HTTP != nil: - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush return http.New(cfg), nil case cfg.Spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) From 061d4b83daac3a5c26fbe12151aa353a88d1a213 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:24:00 +0000 Subject: [PATCH 146/246] regenerate manifests --- api/v1alpha1/zz_generated.deepcopy.go | 21 ------------------- .../operator.gnmic.dev_targetsources.yaml | 7 ++----- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1292,7 +1292,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { *out = new(ProviderSpec) (*in).DeepCopyInto(*out) } - in.Webhook.DeepCopyInto(&out.Webhook) if in.TargetLabels != nil { in, out := &in.TargetLabels, &out.TargetLabels *out = make(map[string]string, len(*in)) @@ -1478,23 +1477,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { - *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. -func (in *WebhookSpec) DeepCopy() *WebhookSpec { - if in == nil { - return nil - } - out := new(WebhookSpec) - in.DeepCopyInto(out) - return out -} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index b385d8e..37d6919 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -49,6 +49,8 @@ spec: type: object http: properties: + acceptPush: + type: boolean url: minLength: 1 type: string @@ -67,11 +69,6 @@ spec: targetProfile: minLength: 1 type: string - webhook: - properties: - enabled: - type: boolean - type: object required: - provider - targetProfile From f8b92b2c56872357418ebcc135259f5c5f86e2ef Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:56:38 +0000 Subject: [PATCH 147/246] verify acceptPush is enabled for webhook config --- internal/apiserver/targets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go index 0e2c48f..2e434b9 100644 --- a/internal/apiserver/targets.go +++ b/internal/apiserver/targets.go @@ -22,7 +22,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } registry, ok := a.DiscoveryRegistry.Get(key) - if !ok { + if !ok || (registry.LoaderConfig.AcceptPush != true) { c.JSON(http.StatusNotFound, gin.H{ "error": "TargetSource not active or does not exist", }) From 41655a0d4e835bc2ff0b8a5a1cdaf55aa4bdfd7a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 14:12:05 +0000 Subject: [PATCH 148/246] remove spec from laoder config --- internal/controller/discovery/core/types.go | 2 -- internal/controller/discovery/loaders.go | 9 +++++---- internal/controller/targetsource_controller.go | 16 ++++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 1dfcc9f..993c84e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -3,7 +3,6 @@ package core import ( "context" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" ) @@ -21,7 +20,6 @@ type DiscoveryRegistryValue struct { type LoaderConfig struct { TargetsourceNN types.NamespacedName - Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int AcceptPush bool } diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 487c76b..d179d3e 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -3,18 +3,19 @@ package discovery import ( "fmt" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name -func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { +func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { switch { - case cfg.Spec.Provider.HTTP != nil: - cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush + case spec.Provider.HTTP != nil: + cfg.AcceptPush = spec.Provider.HTTP.AcceptPush return http.New(cfg), nil - case cfg.Spec.Provider.Consul != nil: + case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 2ba18a2..1cf962d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -168,11 +168,16 @@ func (r *TargetSourceReconciler) startDiscovery( ) error { targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) ctx, cancel := context.WithCancel(context.Background()) + loaderConfig := discoveryTypes.LoaderConfig{ + TargetsourceNN: key, + ChunkSize: r.ChunkSize, + } // Register discovery runtime of targetsource if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - Stop: cancel, + Channel: targetChannel, + Stop: cancel, + LoaderConfig: &loaderConfig, }); err != nil { return err } @@ -205,12 +210,7 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - loaderConfig := discoveryTypes.LoaderConfig{ - TargetsourceNN: key, - Spec: &targetSource.Spec, - ChunkSize: r.ChunkSize, - } - loader, err := discovery.NewLoader(loaderConfig) + loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() From 97849ae9d9a7afdc13aab966882433f0a59f0f7c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 15:08:55 +0000 Subject: [PATCH 149/246] update LoaderConfig in registry --- internal/apiserver/apiserver.go | 2 +- internal/controller/discovery/core/types.go | 4 +- internal/controller/discovery/loaders.go | 8 ++-- .../discovery/loaders/http/loader.go | 20 +++++----- .../controller/targetsource_controller.go | 40 +++++++++++-------- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 705b277..5eb88b8 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" ) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 993c84e..99605b9 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -15,10 +15,10 @@ type DiscoveryRegistryValue struct { // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc - LoaderConfig *LoaderConfig + CommonLoaderConfig *CommonLoaderConfig } -type LoaderConfig struct { +type CommonLoaderConfig struct { TargetsourceNN types.NamespacedName ChunkSize int AcceptPush bool diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index d179d3e..7f2c656 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -9,16 +9,16 @@ import ( ) // NewLoader creates a loader by name -func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, core.CommonLoaderConfig, error) { switch { case spec.Provider.HTTP != nil: cfg.AcceptPush = spec.Provider.HTTP.AcceptPush - return http.New(cfg), nil + return http.New(cfg), cfg, nil case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 17812aa..3325adb 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -13,12 +13,12 @@ import ( ) type Loader struct { - cfg core.LoaderConfig + commonCfg core.CommonLoaderConfig } // New instantiates the http loader with the provided config -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} +func New(cfg core.CommonLoaderConfig) core.Loader { + return &Loader{commonCfg: cfg} } func (l *Loader) Name() string { @@ -29,13 +29,13 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", l.cfg.TargetsourceNN, + "targetsource", l.commonCfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", l.cfg.TargetsourceNN.Name, - "namespace", l.cfg.TargetsourceNN.Namespace, + "targetsource", l.commonCfg.TargetsourceNN.Name, + "namespace", l.commonCfg.TargetsourceNN.Namespace, ) // Only for debugging: emit a static snapshot every 30 seconds @@ -50,21 +50,21 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString()) + snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()}, }, } - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 1cf962d..8207103 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -168,20 +168,11 @@ func (r *TargetSourceReconciler) startDiscovery( ) error { targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) ctx, cancel := context.WithCancel(context.Background()) - loaderConfig := discoveryTypes.LoaderConfig{ + loaderConfig := discoveryTypes.CommonLoaderConfig{ TargetsourceNN: key, ChunkSize: r.ChunkSize, } - // Register discovery runtime of targetsource - if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - Stop: cancel, - LoaderConfig: &loaderConfig, - }); err != nil { - return err - } - // Cleanup function to cleanup discovery runtime of targetsource cleanup := func() { cancel() @@ -189,13 +180,34 @@ func (r *TargetSourceReconciler) startDiscovery( close(targetChannel) } - // Start message processor messageProcessor := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) + loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{ + TargetsourceNN: key, + ChunkSize: r.ChunkSize, + }, + &targetSource.Spec, + ) + if err != nil { + logger.Error(err, "Target loader could not be created") + cleanup() + return err + } + + // Register discovery runtime of targetsource + if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ + Channel: targetChannel, + Stop: cancel, + CommonLoaderConfig: &loaderConfig, + }); err != nil { + return err + } + + // Start message processor go func() { logger.Info("Message processor started") @@ -210,12 +222,6 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec) - if err != nil { - logger.Error(err, "Target loader could not be created") - cleanup() - return err - } go func() { if err := loader.Run(ctx, targetChannel); err != nil { logger.Error(err, "Target loader exited unexpectedly") From 953aaa75a93eccb567d5e31693c71c5309ae3a6a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 15:39:18 +0000 Subject: [PATCH 150/246] fix after merge --- internal/apiserver/targets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go index 2e434b9..7a55feb 100644 --- a/internal/apiserver/targets.go +++ b/internal/apiserver/targets.go @@ -22,7 +22,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } registry, ok := a.DiscoveryRegistry.Get(key) - if !ok || (registry.LoaderConfig.AcceptPush != true) { + if !ok || (registry.CommonLoaderConfig.AcceptPush != true) { c.JSON(http.StatusNotFound, gin.H{ "error": "TargetSource not active or does not exist", }) From f683a44cb7e441c23e554f486034deb53b6c4a09 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 09:06:52 +0000 Subject: [PATCH 151/246] comments from weekly --- internal/apiserver/apiserver.go | 85 ++---------------------------- internal/apiserver/helpers.go | 91 +++++++++++++++++++++++++++++++++ internal/apiserver/openapi.yaml | 2 +- 3 files changed, 97 insertions(+), 81 deletions(-) create mode 100644 internal/apiserver/helpers.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8c01486..63ed1e1 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -4,6 +4,8 @@ package apiserver // To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) // Then use: go generate ./internal/apiserver +// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 + import ( "context" "net/http" @@ -40,13 +42,12 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize clusterReconciler: clusterReconciler, chunkSize: chunkSize, } + // /api/v1/:namespace/target-source/:target_source_name apiBaseURL := "/api/v1/:namespace/:gNMIcControllerName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } -// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 - // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) @@ -74,86 +75,10 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) return } - + // make sure channel is not closed if targetsource in registry is deleted + // timeout for sending to the channel targets := createDiscoveryEvent(payloadTargets) // fmt.Printf("core.DiscoveryEvent was created: %v", targets) core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } - -// createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { - targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { - if target.Address == "" || target.Name == "" || target.Operation == "" { - // no REST API return here as not all targets might - logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") - break - } - if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") - break - } - - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: target.Name, - Address: target.Address, - Labels: convertTargetLabelsToMap(target), - }, - Event: getEvent(target), - }) - } - } - return targets -} - -// getKey returns key for used to identify correct channel in DiscoveryRegistry -func getKey(payloadTargets Targets) types.NamespacedName { - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - return key -} - -// convertTargetLabelsToMap converts target.Labels to map. -func convertTargetLabelsToMap(target Target) map[string]string { - labelToMap := make(map[string]string) - if target.Labels != nil { - for _, tag := range *target.Labels { - if tag.Key == nil || tag.Value == nil || *tag.Key == "" { - continue - } - labelToMap[*tag.Key] = *tag.Value - } - } - return labelToMap -} - -// getEvent converts target.Operation to core.Operation. -func getEvent(target Target) core.EventAction { - event := core.CREATE - switch target.Operation { - case Created: - event = core.UPDATE - case Updated: - event = core.UPDATE - case Deleted: - event = core.DELETE - default: - logger.Warn("Received invalid Operation flag") - } - return event -} - -// parseURI parses URI to urlStruct. -func parseURI(c *gin.Context) (url urlStruct) { - var u urlStruct - if err := c.ShouldBindUri(&u); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - return u -} diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go new file mode 100644 index 0000000..382b44a --- /dev/null +++ b/internal/apiserver/helpers.go @@ -0,0 +1,91 @@ +package apiserver + +import ( + "fmt" + "net/http" + + "github.com/bytedance/gopkg/util/logger" + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" +) + +// createDiscoveryEvent creates object of type core.DiscoveryEvent +func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { + targets := []core.DiscoveryEvent{} + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Name == "" { + // no REST API return here as not all targets might be incomplete + err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) + logger.Error(err, "Failed creating DiscoveryEvent") + break + } + if target.Address == "" { + err := fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + logger.Error(err, "Failed creating DiscoveryEvent") + break + } + + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: target.Name, + Address: target.Address, + Labels: convertTargetLabelsToMap(target), + }, + Event: getEvent(target, i), + }) + } + } + return targets +} + +// getKey returns key for used to identify correct channel in DiscoveryRegistry +func getKey(payloadTargets Targets) types.NamespacedName { + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + return key +} + +// convertTargetLabelsToMap converts target.Labels to map. +func convertTargetLabelsToMap(target Target) map[string]string { + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + return labelToMap +} + +// getEvent converts target.Operation to core.Operation. +func getEvent(target Target, index int) core.EventAction { + event := core.CREATE + switch target.Operation { + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: + event = core.DELETE + default: + err := fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) + logger.Error(err, "Failed creating DiscoveryEvent") + } + return event +} + +// parseURI parses URI to urlStruct. +func parseURI(c *gin.Context) (url urlStruct) { + var u urlStruct + if err := c.ShouldBindUri(&u); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + return u +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 093c1a4..99a9a87 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -66,7 +66,7 @@ components: - created - updated - deleted - Tag: + Tag: # tag will be removed, moved directly under labels type: object properties: key: From fe31b0d3c43225f8dc8e80f0cc2bc6e23b2c4138 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 09:08:36 +0000 Subject: [PATCH 152/246] update manifests --- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..3656535 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -826,6 +826,11 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = new(HTTPConfig) **out = **in } + if in.PULL != nil { + in, out := &in.PULL, &out.PULL + *out = new(WebhookSpec) + (*in).DeepCopyInto(*out) + } if in.Consul != nil { in, out := &in.Consul, &out.Consul *out = new(ConsulConfig) From 08273d0ad1f37bdb4878b7f5a8e5f073aaf6237f Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 11:49:34 +0000 Subject: [PATCH 153/246] udpate contract --- internal/apiserver/apiserver.go | 18 ++++++-------- internal/apiserver/gen.go | 31 ++++++++++++------------ internal/apiserver/helpers.go | 12 ++++----- internal/apiserver/openapi.yaml | 43 ++++++++++++++------------------- internal/apiserver/temp.md | 19 ++++++++++++++- 5 files changed, 64 insertions(+), 59 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 81b7e8f..8abd2ef 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -31,8 +31,8 @@ type APIServer struct { } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` - gNMIcControllerName string `uri:"gNMIcControllerName" binding:"required"` + Namespace string `uri:"namespace" binding:"required"` + Name string `uri:"name" binding:"required"` } func New( @@ -55,14 +55,10 @@ func New( DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, } - // apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" - // RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) a.routes() return a, nil } -// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 - func (a *APIServer) Router() *gin.Engine { return a.router } @@ -75,7 +71,7 @@ func (a *APIServer) routes() { // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) - plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcControllerName) + plan, err := a.clusterReconciler.GetClusterPlan(url.Namespace, url.Name) if err != nil { c.String(404, err.Error()) return @@ -86,17 +82,17 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { logger.Info("received POST request for CreateTargets.") - + url := parseURI(c) var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - registry, ok := a.DiscoveryRegistry.Get(getKey(payloadTargets)) + registry, ok := a.DiscoveryRegistry.Get(getKey(url)) if !ok { - logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) + logger.Error("TargetSource ", url.Namespace, "/", url.Name, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + url.Namespace + " / " + url.Name + " does not exist"}) return } // make sure channel is not closed if targetsource in registry is deleted diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 07a82c2..5811b9b 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -37,8 +37,8 @@ func (e TargetOperation) Valid() bool { } } -// Tag defines model for Tag. -type Tag struct { +// Label defines model for Label. +type Label struct { Key *string `json:"key,omitempty"` Value *string `json:"value,omitempty"` } @@ -46,7 +46,7 @@ type Tag struct { // Target defines model for Target. type Target struct { Address string `json:"address"` - Labels *[]Tag `json:"labels,omitempty"` + Labels *[]Label `json:"labels,omitempty"` Name string `json:"name"` Operation TargetOperation `json:"operation"` Profile *string `json:"profile,omitempty"` @@ -56,11 +56,10 @@ type Target struct { type TargetOperation string // Targets defines model for Targets. -type Targets struct { - TargetList []Target `json:"TargetList"` - TargetSourceName string `json:"TargetSourceName"` - TargetSourceNameSpace string `json:"targetSourceNameSpace"` -} +type Targets = []Target + +// CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. +type CreateTargetsJSONRequestBody = Targets // ServerInterface represents all server handlers. type ServerInterface interface { @@ -141,14 +140,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/5STz2vbMBTH/xXxtqOJ3e7m2yijFLauLLmNHlT5JVEnS9rTU8EU/+9DkpMmtQPdSc96", - "vz9f+RWU672zaDlA+wpB7bGX2dzIXTo8OY/EGvPlHxzSwYNHaCEwabuDsYIXaSIueMbqcOOenlFxit1I", - "2iHPa8uuIwxhsb6RT2iySzP22fhMuIUWPtVvG9TT+HWa/a21JJJD+rayx8XyaQzJ2tnkRRt7aH+DIpSM", - "HVQQfTdZHRpM1mM1L+LJbbW5QIHwb9SEXSqcx6iO+562f7zIK8yBFcd3Hfg/wGT2C2yKZ+0iKby/xInf", - "Ba29VB9YeDltoWd1utOcRaqr7dbljpoTa9jd/7hT4mcm6Ej8+rbeiK8Pd1DBC1LIkkKzalZXk85Weg0t", - "fFldrxqowEveZ2p1kfsUtitgj+rcddDCzVlY2jN4Z0NR5Lq5SodyltHmZOm90Sqn18+hPLAixiU9P6hh", - "mFE+3C9z6zAo0r488kOsmN64CFEpDGEbjRly4RD7XtJwXFjwlKGt4D2Kc+w5pfZG5v2mv/uc2y3yjYmB", - "kR5S2Axck47zIU/iBSFHsti9G+4WWagSJnL7cRzHfwEAAP//JDP+7tUEAAA=", + "H4sIAAAAAAAC/7RTO2/cMAz+KwLb0Tg76eatDYogQB9Bm63IoEi8O6WypFJUACPwfy9EO/foXYAsnURL", + "JL8H6WcwcUgxYOAM/TNks8VBS/hFP6CvQaKYkNihXP/GsR48JoQeMpMLG5gaeNK+4JmXqXm5iQ+PaLjm", + "3mnaIJ/21tYS5ny2v6905MkxDhK8J1xDD+/avYZ2EdDO7PfgmkiP9TvoAc8CVCKaXQz1FUMZoP8FhlAz", + "WmigJLtEFj3W6L45bZIorp1/xQfCP8UR2tpYaDQ7xYfw96869nb5i8Mn+isNF9ZRCDquTGHz7euNUd8F", + "P5L68fnnnfp4ewMNPCFlMQS6Vbe6WFwKOjno4cPqctVBA0nzVvi0s1kHVFPMMuWdthsLPVwdpc22YOZP", + "0cpmmRgYg9TplLwzUtk+5nkys8K36c+z4L3vTAXlIqcY8rx0l93F/4G1mA25NK/UywTVslEqF2Mw53Xx", + "fpTtyGUYNI07gxQvFS4o3qI6HpOUtMlrYbf8Tcc+XyNf+ZIZ6bamncju6nFM8iBfEXKhgPYfctfIysxp", + "SuCnaZr+BgAA//+GH+FbRwQAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 5114dfe..37eb12e 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -11,10 +11,10 @@ import ( ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { +func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { + if len(payloadTargets) > 0 { + for i, target := range payloadTargets { if target.Name == "" { // no REST API return here as not all targets might be incomplete err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) @@ -41,10 +41,10 @@ func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { } // getKey returns key for used to identify correct channel in DiscoveryRegistry -func getKey(payloadTargets Targets) types.NamespacedName { +func getKey(u urlStruct) types.NamespacedName { key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, + Namespace: u.Namespace, + Name: u.Name, } return key } diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 99a9a87..e7745c9 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -14,38 +14,37 @@ paths: post: summary: "Create targets in the gNMIc Operator" operationId: "createTargets" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Targets' responses: '201': description: "Targets created successfully" content: application/json: schema: - type: object - required: - - Targets - properties: - Targets: - $ref: '#/components/schemas/Targets' + $ref: '#/components/schemas/Targets' components: schemas: Targets: + type: array + items: + $ref: '#/components/schemas/Target' + + Label: type: object - required: - - targetSourceNameSpace - - TargetSourceName - - TargetList properties: - targetSourceNameSpace: + key: type: string - TargetSourceName: + value: type: string - TargetList: - type: array - items: - $ref: '#/components/schemas/Target' + Target: type: object - required: + required: - name - address - operation @@ -59,17 +58,11 @@ components: labels: type: array items: - $ref: '#/components/schemas/Tag' + $ref: '#/components/schemas/Label' operation: type: string enum: - created - updated - deleted - Tag: # tag will be removed, moved directly under labels - type: object - properties: - key: - type: string - value: - type: string \ No newline at end of file + \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index c70f463..b1cf607 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,22 @@ +curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ + -H "Content-Type: application/json" \ + -d '[ + { + "address": "1.1.1.1", + "name": "Router1", + "operation": "created", + "profile": "defaultProfile", + "labels": [ + { "key": "tags", "value": "tag1, tag2" } + ] + } + ]' + + +# old (before 4th of may) + ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/target-source/netbox/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"http-discovery", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' From 1bdc2945f0324de47a6bb10e15d2096aba3c272e Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 08:49:17 -0600 Subject: [PATCH 154/246] moved updateStatus function to client.go --- internal/controller/discovery/client.go | 18 +++++++++ .../controller/targetsource_controller.go | 39 +------------------ 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index a9d790f..e5cc5ea 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -7,6 +7,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -69,3 +70,20 @@ func deleteTarget(ctx context.Context, c client.Client, name string, namespace s return err } + +// updateTargetSourceStatus updates the status of the TargetSource Object ts. The only fields updated are targetCount and LastSync, which takes the current timestamp. +func updateTargetSourceStatus(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource, targetCount int32) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &gnmicv1alpha1.TargetSource{} + if err := c.Get(ctx, client.ObjectKeyFromObject(ts), latest); err != nil { + return err + } + + latest.Status.TargetsCount = targetCount + latest.Status.LastSync = metav1.Now() + + return c.Status().Update(ctx, latest) + }) + + return err +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 97a1432..f23a4a0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -20,10 +20,8 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -95,8 +93,8 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request if targetSource.Generation != targetSource.Status.ObservedGeneration { r.reconcileDeletion(ctx, req.NamespacedName, targetSource) } else { - logger.Info("Discovery runtime already running; reconciliation completed, updating status") - return ctrl.Result{}, r.updateStatus(ctx, targetSource) + logger.Info("Discovery runtime already running; reconciliation completed") + return ctrl.Result{}, nil } } @@ -109,10 +107,6 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if err := r.updateStatus(ctx, targetSource); err != nil { - return ctrl.Result{}, err - } - logger.Info("Started discovery runtime") return ctrl.Result{}, nil } @@ -251,35 +245,6 @@ func (r *TargetSourceReconciler) startDiscovery( return nil } -func (r *TargetSourceReconciler) updateStatus(ctx context.Context, ts *gnmicv1alpha1.TargetSource) error { - // Update TargetSource Status field - var targetList gnmicv1alpha1.TargetList - - err := r.Client.List(ctx, &targetList, - client.InNamespace(ts.Namespace), - client.MatchingLabels{ - "gnmic.io/source": ts.Name, - }, - ) - if err != nil { - return err - } - - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - latest := &gnmicv1alpha1.TargetSource{} - if err := r.Get(ctx, client.ObjectKeyFromObject(ts), latest); err != nil { - return err - } - - latest.Status.TargetsCount = int32(len(targetList.Items)) - latest.Status.LastSync = metav1.Now() - - return r.Status().Update(ctx, latest) - }) - - return err -} - // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). From 92552400b583ed5bd8d93cdfea79af00799ff797 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:07:00 -0600 Subject: [PATCH 155/246] changed updateStauts handling --- .../controller/discovery/message_processor.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 14ff401..af8da1f 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -31,6 +31,7 @@ type MessageProcessor struct { activeSnapshot *snapshotBuffer // Events are deferred while snapshot is in progress deferredEvents []core.DiscoveryEvent + targetCount int32 } // NewMessageProcessor wires a MessageProcessor instance @@ -305,6 +306,9 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot } } + m.targetCount = int32(len(allTargets)) + m.updateStatus(logger) + m.activeSnapshot = nil m.deferredEvents = nil return nil @@ -321,6 +325,8 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Info("deleted target object", "name", event.Target.Name, ) + m.targetCount-- + m.updateStatus(logger) } case core.EventApply: target := generateTargetResource(event.Target, m.targetSource) @@ -333,8 +339,20 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Info("applied target object", "name", event.Target.Name, ) + m.targetCount++ + m.updateStatus(logger) } } return nil } + +func (m *MessageProcessor) updateStatus(logger logr.Logger) { + if err := updateTargetSourceStatus(m.ctx, m.client, m.targetSource, m.targetCount); err != nil { + logger.Error(err, "error updating TargetSource status") + } else { + logger.Info("updated target source status", + "targetCount", m.targetCount, + ) + } +} From 29c4974c83070f48779d4dfde8af89cc3e82f1d4 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:16:42 -0600 Subject: [PATCH 156/246] removed owned targets from targetsource reconciliation --- internal/controller/targetsource_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index f23a4a0..6d88b3d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -249,7 +249,6 @@ func (r *TargetSourceReconciler) startDiscovery( func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). - Owns(&gnmicv1alpha1.Target{}). Named("targetsource"). Complete(r) } From 24cc376d1b09193839922b847c5003eb1bf0f9d5 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:37:27 -0600 Subject: [PATCH 157/246] added predicate for targetsource reconciliation --- internal/controller/targetsource_controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 6d88b3d..16b1f2c 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -23,9 +23,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" @@ -248,7 +250,10 @@ func (r *TargetSourceReconciler) startDiscovery( // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&gnmicv1alpha1.TargetSource{}). + For( + &gnmicv1alpha1.TargetSource{}, + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). Named("targetsource"). Complete(r) } From 8da80521138dcd787aaabab6b7bf65bc0e4b067d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:37:42 -0600 Subject: [PATCH 158/246] changed updateStatus calling for event --- .../controller/discovery/message_processor.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index af8da1f..314684a 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -223,7 +223,19 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover } // Apply events - return m.applyEvent(ctx, event, logger) + err := m.applyEvent(ctx, event, logger) + if err == nil { + switch event.Event { + case core.EventApply: + m.targetCount++ + m.updateStatus(logger) + case core.EventDelete: + m.targetCount-- + m.updateStatus(logger) + } + } + + return err } func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { @@ -325,8 +337,6 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Info("deleted target object", "name", event.Target.Name, ) - m.targetCount-- - m.updateStatus(logger) } case core.EventApply: target := generateTargetResource(event.Target, m.targetSource) @@ -339,8 +349,6 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Info("applied target object", "name", event.Target.Name, ) - m.targetCount++ - m.updateStatus(logger) } } From f4d6bac190752ad488194cd4fa8f4745cb2d4c8d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:51:14 -0600 Subject: [PATCH 159/246] added comments to mapper.go --- internal/controller/discovery/mapper.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index f34fe36..6cc28ae 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -1,8 +1,5 @@ package discovery -// This file makes diff between existing and new targets -// file decides which targets to create/update/delete - import ( "maps" "strings" @@ -13,7 +10,9 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) +// generateTargetResource converts a DiscoveredTarget into a Kubernetes Target Object based on the TargetSource Spec func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSource) *gnmicv1alpha1.Target { + // Create object instance t := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ Name: d.Name, @@ -22,29 +21,35 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou }, } + // Add Address from DiscoveredTarget t.Spec.Address = d.Address + // Add default Target Profile from the TargetSource Spec TargetProfile t.Spec.Profile = ts.Spec.TargetProfile + // Copy TargetLabels from TargetSource Spec maps.Copy(t.Labels, ts.Spec.TargetLabels) + // Handle labels from Source of Truth for k, v := range d.Labels { if strings.HasPrefix(k, ExternalLabelPrefix) { switch k { - case ExternalLabelTargetProfile: + case ExternalLabelTargetProfile: // Overwrite TargetProfile if specified by SoT t.Spec.Profile = v default: - // handle unknown label + // TODO: handle unknown label } - } else { + } else { // Copy all other labels into the Target t.Labels[k] = v } } + // Add TargetSource Label to the Target (precedence over all labels) t.Labels[LabelTargetSourceName] = ts.Name return t } +// generateEvents returns a list of DiscoveryEvents. Needed for snapshot handling to determine which devices get deleted and which applied. func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.DiscoveredTarget) []core.DiscoveryEvent { var events []core.DiscoveryEvent @@ -53,6 +58,7 @@ func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.Discovere discoveredMap[d.Name] = d } + // Create delete events for targets which are present in existing but not in discovered for _, e := range existing { if _, found := discoveredMap[e.Name]; !found { events = append(events, core.DiscoveryEvent{ @@ -64,6 +70,7 @@ func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.Discovere } } + // Create apply events for all targets in discovered for _, d := range discovered { events = append(events, core.DiscoveryEvent{ Target: d, From 41c1fec8ef46c89ae2b5e1a9f463e774a5d0283e Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 09:59:48 -0600 Subject: [PATCH 160/246] added more comments --- internal/controller/discovery/mapper.go | 1 + internal/controller/discovery/message_processor.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 6cc28ae..36ce541 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -81,6 +81,7 @@ func generateEvents(existing []gnmicv1alpha1.Target, discovered []core.Discovere return events } +// normalizeTarget adds the prefix to the target name for identification in Kubernetes func normalizeTarget(t core.DiscoveredTarget, tsName string) core.DiscoveredTarget { t.Name = tsName + "-" + t.Name return t diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 314684a..2899940 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -94,6 +94,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { return nil } +// processMessage handles all of the incoming messages from the channel func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err @@ -215,6 +216,7 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } +// processEvent handles a single DiscoveryEvent message. If a snapshot is in the queue, the events get deferred and applied after. func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events if m.activeSnapshot != nil { @@ -238,6 +240,7 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover return err } +// applySnapshot is in charge of getting the Events for the discovered targets and applying them through applyEvent func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): @@ -318,6 +321,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot } } + // Because of idempotency, allTargets = desired state = targets existing in Kubernetes. Overwrites the counter to "reset" it. m.targetCount = int32(len(allTargets)) m.updateStatus(logger) From 4a469085c184ff03bd7785a846ce5ec8d41287e7 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 10:01:55 -0600 Subject: [PATCH 161/246] added initial targetCount fetch to deal with process restarts --- internal/controller/discovery/message_processor.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 2899940..9d26be6 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -57,6 +57,12 @@ func (m *MessageProcessor) Run(ctx context.Context) error { logger.Info("Message processor started") + if existing, err := fetchExistingTargets(m.ctx, m.client, m.targetSource); err != nil { + logger.Error(err, "error fetching existing targets") + } else { + m.targetCount = int32(len(existing)) + } + for m.ctx.Err() == nil { select { case batch, ok := <-m.in: From a0b55b3278ff0edfba282109163cd0baae18bb65 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 10:02:09 -0600 Subject: [PATCH 162/246] added comment --- internal/controller/discovery/message_processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 9d26be6..9f8aa04 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -57,6 +57,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { logger.Info("Message processor started") + // Update internal counter in case of a process restart if existing, err := fetchExistingTargets(m.ctx, m.client, m.targetSource); err != nil { logger.Error(err, "error fetching existing targets") } else { From 426e27ae1e39a33a963d6e24ea25362b56683f6f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Mon, 4 May 2026 18:15:49 +0000 Subject: [PATCH 163/246] fix: use defined variable --- internal/controller/targetsource_controller.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8207103..c65e254 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -186,10 +186,7 @@ func (r *TargetSourceReconciler) startDiscovery( targetSource, targetChannel, ) - loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{ - TargetsourceNN: key, - ChunkSize: r.ChunkSize, - }, + loader, loaderConfig, err := discovery.NewLoader(loaderConfig, &targetSource.Spec, ) if err != nil { From d14b10cb8b2f6dfeb7ef5419c0b02e3d109d0f7e Mon Sep 17 00:00:00 2001 From: mcdillson Date: Mon, 4 May 2026 23:41:51 +0000 Subject: [PATCH 164/246] fixed cluster variable for netbox deployment --- netbox.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox.mk b/netbox.mk index 555c2f1..a133554 100644 --- a/netbox.mk +++ b/netbox.mk @@ -23,7 +23,7 @@ ifndef NETBOX_CLUSTER_NAME $(error NETBOX_CLUSTER_NAME is required. Usage: make netbox-deploy-cluster NETBOX_CLUSTER_NAME=cluster-name) endif kind get clusters | grep -q "$(NETBOX_CLUSTER_NAME)" || kind create cluster --name $(NETBOX_CLUSTER_NAME) - kubectl config use-context kind-$(CLUSTER_NAME) + kubectl config use-context kind-$(NETBOX_CLUSTER_NAME) .PHONY: netbox-undeploy netbox-undeploy: ## Undeploy the netbox cluster From f04c13024ee9f36f77e3586fcecb578d610ead41 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Mon, 4 May 2026 23:42:36 +0000 Subject: [PATCH 165/246] added netbox integration test setup --- test.mk | 23 +++++++++++ .../netbox/initializers/device-roles.yaml | 3 ++ .../netbox/initializers/device-types.yaml | 16 ++++++++ .../netbox/initializers/devices.yaml | 36 +++++++++++++++++ .../netbox/initializers/interfaces.yaml | 40 +++++++++++++++++++ .../netbox/initializers/ip-addresses.yaml | 24 +++++++++++ .../netbox/initializers/manufacturers.yaml | 4 ++ .../netbox/initializers/sites.yaml | 2 + test/integration/t2.clab.yaml | 28 +++++++++++++ 9 files changed, 176 insertions(+) create mode 100644 test/integration/netbox/initializers/device-roles.yaml create mode 100644 test/integration/netbox/initializers/device-types.yaml create mode 100644 test/integration/netbox/initializers/devices.yaml create mode 100644 test/integration/netbox/initializers/interfaces.yaml create mode 100644 test/integration/netbox/initializers/ip-addresses.yaml create mode 100644 test/integration/netbox/initializers/manufacturers.yaml create mode 100644 test/integration/netbox/initializers/sites.yaml create mode 100644 test/integration/t2.clab.yaml diff --git a/test.mk b/test.mk index 3497c2b..e998a6a 100644 --- a/test.mk +++ b/test.mk @@ -85,6 +85,29 @@ deploy-test-topology: ## Deploy a test topology for testing undeploy-test-topology: ## Undeploy a test topology for testing sudo containerlab destroy -t test/integration/t1.clab.yaml -c +.PHONY: deploy-test-netbox-instance +deploy-test-netbox-instance: + $(MAKE) netbox-setup \ + NETBOX_CLUSTER_NAME=test-kind \ + NETBOX_PASSWORD=Netbox123 + +.PHONY: deploy-test-netbox-instance +deploy-test-netbox-topology: + sudo containerlab deploy -t test/integration/t2.clab.yaml -c + kubectl port-forward svc/netbox 8082:80 -n netbox --context kind-test-kind --address=0.0.0.0 >/dev/null 2>&1 & + +.PHONY: sync-netbox-test-data +sync-netbox-test-data: + $(MAKE) netbox-sync-data \ + NETBOX_CLUSTER_NAME=test-kind \ + NETBOX_URL=http://localhost:8082 \ + NETBOX_INIT=test/integration/netbox/initializers + +.PHONY: undeploy-test-netbox-instance +undeploy-test-netbox-instance: + $(MAKE) netbox-delete \ + NETBOX_CLUSTER_NAME=test-kind + .PHONY: apply-test-targets apply-test-targets: ## Apply the test targets for testing kubectl apply -f test/integration/resources/targets/profile diff --git a/test/integration/netbox/initializers/device-roles.yaml b/test/integration/netbox/initializers/device-roles.yaml new file mode 100644 index 0000000..9167dab --- /dev/null +++ b/test/integration/netbox/initializers/device-roles.yaml @@ -0,0 +1,3 @@ +- name: Router + slug: router + color: ff0000 diff --git a/test/integration/netbox/initializers/device-types.yaml b/test/integration/netbox/initializers/device-types.yaml new file mode 100644 index 0000000..a6279ed --- /dev/null +++ b/test/integration/netbox/initializers/device-types.yaml @@ -0,0 +1,16 @@ +- model: ixr-d2l + slug: arista-ixr-d2l + manufacturer: + name: Arista +- model: ixr-d2l + slug: nokia-ixr-d2l + manufacturer: + name: Nokia +- model: ixr-d2l-leaf + slug: nokia-ixr-d2l-leaf + manufacturer: + name: Nokia +- model: ixr-d3l + slug: nokia-ixr-d3l + manufacturer: + name: Nokia diff --git a/test/integration/netbox/initializers/devices.yaml b/test/integration/netbox/initializers/devices.yaml new file mode 100644 index 0000000..17ed036 --- /dev/null +++ b/test/integration/netbox/initializers/devices.yaml @@ -0,0 +1,36 @@ +- name: ceos1 + role: + slug: router + manufacturer: + name: Arista + device_type: + slug: arista-ixr-d2l + site: + name: Lab +- name: leaf1 + role: + slug: router + manufacturer: + name: Nokia + device_type: + slug: nokia-ixr-d2l + site: + name: Lab +- name: leaf2 + role: + slug: router + manufacturer: + name: Nokia + device_type: + slug: nokia-ixr-d2l-leaf + site: + name: Lab +- name: spine1 + role: + slug: router + manufacturer: + name: Nokia + device_type: + slug: nokia-ixr-d3l + site: + name: Lab diff --git a/test/integration/netbox/initializers/interfaces.yaml b/test/integration/netbox/initializers/interfaces.yaml new file mode 100644 index 0000000..05e8d24 --- /dev/null +++ b/test/integration/netbox/initializers/interfaces.yaml @@ -0,0 +1,40 @@ +- device: + name: spine1 + name: e1-1 + type: 1000base-t +- device: + name: leaf1 + name: e1-49 + type: 1000base-t +- device: + name: spine1 + name: e1-2 + type: 1000base-t +- device: + name: leaf2 + name: e1-49 + type: 1000base-t +- device: + name: spine1 + name: e1-3 + type: 1000base-t +- device: + name: ceos1 + name: eth1 + type: 1000base-t +- device: + name: spine1 + name: mgmt0 + type: 1000base-t +- device: + name: leaf1 + name: mgmt0 + type: 1000base-t +- device: + name: leaf2 + name: mgmt0 + type: 1000base-t +- device: + name: ceos1 + name: mgmt0 + type: 1000base-t diff --git a/test/integration/netbox/initializers/ip-addresses.yaml b/test/integration/netbox/initializers/ip-addresses.yaml new file mode 100644 index 0000000..de95cc8 --- /dev/null +++ b/test/integration/netbox/initializers/ip-addresses.yaml @@ -0,0 +1,24 @@ +- address: 172.18.1.10/32 + assigned_object: + device: + name: spine1 + name: mgmt0 + status: active + primary: true + dns_name: t2-nodes-spine1 +- address: 172.18.1.11/32 + assigned_object: + device: + name: leaf1 + name: mgmt0 + status: active + primary: true + dns_name: t2-nodes-leaf1 +- address: 172.18.1.12/32 + assigned_object: + device: + name: leaf2 + name: mgmt0 + status: active + primary: true + dns_name: t2-nodes-leaf2 diff --git a/test/integration/netbox/initializers/manufacturers.yaml b/test/integration/netbox/initializers/manufacturers.yaml new file mode 100644 index 0000000..68627af --- /dev/null +++ b/test/integration/netbox/initializers/manufacturers.yaml @@ -0,0 +1,4 @@ +- name: Nokia + slug: nokia +- name: Arista + slug: arista diff --git a/test/integration/netbox/initializers/sites.yaml b/test/integration/netbox/initializers/sites.yaml new file mode 100644 index 0000000..bc8ed18 --- /dev/null +++ b/test/integration/netbox/initializers/sites.yaml @@ -0,0 +1,2 @@ +- name: Lab + slug: lab diff --git a/test/integration/t2.clab.yaml b/test/integration/t2.clab.yaml new file mode 100644 index 0000000..f79f63c --- /dev/null +++ b/test/integration/t2.clab.yaml @@ -0,0 +1,28 @@ +name: t2 + +mgmt: + network: kind + +topology: + defaults: + kind: nokia_srlinux + image: ghcr.io/nokia/srlinux:25.10.1 + + kinds: + nokia_srlinux: + image: ghcr.io/nokia/srlinux:25.10.1 + type: ixr-d2l + + nodes: + spine1: + type: ixr-d3l + mgmt-ipv4: 172.18.1.10 + + leaf1: + mgmt-ipv4: 172.18.1.11 + leaf2: + mgmt-ipv4: 172.18.1.12 + + links: + - endpoints: ["spine1:e1-1", "leaf1:e1-49"] + - endpoints: ["spine1:e1-2", "leaf2:e1-49"] From 765c6edfcd277f9ee17c8f2c65405837bcabdaa5 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Mon, 4 May 2026 23:56:04 +0000 Subject: [PATCH 166/246] renamed sync-test-netbox-data --- test.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.mk b/test.mk index e998a6a..a43da30 100644 --- a/test.mk +++ b/test.mk @@ -97,7 +97,7 @@ deploy-test-netbox-topology: kubectl port-forward svc/netbox 8082:80 -n netbox --context kind-test-kind --address=0.0.0.0 >/dev/null 2>&1 & .PHONY: sync-netbox-test-data -sync-netbox-test-data: +sync-test-netbox-data: $(MAKE) netbox-sync-data \ NETBOX_CLUSTER_NAME=test-kind \ NETBOX_URL=http://localhost:8082 \ From 935c49f051a8ed563f8f83e4241cee9caedcb0ed Mon Sep 17 00:00:00 2001 From: mcdillson Date: Mon, 4 May 2026 23:57:29 +0000 Subject: [PATCH 167/246] moved netbox clab topology into netbox folder --- test/integration/{t2.clab.yaml => netbox/netbox.clab.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/{t2.clab.yaml => netbox/netbox.clab.yaml} (100%) diff --git a/test/integration/t2.clab.yaml b/test/integration/netbox/netbox.clab.yaml similarity index 100% rename from test/integration/t2.clab.yaml rename to test/integration/netbox/netbox.clab.yaml From 5fb77005116167e9a691823bd7c36a0b939b17f2 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Mon, 4 May 2026 23:58:50 +0000 Subject: [PATCH 168/246] added targetsource integration resource --- test.mk | 4 ++++ test/integration/resources/targetsources/netbox.yaml | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 test/integration/resources/targetsources/netbox.yaml diff --git a/test.mk b/test.mk index a43da30..c064825 100644 --- a/test.mk +++ b/test.mk @@ -108,6 +108,10 @@ undeploy-test-netbox-instance: $(MAKE) netbox-delete \ NETBOX_CLUSTER_NAME=test-kind +.PHONY apply-test-targetsources +apply-test-targetsources: + kubectl apply -f test/integration/resources/targetsources + .PHONY: apply-test-targets apply-test-targets: ## Apply the test targets for testing kubectl apply -f test/integration/resources/targets/profile diff --git a/test/integration/resources/targetsources/netbox.yaml b/test/integration/resources/targetsources/netbox.yaml new file mode 100644 index 0000000..0b8fd23 --- /dev/null +++ b/test/integration/resources/targetsources/netbox.yaml @@ -0,0 +1,11 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: netbox-ts +spec: + provider: + http: + url: http://localhost:8082/targets + targetLabels: + integration-test: netbox + profile: netbox-default \ No newline at end of file From 1c349ae28ee27c502f6c4c4c8b70fa5699dd7629 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Tue, 5 May 2026 00:01:50 +0000 Subject: [PATCH 169/246] added comments + fixed netbox test topology path --- test.mk | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test.mk b/test.mk index c064825..1060623 100644 --- a/test.mk +++ b/test.mk @@ -86,30 +86,34 @@ undeploy-test-topology: ## Undeploy a test topology for testing sudo containerlab destroy -t test/integration/t1.clab.yaml -c .PHONY: deploy-test-netbox-instance -deploy-test-netbox-instance: +deploy-test-netbox-instance: ## Deploy the test netbox instance for testing $(MAKE) netbox-setup \ NETBOX_CLUSTER_NAME=test-kind \ NETBOX_PASSWORD=Netbox123 .PHONY: deploy-test-netbox-instance -deploy-test-netbox-topology: - sudo containerlab deploy -t test/integration/t2.clab.yaml -c +deploy-test-netbox-topology: ## Deploy the netbox test topology for testing + sudo containerlab deploy -t test/integration/netbox/netbox.clab.yaml -c kubectl port-forward svc/netbox 8082:80 -n netbox --context kind-test-kind --address=0.0.0.0 >/dev/null 2>&1 & .PHONY: sync-netbox-test-data -sync-test-netbox-data: +sync-test-netbox-data: ## Sync the netbox instance with the test topology for testing $(MAKE) netbox-sync-data \ NETBOX_CLUSTER_NAME=test-kind \ NETBOX_URL=http://localhost:8082 \ NETBOX_INIT=test/integration/netbox/initializers .PHONY: undeploy-test-netbox-instance -undeploy-test-netbox-instance: +undeploy-test-netbox-instance: ## Undeploy the netbox instance from the test cluster $(MAKE) netbox-delete \ NETBOX_CLUSTER_NAME=test-kind +.PHONY: undeploy-test-netbox-topology +undeploy-test-netbox-topology: ## Undeploy the netbox test topology for testing + sudo containerlab destroy -t test/integration/netbox/netbox.clab.yaml -c + .PHONY apply-test-targetsources -apply-test-targetsources: +apply-test-targetsources: ## Apply the test targetsources for testing kubectl apply -f test/integration/resources/targetsources .PHONY: apply-test-targets From 012a6a5ed4d3db4f74c683306dfdfcdc3961fbbb Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 18:47:33 -0600 Subject: [PATCH 170/246] fixed missing separator --- test.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.mk b/test.mk index 1060623..270f846 100644 --- a/test.mk +++ b/test.mk @@ -112,7 +112,7 @@ undeploy-test-netbox-instance: ## Undeploy the netbox instance from the test clu undeploy-test-netbox-topology: ## Undeploy the netbox test topology for testing sudo containerlab destroy -t test/integration/netbox/netbox.clab.yaml -c -.PHONY apply-test-targetsources +.PHONY: apply-test-targetsources apply-test-targetsources: ## Apply the test targetsources for testing kubectl apply -f test/integration/resources/targetsources From 09aaaa4127c135eab4baf64cca1a38a861a39c73 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 18:49:00 -0600 Subject: [PATCH 171/246] fixed targetProfile key --- test/integration/resources/targetsources/netbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/resources/targetsources/netbox.yaml b/test/integration/resources/targetsources/netbox.yaml index 0b8fd23..39ab922 100644 --- a/test/integration/resources/targetsources/netbox.yaml +++ b/test/integration/resources/targetsources/netbox.yaml @@ -8,4 +8,4 @@ spec: url: http://localhost:8082/targets targetLabels: integration-test: netbox - profile: netbox-default \ No newline at end of file + targetProfile: netbox-default \ No newline at end of file From 4bcd01e644d3707a4342855b9ce632d9ab9273d3 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 4 May 2026 19:46:43 -0600 Subject: [PATCH 172/246] fixed name normalization --- internal/controller/discovery/message_processor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 9f8aa04..a918ce2 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -119,7 +119,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc ) for i := range msg.Targets { - msg.Targets[i] = normalizeTarget(msg.Targets[i], m.targetSource.Namespace) + msg.Targets[i] = normalizeTarget(msg.Targets[i], m.targetSource.Name) } return m.processSnapshot(ctx, msg, logger) @@ -132,7 +132,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc "target", msg.Target.Name, ) - msg.Target = normalizeTarget(msg.Target, m.targetSource.Namespace) + msg.Target = normalizeTarget(msg.Target, m.targetSource.Name) return m.processEvent(ctx, msg, logger) default: From b9ab471f42d6348c83638e993b29809ec4bbf5d4 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Tue, 5 May 2026 08:27:45 -0600 Subject: [PATCH 173/246] eliminated recursive make calls --- test.mk | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/test.mk b/test.mk index 270f846..b2d550e 100644 --- a/test.mk +++ b/test.mk @@ -4,6 +4,8 @@ GNMIC_VERSION ?= 0.44.1 KUBECTL_VERSION ?= v1.31.0 TEST_CLUSTER_NAME ?= test-kind CERT_MANAGER_VERSION ?= v1.19.3 +NETBOX_TEST_PORT ?= 8082 + .PHONY: install-kubectl install-kubectl: ## Install kubectl if not present @@ -86,27 +88,24 @@ undeploy-test-topology: ## Undeploy a test topology for testing sudo containerlab destroy -t test/integration/t1.clab.yaml -c .PHONY: deploy-test-netbox-instance -deploy-test-netbox-instance: ## Deploy the test netbox instance for testing - $(MAKE) netbox-setup \ - NETBOX_CLUSTER_NAME=test-kind \ - NETBOX_PASSWORD=Netbox123 +deploy-test-netbox-instance: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Deploy the test netbox instance for testing +deploy-test-netbox-instance: NETBOX_PASSWORD=Netbox123 +deploy-test-netbox-instance: netbox-setup .PHONY: deploy-test-netbox-instance deploy-test-netbox-topology: ## Deploy the netbox test topology for testing sudo containerlab deploy -t test/integration/netbox/netbox.clab.yaml -c - kubectl port-forward svc/netbox 8082:80 -n netbox --context kind-test-kind --address=0.0.0.0 >/dev/null 2>&1 & + kubectl port-forward svc/netbox $(NETBOX_TEST_PORT):80 -n netbox --context kind-$(TEST_CLUSTER_NAME) --address=0.0.0.0 >/dev/null 2>&1 & .PHONY: sync-netbox-test-data -sync-test-netbox-data: ## Sync the netbox instance with the test topology for testing - $(MAKE) netbox-sync-data \ - NETBOX_CLUSTER_NAME=test-kind \ - NETBOX_URL=http://localhost:8082 \ - NETBOX_INIT=test/integration/netbox/initializers +sync-test-netbox-data: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Sync the netbox instance with the test topology for testing +sync-test-netbox-data: NETBOX_URL=http://localhost:$(NETBOX_TEST_PORT) +sync-test-netbox-data: NETBOX_INIT=test/integration/netbox/initializers +sync-test-netbox-data: netbox-sync-data .PHONY: undeploy-test-netbox-instance -undeploy-test-netbox-instance: ## Undeploy the netbox instance from the test cluster - $(MAKE) netbox-delete \ - NETBOX_CLUSTER_NAME=test-kind +undeploy-test-netbox-instance: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Undeploy the netbox instance from the test cluster +undeploy-test-netbox-instance: netbox-delete .PHONY: undeploy-test-netbox-topology undeploy-test-netbox-topology: ## Undeploy the netbox test topology for testing From 1a6239af453bbf8a1d8b12365a531f22671fdbbc Mon Sep 17 00:00:00 2001 From: mcdillson Date: Tue, 5 May 2026 14:32:54 +0000 Subject: [PATCH 174/246] added recursive clab folder to gitignore --- .gitignore | 2 +- test/integration/netbox/netbox.clab.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 29d31af..ee83f89 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ notes/ docs/public docs/resources/_gen/ docs/.hugo_build.lock -test/integration/clab-* \ No newline at end of file +test/integration/**/clab-* \ No newline at end of file diff --git a/test/integration/netbox/netbox.clab.yaml b/test/integration/netbox/netbox.clab.yaml index f79f63c..ddd1705 100644 --- a/test/integration/netbox/netbox.clab.yaml +++ b/test/integration/netbox/netbox.clab.yaml @@ -1,4 +1,4 @@ -name: t2 +name: netbox mgmt: network: kind From 4f879aa50cfd86f67ae87b88213ce8aafae53293 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Tue, 5 May 2026 14:34:00 +0000 Subject: [PATCH 175/246] added resources for http static server test --- lab/dev/http/targets.json | 13 ++++++++++ .../integration/http/resources/configmap.yaml | 22 +++++++++++++++++ .../http/resources/deployment.yaml | 24 +++++++++++++++++++ test/integration/http/resources/service.yaml | 10 ++++++++ .../resources/targetsources/http.yaml | 11 +++++++++ 5 files changed, 80 insertions(+) create mode 100644 lab/dev/http/targets.json create mode 100644 test/integration/http/resources/configmap.yaml create mode 100644 test/integration/http/resources/deployment.yaml create mode 100644 test/integration/http/resources/service.yaml create mode 100644 test/integration/resources/targetsources/http.yaml diff --git a/lab/dev/http/targets.json b/lab/dev/http/targets.json new file mode 100644 index 0000000..882faae --- /dev/null +++ b/lab/dev/http/targets.json @@ -0,0 +1,13 @@ +[ + { + "address": "10.0.0.1:57000", + "name": "router1" + }, + { + "address": "10.0.0.2:57000", + "name": "router2", + "labels": { + "test": "asdf" + } + } +] \ No newline at end of file diff --git a/test/integration/http/resources/configmap.yaml b/test/integration/http/resources/configmap.yaml new file mode 100644 index 0000000..0e8b35c --- /dev/null +++ b/test/integration/http/resources/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: targets-config +data: + targets.json: | + [ + { + "address": "10.0.0.1:57000", + "name": "router1", + "labels": { + "label1": "test" + } + }, + { + "address": "10.0.0.2:57000", + "name": "router2", + "labels": { + "label2": "test2" + } + } + ] \ No newline at end of file diff --git a/test/integration/http/resources/deployment.yaml b/test/integration/http/resources/deployment.yaml new file mode 100644 index 0000000..3dc1f61 --- /dev/null +++ b/test/integration/http/resources/deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: targets-server +spec: + replicas: 1 + selector: + matchLabels: + app: targets-server + template: + metadata: + labels: + app: targets-server + spec: + containers: + - name: nginx + image: nginx:alpine + volumeMounts: + - name: data + mountPath: /usr/share/nginx/html + volumes: + - name: data + configMap: + name: targets-config \ No newline at end of file diff --git a/test/integration/http/resources/service.yaml b/test/integration/http/resources/service.yaml new file mode 100644 index 0000000..03f0efa --- /dev/null +++ b/test/integration/http/resources/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: targets +spec: + selector: + app: targets-server + ports: + - port: 80 + targetPort: 80 \ No newline at end of file diff --git a/test/integration/resources/targetsources/http.yaml b/test/integration/resources/targetsources/http.yaml new file mode 100644 index 0000000..199fcf3 --- /dev/null +++ b/test/integration/resources/targetsources/http.yaml @@ -0,0 +1,11 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: http-ts +spec: + provider: + http: + url: http://targets.default.svc/targets.json + targetLabels: + integration-test: http + targetProfile: http-default \ No newline at end of file From a53440605e5345b20c2182c67fe5e3b2d0ea32c8 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Tue, 5 May 2026 14:38:25 +0000 Subject: [PATCH 176/246] added make targets for http testing pod --- test.mk | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.mk b/test.mk index b2d550e..f92c4ec 100644 --- a/test.mk +++ b/test.mk @@ -87,6 +87,14 @@ deploy-test-topology: ## Deploy a test topology for testing undeploy-test-topology: ## Undeploy a test topology for testing sudo containerlab destroy -t test/integration/t1.clab.yaml -c +.PHONY: deploy-test-http-server +deploy-test-http-server: ## Deploy a test http pod with a static file inventory for testing + kubectl apply -f test/integration/http/resources/ + +.PHONY: undeploy-test-http-server +undeploy-test-http-server: ## Undeploy the http pod for testing + kubectl delete -f test/integration/http/resources/ + .PHONY: deploy-test-netbox-instance deploy-test-netbox-instance: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Deploy the test netbox instance for testing deploy-test-netbox-instance: NETBOX_PASSWORD=Netbox123 From a6e449d04266d5c71f0503ed5753b51835f3b58b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 6 May 2026 07:57:31 +0000 Subject: [PATCH 177/246] load spec into loader --- internal/controller/discovery/loaders.go | 2 +- internal/controller/discovery/loaders/http/loader.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 7f2c656..c75c5fa 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -14,7 +14,7 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec switch { case spec.Provider.HTTP != nil: cfg.AcceptPush = spec.Provider.HTTP.AcceptPush - return http.New(cfg), cfg, nil + return http.New(cfg, *spec.Provider.HTTP), cfg, nil case spec.Provider.Consul != nil: return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index cf081b6..b6941e9 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -8,11 +8,14 @@ import ( "net/http" "time" + "github.com/google/uuid" + "k8s.io/kube-openapi/pkg/validation/spec" + "sigs.k8s.io/controller-runtime/pkg/log" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" - "github.com/google/uuid" ) const ( @@ -23,11 +26,12 @@ const ( // Loader implements the HTTP pull discovery mechanism type Loader struct { commonCfg core.CommonLoaderConfig + spec *gnmicv1alpha1.HTTPConfig } // New instantiates the http loader with the provided config -func New(cfg core.CommonLoaderConfig) core.Loader { - return &Loader{commonCfg: cfg} +func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader { + return &Loader{commonCfg: cfg, spec: &httpConfig} } func (l *Loader) Name() string { @@ -50,7 +54,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger.Info("HTTP loader started") // Input Validation of spec - if spec.Provider == nil || spec.Provider.HTTP == nil { + if spec. == nil { return errors.New("HTTP loader requires spec.provider.http to be set") } From e908953342531ffb2b0053e288d03095cca0a8dd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 6 May 2026 08:13:59 +0000 Subject: [PATCH 178/246] update httpconfig --- api/v1alpha1/targetsource_types.go | 87 +++++- api/v1alpha1/zz_generated.deepcopy.go | 261 +++++++++++++++-- .../operator.gnmic.dev_targetsources.yaml | 275 +++++++++++++++++- 3 files changed, 588 insertions(+), 35 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 3d69743..ab48614 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -32,22 +33,98 @@ type TargetSourceSpec struct { TargetProfile string `json:"targetProfile"` } -// +kubebuilder:validation:ExactlyOneOf=http;consul +// +kubebuilder:validation:ExactlyOneOf=http type ProviderSpec struct { - HTTP *HTTPConfig `json:"http,omitempty"` - Consul *ConsulConfig `json:"consul,omitempty"` + HTTP *HTTPConfig `json:"http,omitempty"` +} + +type WebhookSpec struct { + Enabled *bool `json:"enabled,omitempty"` } type HTTPConfig struct { // +kubebuilder:validation:MinLength=1 URL string `json:"url"` // +kubebuilder:validation:Optional + Authorization *AuthorizationSpec `json:"authorization,omitempty"` + // +kubebuilder:validation:Optional + PollInterval *metav1.Duration `json:"interval,omitempty"` + // +kubebuilder:validation:Optional + Timeout *metav1.Duration `json:"timeout,omitempty"` + // +kubebuilder:validation:Optional + TLS *ClientTLSConfig `json:"tls,omitempty"` + // +kubebuilder:validation:Optional + Pagination *PaginationSpec `json:"pagination,omitempty"` + // +kubebuilder:validation:Optional + ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"` + // +kubebuilder:validation:Optional AcceptPush bool `json:"acceptPush,omitempty"` } -type ConsulConfig struct { +type ClientTLSConfig struct { + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + CASecretRef *corev1.SecretKeySelector `json:"caSecretRef,omitempty"` +} + +// +kubebuilder:validation:ExactlyOneOf=basic;bearer;jwt;token +type AuthorizationSpec struct { + Basic *BasicAuthSpec `json:"basic,omitempty"` + Bearer *BearerAuthSpec `json:"bearer,omitempty"` + Token *TokenAuthSpec `json:"token,omitempty"` + JWT *JWTAuthSpec `json:"jwt,omitempty"` +} + +// Enforce EITHER inline creds OR secret ref +// +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix" +type BasicAuthSpec struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"` +} + +// +kubebuilder:validation:ExactlyOneOf=token;tokenSecretRef +type BearerAuthSpec struct { + Token string `json:"token,omitempty"` + TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both" +type TokenAuthSpec struct { // +kubebuilder:validation:MinLength=1 - URL string `json:"url,omitempty"` + Scheme string `json:"scheme"` + Token string `json:"token,omitempty"` + TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` +} + +// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" +// +kubebuilder:validation:XValidation:rule="!( (has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)) )",message="static JWT token and generated JWT configuration cannot be combined" +// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != ”",message="algorithm must be specified when generating a JWT" +type JWTAuthSpec struct { + // Static pre-generated JWT + Token string `json:"token,omitempty"` + TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` + // Optional: generate JWT dynamically + Claims map[string]string `json:"claims,omitempty"` + SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"` + // HS256, RS256, ES256, etc. + Algorithm string `json:"algorithm,omitempty"` + TTL *metav1.Duration `json:"ttl,omitempty"` +} + +type PaginationSpec struct { + Enabled bool `json:"enabled"` + // Example: "results" + ItemsField string `json:"itemsField,omitempty"` + // Example: "next" + NextField string `json:"nextField,omitempty"` +} + +// JSONPath-style expressions +type ResponseMappingSpec struct { + Name string `json:"name"` + Address string `json:"address"` + Port string `json:"port,omitempty"` + Labels map[string]string `json:"labels,omitempty"` } // TargetSourceStatus defines the observed state of TargetSource diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 61e81fd..602ede5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,101 @@ func (in *APIConfig) DeepCopy() *APIConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) { + *out = *in + if in.Basic != nil { + in, out := &in.Basic, &out.Basic + *out = new(BasicAuthSpec) + (*in).DeepCopyInto(*out) + } + if in.Bearer != nil { + in, out := &in.Bearer, &out.Bearer + *out = new(BearerAuthSpec) + (*in).DeepCopyInto(*out) + } + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(TokenAuthSpec) + (*in).DeepCopyInto(*out) + } + if in.JWT != nil { + in, out := &in.JWT, &out.JWT + *out = new(JWTAuthSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationSpec. +func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec { + if in == nil { + return nil + } + out := new(AuthorizationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) { + *out = *in + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuthSpec. +func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec { + if in == nil { + return nil + } + out := new(BasicAuthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BearerAuthSpec) DeepCopyInto(out *BearerAuthSpec) { + *out = *in + if in.TokenSecretRef != nil { + in, out := &in.TokenSecretRef, &out.TokenSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BearerAuthSpec. +func (in *BearerAuthSpec) DeepCopy() *BearerAuthSpec { + if in == nil { + return nil + } + out := new(BearerAuthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) { + *out = *in + if in.CASecretRef != nil { + in, out := &in.CASecretRef, &out.CASecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientTLSConfig. +func (in *ClientTLSConfig) DeepCopy() *ClientTLSConfig { + if in == nil { + return nil + } + out := new(ClientTLSConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cluster) DeepCopyInto(out *Cluster) { *out = *in @@ -213,21 +308,6 @@ func (in *ClusterTargetState) DeepCopy() *ClusterTargetState { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConsulConfig) DeepCopyInto(out *ConsulConfig) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsulConfig. -func (in *ConsulConfig) DeepCopy() *ConsulConfig { - if in == nil { - return nil - } - out := new(ConsulConfig) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GRPCKeepAliveConfig) DeepCopyInto(out *GRPCKeepAliveConfig) { *out = *in @@ -273,6 +353,36 @@ func (in *GRPCTunnelConfig) DeepCopy() *GRPCTunnelConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) { *out = *in + if in.Authorization != nil { + in, out := &in.Authorization, &out.Authorization + *out = new(AuthorizationSpec) + (*in).DeepCopyInto(*out) + } + if in.PollInterval != nil { + in, out := &in.PollInterval, &out.PollInterval + *out = new(metav1.Duration) + **out = **in + } + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(metav1.Duration) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(ClientTLSConfig) + (*in).DeepCopyInto(*out) + } + if in.Pagination != nil { + in, out := &in.Pagination, &out.Pagination + *out = new(PaginationSpec) + **out = **in + } + if in.ResponseMapping != nil { + in, out := &in.ResponseMapping, &out.ResponseMapping + *out = new(ResponseMappingSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig. @@ -414,6 +524,43 @@ func (in *InputStatus) DeepCopy() *InputStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { + *out = *in + if in.TokenSecretRef != nil { + in, out := &in.TokenSecretRef, &out.TokenSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.Claims != nil { + in, out := &in.Claims, &out.Claims + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.SigningKeySecretRef != nil { + in, out := &in.SigningKeySecretRef, &out.SigningKeySecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.TTL != nil { + in, out := &in.TTL, &out.TTL + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthSpec. +func (in *JWTAuthSpec) DeepCopy() *JWTAuthSpec { + if in == nil { + return nil + } + out := new(JWTAuthSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Output) DeepCopyInto(out *Output) { *out = *in @@ -587,6 +734,21 @@ func (in *OutputStatus) DeepCopy() *OutputStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec. +func (in *PaginationSpec) DeepCopy() *PaginationSpec { + if in == nil { + return nil + } + out := new(PaginationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pipeline) DeepCopyInto(out *Pipeline) { *out = *in @@ -824,12 +986,7 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP *out = new(HTTPConfig) - **out = **in - } - if in.Consul != nil { - in, out := &in.Consul, &out.Consul - *out = new(ConsulConfig) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -843,6 +1000,28 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResponseMappingSpec. +func (in *ResponseMappingSpec) DeepCopy() *ResponseMappingSpec { + if in == nil { + return nil + } + out := new(ResponseMappingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -1384,6 +1563,26 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) { + *out = *in + if in.TokenSecretRef != nil { + in, out := &in.TokenSecretRef, &out.TokenSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenAuthSpec. +func (in *TokenAuthSpec) DeepCopy() *TokenAuthSpec { + if in == nil { + return nil + } + out := new(TokenAuthSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TunnelTargetPolicy) DeepCopyInto(out *TunnelTargetPolicy) { *out = *in @@ -1477,3 +1676,23 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. +func (in *WebhookSpec) DeepCopy() *WebhookSpec { + if in == nil { + return nil + } + out := new(WebhookSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 37d6919..23360c5 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -41,16 +41,274 @@ spec: properties: provider: properties: - consul: - properties: - url: - minLength: 1 - type: string - type: object http: properties: acceptPush: type: boolean + authorization: + properties: + basic: + description: Enforce EITHER inline creds OR secret ref + properties: + credentialsSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + password: + type: string + username: + type: string + type: object + x-kubernetes-validations: + - message: either credentialsSecretRef OR both username + and password must be set, but not a mix + rule: (has(self.credentialsSecretRef) && !has(self.username) + && !has(self.password)) || (!has(self.credentialsSecretRef) + && has(self.username) && has(self.password)) + bearer: + properties: + token: + type: string + tokenSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [token tokenSecretRef] + must be set + rule: '[has(self.token),has(self.tokenSecretRef)].filter(x,x==true).size() + == 1' + jwt: + properties: + algorithm: + description: HS256, RS256, ES256, etc. + type: string + claims: + additionalProperties: + type: string + description: 'Optional: generate JWT dynamically' + type: object + signingKeySecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + token: + description: Static pre-generated JWT + type: string + tokenSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + ttl: + type: string + type: object + x-kubernetes-validations: + - message: static JWT token and generated JWT configuration + cannot be combined + rule: '!((has(self.token) || has(self.tokenSecretRef)) + && (has(self.signingKeySecretRef) || has(self.claims)))' + - message: static JWT token and generated JWT configuration + cannot be combined + rule: '!( (has(self.token) || has(self.tokenSecretRef)) + && (has(self.signingKeySecretRef) || has(self.claims)) + )' + - message: algorithm must be specified when generating + a JWT + rule: '!has(self.signingKeySecretRef) || self.algorithm + != ”' + token: + properties: + scheme: + minLength: 1 + type: string + token: + type: string + tokenSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + required: + - scheme + type: object + x-kubernetes-validations: + - message: either token or tokenSecretRef must be set, + but not both + rule: has(self.token) != has(self.tokenSecretRef) + type: object + x-kubernetes-validations: + - message: exactly one of the fields in [basic bearer jwt + token] must be set + rule: '[has(self.basic),has(self.bearer),has(self.jwt),has(self.token)].filter(x,x==true).size() + == 1' + interval: + type: string + mapping: + description: JSONPath-style expressions + properties: + address: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + port: + type: string + required: + - address + - name + type: object + pagination: + properties: + enabled: + type: boolean + itemsField: + description: 'Example: "results"' + type: string + nextField: + description: 'Example: "next"' + type: string + required: + - enabled + type: object + timeout: + type: string + tls: + properties: + caSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + insecureSkipVerify: + type: boolean + type: object url: minLength: 1 type: string @@ -59,9 +317,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: exactly one of the fields in [http consul] must be set - rule: '[has(self.http),has(self.consul)].filter(x,x==true).size() - == 1' + - message: exactly one of the fields in [http] must be set + rule: '[has(self.http)].filter(x,x==true).size() == 1' targetLabels: additionalProperties: type: string From 85278df46d3d8f217928f9783d631b78cf1daec8 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 6 May 2026 08:14:09 +0000 Subject: [PATCH 179/246] use httpconfig within loader --- internal/controller/discovery/loaders.go | 2 -- .../discovery/loaders/http/loader.go | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index c75c5fa..646fb0a 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -15,8 +15,6 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec case spec.Provider.HTTP != nil: cfg.AcceptPush = spec.Provider.HTTP.AcceptPush return http.New(cfg, *spec.Provider.HTTP), cfg, nil - case spec.Provider.Consul != nil: - return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index b6941e9..a009b76 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -3,13 +3,11 @@ package http import ( "context" "encoding/json" - "errors" "fmt" "net/http" "time" "github.com/google/uuid" - "k8s.io/kube-openapi/pkg/validation/spec" "sigs.k8s.io/controller-runtime/pkg/log" @@ -54,9 +52,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger.Info("HTTP loader started") // Input Validation of spec - if spec. == nil { - return errors.New("HTTP loader requires spec.provider.http to be set") - } + // if l.spec.URL == "nil" { + // return errors.New("HTTP loader requires spec.provider.http to be set") + // } client := &http.Client{ Timeout: defaultTimeoutSeconds * time.Second, @@ -68,7 +66,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger.Info( "HTTP polling discovery started", "interval", interval.String(), - "url", spec.Provider.HTTP.URL, + "url", l.spec.URL, ) // helper function to fetch targets and emit discovery messages @@ -76,14 +74,15 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er targets, err := l.fetchTargetsFromHTTPEndpoint( ctx, client, - spec.Provider.HTTP.URL, - spec.Provider.HTTP.Token, + l.spec.URL, + l.spec.Authorization.Token.Scheme, + l.spec.Authorization.Token.Token, ) if err != nil { logger.Error( err, "Failed to fetch targets from HTTP endpoint", - "url", spec.Provider.HTTP.URL, + "url", l.spec.URL, ) return } @@ -126,6 +125,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( ctx context.Context, client *http.Client, url string, + scheme string, token string, ) ([]core.DiscoveredTarget, error) { @@ -135,7 +135,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( } req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", "Token "+token) + req.Header.Set("Authorization", fmt.Sprintf("%s %s", scheme, token)) resp, err := client.Do(req) if err != nil { From 6c82320ff05d704910675eb011e6e05c882b0d4a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 6 May 2026 08:27:17 +0000 Subject: [PATCH 180/246] refactor --- api/v1alpha1/targetsource_types.go | 4 ++-- config/crd/bases/operator.gnmic.dev_targetsources.yaml | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index ab48614..44e605e 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -97,8 +97,8 @@ type TokenAuthSpec struct { } // +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" -// +kubebuilder:validation:XValidation:rule="!( (has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)) )",message="static JWT token and generated JWT configuration cannot be combined" -// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != ”",message="algorithm must be specified when generating a JWT" +// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" +// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" type JWTAuthSpec struct { // Static pre-generated JWT Token string `json:"token,omitempty"` diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 23360c5..65c48c0 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -194,13 +194,12 @@ spec: && (has(self.signingKeySecretRef) || has(self.claims)))' - message: static JWT token and generated JWT configuration cannot be combined - rule: '!( (has(self.token) || has(self.tokenSecretRef)) - && (has(self.signingKeySecretRef) || has(self.claims)) - )' + rule: '!((has(self.token) || has(self.tokenSecretRef)) + && (has(self.signingKeySecretRef) || has(self.claims)))' - message: algorithm must be specified when generating a JWT rule: '!has(self.signingKeySecretRef) || self.algorithm - != ”' + != ""' token: properties: scheme: From 84012af63692c50f1b94e1d8c6f85389db7216a9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 6 May 2026 09:44:55 +0000 Subject: [PATCH 181/246] change logger --- internal/apiserver/apiserver.go | 38 +++++++++++++++++++---- internal/apiserver/apiserver_test.go | 2 ++ internal/apiserver/helpers.go | 31 +++++++++---------- internal/apiserver/temp.md | 45 ++++++++-------------------- 4 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 internal/apiserver/apiserver_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8abd2ef..c6e7d4b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -8,15 +8,17 @@ package apiserver import ( "context" + "fmt" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) type APIServer struct { @@ -28,6 +30,7 @@ type APIServer struct { core.DiscoveryRegistryValue, ] chunzSize int + logger logr.Logger } type urlStruct struct { @@ -45,6 +48,7 @@ func New( discoveryChunksize int, ) (*APIServer, error) { router := gin.Default() + logger := log.Log.WithValues("component", "api-server") a := &APIServer{ Server: &http.Server{ Addr: addr, @@ -54,12 +58,15 @@ func New( clusterReconciler: clusterReconciler, DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, + logger: logger, } + logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() return a, nil } func (a *APIServer) Router() *gin.Engine { + gin.SetMode(gin.ReleaseMode) // gin logs return a.router } @@ -71,34 +78,55 @@ func (a *APIServer) routes() { // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", url.Namespace, + "cluster", url.Name, + ) + logger.Info("Received GET request for GetClusterPlan") + plan, err := a.clusterReconciler.GetClusterPlan(url.Namespace, url.Name) if err != nil { + logger.Error(err, "Failed to get cluster plan") c.String(404, err.Error()) return } + logger.Info("Successfully returned cluster plan") c.JSON(200, plan) } // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { - logger.Info("received POST request for CreateTargets.") url := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", url.Namespace, + "targetsource", url.Name, + ) + logger.Info("Received POST request for CreateTargets") + var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { + logger.Error(err, "Failed to bind request payload") c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } registry, ok := a.DiscoveryRegistry.Get(getKey(url)) if !ok { - logger.Error("TargetSource ", url.Namespace, "/", url.Name, "does not exist.") + err := fmt.Errorf("targetSource %s/%s does not exist", url.Namespace, url.Name) + logger.Error(err, "TargetSource lookup failed") c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + url.Namespace + " / " + url.Name + " does not exist"}) return } // make sure channel is not closed if targetsource in registry is deleted // timeout for sending to the channel - targets := createDiscoveryEvent(payloadTargets) - // fmt.Printf("core.DiscoveryEvent was created: %v", targets) + targets, err := createDiscoveryEvent(payloadTargets) + if err != nil{ + logger.Error(err, "failed creating discoveryEvent") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) + } utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) + logger.Info("CreateTargets request processed successfully", "count", len(targets)) c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go new file mode 100644 index 0000000..c129d1a --- /dev/null +++ b/internal/apiserver/apiserver_test.go @@ -0,0 +1,2 @@ +package apiserver + diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 37eb12e..cc556a4 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -4,27 +4,27 @@ import ( "fmt" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { +func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { targets := []core.DiscoveryEvent{} + if len(payloadTargets) > 0 { for i, target := range payloadTargets { if target.Name == "" { - // no REST API return here as not all targets might be incomplete - err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) - logger.Error(err, "Failed creating DiscoveryEvent") - break + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) } if target.Address == "" { - err := fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) - logger.Error(err, "Failed creating DiscoveryEvent") - break + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + } + event, err := getEvent(target, i) + if err != nil { + return nil, err } targets = append(targets, core.DiscoveryEvent{ @@ -33,11 +33,11 @@ func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { Address: target.Address, Labels: convertTargetLabelsToMap(target), }, - Event: getEvent(target, i), + Event: event, }) } } - return targets + return targets, nil } // getKey returns key for used to identify correct channel in DiscoveryRegistry @@ -64,7 +64,7 @@ func convertTargetLabelsToMap(target Target) map[string]string { } // getEvent converts target.Operation to core.Operation. -func getEvent(target Target, index int) core.EventAction { +func getEvent(target Target, index int) (core.EventAction, error) { event := core.EventApply switch target.Operation { case Created: @@ -74,16 +74,17 @@ func getEvent(target Target, index int) core.EventAction { case Deleted: event = core.EventDelete default: - err := fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) - logger.Error(err, "Failed creating DiscoveryEvent") + return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) } - return event + return event, nil } // parseURI parses URI to urlStruct. func parseURI(c *gin.Context) (url urlStruct) { + logger := log.FromContext(c.Request.Context()).WithValues("component", "apiserver", "action", "parse-uri") var u urlStruct if err := c.ShouldBindUri(&u); err != nil { + logger.Error(err, "Failed to bind request URI") c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index b1cf607..afc0eb9 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -13,39 +13,18 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/ ]' -# old (before 4th of may) - -## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/target-source/netbox/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"http-discovery", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' - -## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' - -## Empty Target in Target List -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' - -## Empty TargetSourceName -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' - -## Wrong operation -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' - - - -{ - "TargetSourceName": "webhook-test", - "TargetSourceNameSpace": "default", - "TargetList": [ +http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/default/target-source/http-discovery/createTargets +[ + { + "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 and data.primary_ip4.address else '' }}:{{ data.custom_fields.port }}", + "name": "{{ data.name }}", + "operation": "{{ event }}", + "profile": "{{ data.custom_fields.profile | default('') }}", + "labels": [ { - "name": "{{ data.name }}", - "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", - "profile": "{{ data.custom_fields.profile | default('') }}", - "labels": [{ - "key": "tags", - "value": "{{ data.tags | map(attribute='name') | join(', ') }}" - }, - "operation": "{{ event }}" - ] + "Key": "tags", + "Value": "{{ data.tags | map(attribute='name') | join(', ') }}" } ] -} \ No newline at end of file + } +] \ No newline at end of file From 7fa3e5baf40d2f75115f15af20251e0671fc3af9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 6 May 2026 12:46:39 +0000 Subject: [PATCH 182/246] unit test for helpers --- internal/apiserver/apiserver.go | 6 +- internal/apiserver/apiserver_test.go | 2 - internal/apiserver/gen.go | 6 +- internal/apiserver/helpers.go | 12 +- internal/apiserver/helpers_test.go | 265 +++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 15 deletions(-) delete mode 100644 internal/apiserver/apiserver_test.go create mode 100644 internal/apiserver/helpers_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index c6e7d4b..a46837f 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -47,6 +47,7 @@ func New( ], discoveryChunksize int, ) (*APIServer, error) { + gin.SetMode(gin.ReleaseMode) // To double-check router := gin.Default() logger := log.Log.WithValues("component", "api-server") a := &APIServer{ @@ -66,7 +67,6 @@ func New( } func (a *APIServer) Router() *gin.Engine { - gin.SetMode(gin.ReleaseMode) // gin logs return a.router } @@ -91,7 +91,6 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.String(404, err.Error()) return } - logger.Info("Successfully returned cluster plan") c.JSON(200, plan) } @@ -122,11 +121,10 @@ func (a *APIServer) CreateTargets(c *gin.Context) { // make sure channel is not closed if targetsource in registry is deleted // timeout for sending to the channel targets, err := createDiscoveryEvent(payloadTargets) - if err != nil{ + if err != nil { logger.Error(err, "failed creating discoveryEvent") c.JSON(http.StatusBadRequest, gin.H{"error": err}) } utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) - logger.Info("CreateTargets request processed successfully", "count", len(targets)) c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go deleted file mode 100644 index c129d1a..0000000 --- a/internal/apiserver/apiserver_test.go +++ /dev/null @@ -1,2 +0,0 @@ -package apiserver - diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 5811b9b..80f2c5f 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -43,8 +43,8 @@ type Label struct { Value *string `json:"value,omitempty"` } -// Target defines model for Target. -type Target struct { +// target defines model for target. +type target struct { Address string `json:"address"` Labels *[]Label `json:"labels,omitempty"` Name string `json:"name"` @@ -56,7 +56,7 @@ type Target struct { type TargetOperation string // Targets defines model for Targets. -type Targets = []Target +type Targets = []target // CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. type CreateTargetsJSONRequestBody = Targets diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index cc556a4..487e371 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -11,16 +11,16 @@ import ( ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { +func createDiscoveryEvent(payloadTargets []target) ([]core.DiscoveryEvent, error) { targets := []core.DiscoveryEvent{} if len(payloadTargets) > 0 { for i, target := range payloadTargets { if target.Name == "" { - return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name.", i) } if target.Address == "" { - return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address.", i) } event, err := getEvent(target, i) if err != nil { @@ -50,7 +50,7 @@ func getKey(u urlStruct) types.NamespacedName { } // convertTargetLabelsToMap converts target.Labels to map. -func convertTargetLabelsToMap(target Target) map[string]string { +func convertTargetLabelsToMap(target target) map[string]string { labelToMap := make(map[string]string) if target.Labels != nil { for _, tag := range *target.Labels { @@ -64,7 +64,7 @@ func convertTargetLabelsToMap(target Target) map[string]string { } // getEvent converts target.Operation to core.Operation. -func getEvent(target Target, index int) (core.EventAction, error) { +func getEvent(target target, index int) (core.EventAction, error) { event := core.EventApply switch target.Operation { case Created: @@ -74,7 +74,7 @@ func getEvent(target Target, index int) (core.EventAction, error) { case Deleted: event = core.EventDelete default: - return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) + return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation", index) } return event, nil } diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go new file mode 100644 index 0000000..9bfa41b --- /dev/null +++ b/internal/apiserver/helpers_test.go @@ -0,0 +1,265 @@ +package apiserver + +import ( + "reflect" + "testing" + + "net/http" + "net/http/httptest" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" +) + +func TestGetEventApply(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "created", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventDelete(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "deleted", + } + event, err := getEvent(target, 0) + if event != core.EventDelete { + t.Errorf("getEvent(target) = %d, want core.EventDelete", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventEmptyOperation(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "", + } + event, err := getEvent(target, 0) + if err == nil { + t.Errorf("getEvent(target, 0) = %d, want error", event) + } +} + +func TestGetEventUpdate(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetKey(t *testing.T) { + u := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + expected := types.NamespacedName{ + Namespace: "default", + Name: "http-discovery", + } + result := getKey(u) + if result != expected { + t.Errorf("getKey(%v) = %v; want %v", u, result, expected) + } +} + +func TestConvertTargetLabelsToMapEmpty(t *testing.T) { + target := target{} + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMap(t *testing.T) { + key := "Tag" + value := "TT1, TT2" + label := Label{ + Key: &key, + Value: &value, + } + target := target{ + Labels: &[]Label{label}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestConvertTargetLabelsToMapEmptyKey(t *testing.T) { + key := "" + value := "TT1, TT2" + label := Label{ + Key: &key, + Value: &value, + } + target := target{ + Labels: &[]Label{label}, + } + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { + key := "Tag" + key2 := "Tag1" + value := "TT1, TT2" + value2 := "TT1" + label := Label{ + Key: &key, + Value: &value, + } + label2 := Label{ + Key: &key2, + Value: &value2, + } + target := target{ + Labels: &[]Label{label, label2}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + "Tag1": "TT1", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEvent(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated"}} + + expected := []core.DiscoveryEvent{ + { + Target: core.DiscoveredTarget{ + Name: "routername", + Address: "1.1.1.1", + Labels: map[string]string{}, + }, + Event: core.EventApply, + }, + } + result, _ := createDiscoveryEvent(targets) + if !reflect.DeepEqual(result, expected) { + t.Errorf("createDiscoveryEvent(targets) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEventEmptyName(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "", + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing name error", result) + } +} + +func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { + targets := []target{{ + Address: "", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing address error", result) + } +} + +func TestCreateDiscoveryEventWrongEvent(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "", + Labels: &[]Label{}, + Operation: "wrongOperation"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want wrong Operation error", result) + } +} + +func TestParseURI(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source/http-discovery/createTargets", nil) + router.ServeHTTP(recorder, req) + + expected := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("parseURI(ctx) = %v; want %v", result, expected) + } + if recorder.Code != http.StatusOK { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusOK) + } +} + +func TestParseURIMissingName(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source//createTargets", nil) + router.ServeHTTP(recorder, req) + + if !reflect.DeepEqual(result, urlStruct{}) { + t.Errorf("parseURI(ctx) = %v; want empty urlStruct", result) + } + if recorder.Code != http.StatusBadRequest { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusBadRequest) + } +} From deb9e90802f6138f1f71eb25ea4531bd2812d15b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 8 May 2026 12:37:00 +0000 Subject: [PATCH 183/246] git ignore sonar scanner --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7515fa3..ef68c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ test/integration/clab-* # Only for development and testing purposes # To be removed after development of targetsource # ignored in order to not add unnecassary logging messages -lab/dev/resources/targetsources \ No newline at end of file +lab/dev/resources/targetsources +.scannerwork/ \ No newline at end of file From b088db2eb0cae0fd5f9cc5142b42961d29365954 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 8 May 2026 15:27:15 +0000 Subject: [PATCH 184/246] add defaulting for targetsource crd --- api/v1alpha1/targetsource_types.go | 55 ++++---- api/v1alpha1/zz_generated.deepcopy.go | 122 +++++++++++------- .../operator.gnmic.dev_targetsources.yaml | 53 ++------ internal/controller/discovery/loaders.go | 2 +- .../discovery/loaders/http/const.go | 9 ++ .../discovery/loaders/http/loader.go | 27 ++-- .../webhook/v1alpha1/targetsource_webhook.go | 17 ++- 7 files changed, 145 insertions(+), 140 deletions(-) create mode 100644 internal/controller/discovery/loaders/http/const.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 44e605e..78a2f74 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -38,13 +38,10 @@ type ProviderSpec struct { HTTP *HTTPConfig `json:"http,omitempty"` } -type WebhookSpec struct { - Enabled *bool `json:"enabled,omitempty"` -} - +// +kubebuilder:validation:AtLeastOneOf=url;acceptPush type HTTPConfig struct { - // +kubebuilder:validation:MinLength=1 - URL string `json:"url"` + // +kubebuilder:validation:Optional + URL string `json:"url,omitempty"` // +kubebuilder:validation:Optional Authorization *AuthorizationSpec `json:"authorization,omitempty"` // +kubebuilder:validation:Optional @@ -58,41 +55,34 @@ type HTTPConfig struct { // +kubebuilder:validation:Optional ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"` // +kubebuilder:validation:Optional - AcceptPush bool `json:"acceptPush,omitempty"` + AcceptPush *bool `json:"acceptPush,omitempty"` } type ClientTLSConfig struct { - InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + InsecureSkipVerify *bool `json:"insecureSkipVerify,omitempty"` CASecretRef *corev1.SecretKeySelector `json:"caSecretRef,omitempty"` } -// +kubebuilder:validation:ExactlyOneOf=basic;bearer;jwt;token +// +kubebuilder:validation:ExactlyOneOf=basic;jwt;token type AuthorizationSpec struct { - Basic *BasicAuthSpec `json:"basic,omitempty"` - Bearer *BearerAuthSpec `json:"bearer,omitempty"` - Token *TokenAuthSpec `json:"token,omitempty"` - JWT *JWTAuthSpec `json:"jwt,omitempty"` + Basic *BasicAuthSpec `json:"basic,omitempty"` + Token *TokenAuthSpec `json:"token,omitempty"` + JWT *JWTAuthSpec `json:"jwt,omitempty"` } // Enforce EITHER inline creds OR secret ref // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix" type BasicAuthSpec struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` + Username *string `json:"username,omitempty"` + Password *string `json:"password,omitempty"` CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"` } -// +kubebuilder:validation:ExactlyOneOf=token;tokenSecretRef -type BearerAuthSpec struct { - Token string `json:"token,omitempty"` - TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` -} - // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both" type TokenAuthSpec struct { // +kubebuilder:validation:MinLength=1 - Scheme string `json:"scheme"` - Token string `json:"token,omitempty"` + Scheme *string `json:"scheme"` + Token *string `json:"token,omitempty"` TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } @@ -101,36 +91,35 @@ type TokenAuthSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" type JWTAuthSpec struct { // Static pre-generated JWT - Token string `json:"token,omitempty"` + Token *string `json:"token,omitempty"` TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` // Optional: generate JWT dynamically Claims map[string]string `json:"claims,omitempty"` SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"` // HS256, RS256, ES256, etc. - Algorithm string `json:"algorithm,omitempty"` + Algorithm *string `json:"algorithm,omitempty"` TTL *metav1.Duration `json:"ttl,omitempty"` } type PaginationSpec struct { - Enabled bool `json:"enabled"` // Example: "results" - ItemsField string `json:"itemsField,omitempty"` + ItemsField *string `json:"itemsField,omitempty"` // Example: "next" - NextField string `json:"nextField,omitempty"` + NextField *string `json:"nextField,omitempty"` } // JSONPath-style expressions type ResponseMappingSpec struct { - Name string `json:"name"` - Address string `json:"address"` - Port string `json:"port,omitempty"` + Name *string `json:"name"` + Address *string `json:"address"` + Port *string `json:"port,omitempty"` Labels map[string]string `json:"labels,omitempty"` } // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { - Status string `json:"status"` - TargetsCount int32 `json:"targetsCount"` + Status *string `json:"status"` + TargetsCount *int32 `json:"targetsCount"` LastSync metav1.Time `json:"lastSync"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 602ede5..453dcc9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -54,11 +54,6 @@ func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) { *out = new(BasicAuthSpec) (*in).DeepCopyInto(*out) } - if in.Bearer != nil { - in, out := &in.Bearer, &out.Bearer - *out = new(BearerAuthSpec) - (*in).DeepCopyInto(*out) - } if in.Token != nil { in, out := &in.Token, &out.Token *out = new(TokenAuthSpec) @@ -84,6 +79,16 @@ func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) { *out = *in + if in.Username != nil { + in, out := &in.Username, &out.Username + *out = new(string) + **out = **in + } + if in.Password != nil { + in, out := &in.Password, &out.Password + *out = new(string) + **out = **in + } if in.CredentialsSecretRef != nil { in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef *out = new(v1.SecretKeySelector) @@ -101,29 +106,14 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BearerAuthSpec) DeepCopyInto(out *BearerAuthSpec) { - *out = *in - if in.TokenSecretRef != nil { - in, out := &in.TokenSecretRef, &out.TokenSecretRef - *out = new(v1.SecretKeySelector) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BearerAuthSpec. -func (in *BearerAuthSpec) DeepCopy() *BearerAuthSpec { - if in == nil { - return nil - } - out := new(BearerAuthSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) { *out = *in + if in.InsecureSkipVerify != nil { + in, out := &in.InsecureSkipVerify, &out.InsecureSkipVerify + *out = new(bool) + **out = **in + } if in.CASecretRef != nil { in, out := &in.CASecretRef, &out.CASecretRef *out = new(v1.SecretKeySelector) @@ -376,13 +366,18 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) { if in.Pagination != nil { in, out := &in.Pagination, &out.Pagination *out = new(PaginationSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.ResponseMapping != nil { in, out := &in.ResponseMapping, &out.ResponseMapping *out = new(ResponseMappingSpec) (*in).DeepCopyInto(*out) } + if in.AcceptPush != nil { + in, out := &in.AcceptPush, &out.AcceptPush + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig. @@ -527,6 +522,11 @@ func (in *InputStatus) DeepCopy() *InputStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { *out = *in + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(string) + **out = **in + } if in.TokenSecretRef != nil { in, out := &in.TokenSecretRef, &out.TokenSecretRef *out = new(v1.SecretKeySelector) @@ -544,6 +544,11 @@ func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } + if in.Algorithm != nil { + in, out := &in.Algorithm, &out.Algorithm + *out = new(string) + **out = **in + } if in.TTL != nil { in, out := &in.TTL, &out.TTL *out = new(metav1.Duration) @@ -737,6 +742,16 @@ func (in *OutputStatus) DeepCopy() *OutputStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) { *out = *in + if in.ItemsField != nil { + in, out := &in.ItemsField, &out.ItemsField + *out = new(string) + **out = **in + } + if in.NextField != nil { + in, out := &in.NextField, &out.NextField + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec. @@ -1003,6 +1018,21 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) { *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.Address != nil { + in, out := &in.Address, &out.Address + *out = new(string) + **out = **in + } + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(string) + **out = **in + } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) @@ -1493,6 +1523,16 @@ func (in *TargetSourceSpec) DeepCopy() *TargetSourceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetSourceStatus) DeepCopyInto(out *TargetSourceStatus) { *out = *in + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(string) + **out = **in + } + if in.TargetsCount != nil { + in, out := &in.TargetsCount, &out.TargetsCount + *out = new(int32) + **out = **in + } in.LastSync.DeepCopyInto(&out.LastSync) } @@ -1566,6 +1606,16 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) { *out = *in + if in.Scheme != nil { + in, out := &in.Scheme, &out.Scheme + *out = new(string) + **out = **in + } + if in.Token != nil { + in, out := &in.Token, &out.Token + *out = new(string) + **out = **in + } if in.TokenSecretRef != nil { in, out := &in.TokenSecretRef, &out.TokenSecretRef *out = new(v1.SecretKeySelector) @@ -1676,23 +1726,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { - *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. -func (in *WebhookSpec) DeepCopy() *WebhookSpec { - if in == nil { - return nil - } - out := new(WebhookSpec) - in.DeepCopyInto(out) - return out -} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 65c48c0..588bccf 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -86,41 +86,6 @@ spec: rule: (has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password)) - bearer: - properties: - token: - type: string - tokenSecretRef: - description: SecretKeySelector selects a key of a - Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - x-kubernetes-validations: - - message: exactly one of the fields in [token tokenSecretRef] - must be set - rule: '[has(self.token),has(self.tokenSecretRef)].filter(x,x==true).size() - == 1' jwt: properties: algorithm: @@ -241,9 +206,9 @@ spec: rule: has(self.token) != has(self.tokenSecretRef) type: object x-kubernetes-validations: - - message: exactly one of the fields in [basic bearer jwt - token] must be set - rule: '[has(self.basic),has(self.bearer),has(self.jwt),has(self.token)].filter(x,x==true).size() + - message: exactly one of the fields in [basic jwt token] + must be set + rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size() == 1' interval: type: string @@ -266,16 +231,12 @@ spec: type: object pagination: properties: - enabled: - type: boolean itemsField: description: 'Example: "results"' type: string nextField: description: 'Example: "next"' type: string - required: - - enabled type: object timeout: type: string @@ -309,11 +270,13 @@ spec: type: boolean type: object url: - minLength: 1 type: string - required: - - url type: object + x-kubernetes-validations: + - message: at least one of the fields in [url acceptPush] must + be set + rule: '[has(self.url),has(self.acceptPush)].filter(x,x==true).size() + >= 1' type: object x-kubernetes-validations: - message: exactly one of the fields in [http] must be set diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 646fb0a..66bec04 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -13,7 +13,7 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec switch { case spec.Provider.HTTP != nil: - cfg.AcceptPush = spec.Provider.HTTP.AcceptPush + cfg.AcceptPush = *spec.Provider.HTTP.AcceptPush return http.New(cfg, *spec.Provider.HTTP), cfg, nil default: return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) diff --git a/internal/controller/discovery/loaders/http/const.go b/internal/controller/discovery/loaders/http/const.go new file mode 100644 index 0000000..4b18f6c --- /dev/null +++ b/internal/controller/discovery/loaders/http/const.go @@ -0,0 +1,9 @@ +package http + +import "time" + +const ( + DefaultPollInterval = 1 * time.Hour + DefaultTimeout = 30 * time.Second + DefaultAcceptPush = false +) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index a009b76..b55bcef 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -16,20 +16,15 @@ import ( loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" ) -const ( - defaultPollInterval = 30 * time.Second - defaultTimeoutSeconds = 30 -) - // Loader implements the HTTP pull discovery mechanism type Loader struct { - commonCfg core.CommonLoaderConfig + loaderCfg core.CommonLoaderConfig spec *gnmicv1alpha1.HTTPConfig } // New instantiates the http loader with the provided config func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader { - return &Loader{commonCfg: cfg, spec: &httpConfig} + return &Loader{loaderCfg: cfg, spec: &httpConfig} } func (l *Loader) Name() string { @@ -37,16 +32,20 @@ func (l *Loader) Name() string { } func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error { + if l.spec.URL == "" { + return nil + } + logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", l.commonCfg.TargetsourceNN, + "targetsource", l.loaderCfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", l.commonCfg.TargetsourceNN.Name, - "namespace", l.commonCfg.TargetsourceNN.Namespace, + "targetsource", l.loaderCfg.TargetsourceNN.Name, + "namespace", l.loaderCfg.TargetsourceNN.Namespace, ) logger.Info("HTTP loader started") @@ -57,9 +56,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er // } client := &http.Client{ - Timeout: defaultTimeoutSeconds * time.Second, + Timeout: l.spec.Timeout.Duration, } - interval := defaultPollInterval + interval := l.spec.PollInterval.Duration ticker := time.NewTicker(interval) defer ticker.Stop() @@ -87,8 +86,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er return } - snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { + snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString()) + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.loaderCfg.ChunkSize); err != nil { logger.Error( err, "Failed to send discovery snapshot", diff --git a/internal/webhook/v1alpha1/targetsource_webhook.go b/internal/webhook/v1alpha1/targetsource_webhook.go index b3eb960..992d67b 100644 --- a/internal/webhook/v1alpha1/targetsource_webhook.go +++ b/internal/webhook/v1alpha1/targetsource_webhook.go @@ -21,12 +21,14 @@ import ( "fmt" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + httpDefaults "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // nolint:unused @@ -65,7 +67,20 @@ func (d *TargetSourceCustomDefaulter) Default(_ context.Context, obj runtime.Obj } targetsourcelog.Info("Defaulting for TargetSource", "name", targetsource.GetName()) - // TODO(user): fill in your defaulting logic. + // HTTP Config Defaulting + if targetsource.Spec.Provider.HTTP != nil { + http := targetsource.Spec.Provider.HTTP + + if http.PollInterval == nil { + http.PollInterval.Duration = httpDefaults.DefaultPollInterval + } + if http.Timeout == nil { + http.Timeout.Duration = httpDefaults.DefaultTimeout + } + if http.AcceptPush == nil { + http.AcceptPush = pointer.Bool(httpDefaults.DefaultAcceptPush) + } + } return nil } From 9ff3ba1a2fdaea9630c0049c106b22bca5d91186 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:53:05 +0000 Subject: [PATCH 185/246] fixed make target name --- test.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.mk b/test.mk index f92c4ec..23c5983 100644 --- a/test.mk +++ b/test.mk @@ -100,7 +100,7 @@ deploy-test-netbox-instance: NETBOX_CLUSTER_NAME=$(TEST_CLUSTER_NAME) ## Deploy deploy-test-netbox-instance: NETBOX_PASSWORD=Netbox123 deploy-test-netbox-instance: netbox-setup -.PHONY: deploy-test-netbox-instance +.PHONY: deploy-test-netbox-topology deploy-test-netbox-topology: ## Deploy the netbox test topology for testing sudo containerlab deploy -t test/integration/netbox/netbox.clab.yaml -c kubectl port-forward svc/netbox $(NETBOX_TEST_PORT):80 -n netbox --context kind-$(TEST_CLUSTER_NAME) --address=0.0.0.0 >/dev/null 2>&1 & From 0031b5ac20d3f7597db061658691f3ada8d0005e Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:54:10 +0000 Subject: [PATCH 186/246] added http targetsource integration test to Makefile --- Makefile | 3 ++- test.mk | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fdcc2b2..98c42e7 100644 --- a/Makefile +++ b/Makefile @@ -308,9 +308,10 @@ delete-targetsources-dev-lab: ## Delete the target sources for the development l ##@ Testing Lab .PHONY: run-integration-tests -run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources +run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy deploy-test-http-server install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources kubectl wait --for=condition=Ready cluster --all --timeout=180s kubectl wait --for=condition=Ready pipeline --all --timeout=180s + kubectl wait --for=jsonpath='{.status.targetsCount}'=3 targetsource --all --timeout=180s kubectl wait --for=jsonpath='{.status.connectionState}'=READY target --all --timeout=180s kubectl get subscriptions -o yaml kubectl get outputs -o yaml diff --git a/test.mk b/test.mk index 23c5983..fb30c30 100644 --- a/test.mk +++ b/test.mk @@ -153,5 +153,5 @@ apply-test-clusters: ## Apply the test clusters for testing kubectl apply -f test/integration/resources/clusters .PHONY: apply-test-resources -apply-test-resources: apply-test-targets apply-test-subscriptions apply-test-outputs apply-test-pipelines apply-test-clusters +apply-test-resources: apply-test-targets apply-test-subscriptions apply-test-outputs apply-test-pipelines apply-test-clusters apply-test-targetsources From c3dc34ca86cb126a8f4bc70abdc850fcbba7c505 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:56:16 +0000 Subject: [PATCH 187/246] changed resource names + mapped target inventory to clab --- .../integration/http/resources/configmap.yaml | 24 +++++++++++++------ .../http/resources/deployment.yaml | 8 +++---- test/integration/http/resources/service.yaml | 4 ++-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/test/integration/http/resources/configmap.yaml b/test/integration/http/resources/configmap.yaml index 0e8b35c..f017566 100644 --- a/test/integration/http/resources/configmap.yaml +++ b/test/integration/http/resources/configmap.yaml @@ -1,22 +1,32 @@ apiVersion: v1 kind: ConfigMap metadata: - name: targets-config + name: http-target-cfg data: targets.json: | [ { - "address": "10.0.0.1:57000", - "name": "router1", + "address": "clab-t1-spine1:57400", + "name": "spine1", "labels": { - "label1": "test" + "vendor": "nokia_srlinux", + "role": "spine" } }, { - "address": "10.0.0.2:57000", - "name": "router2", + "address": "clab-t1-leaf1:57400", + "name": "leaf1", "labels": { - "label2": "test2" + "vendor": "nokia_srlinux", + "role": "leaf" + } + }, + { + "address": "clab-t1-leaf2:57400", + "name": "leaf2", + "labels": { + "vendor": "nokia_srlinux", + "role": "leaf" } } ] \ No newline at end of file diff --git a/test/integration/http/resources/deployment.yaml b/test/integration/http/resources/deployment.yaml index 3dc1f61..785c1e3 100644 --- a/test/integration/http/resources/deployment.yaml +++ b/test/integration/http/resources/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: targets-server + name: http-target-inv spec: replicas: 1 selector: matchLabels: - app: targets-server + app: http-target-inv template: metadata: labels: - app: targets-server + app: http-target-inv spec: containers: - name: nginx @@ -21,4 +21,4 @@ spec: volumes: - name: data configMap: - name: targets-config \ No newline at end of file + name: http-target-cfg \ No newline at end of file diff --git a/test/integration/http/resources/service.yaml b/test/integration/http/resources/service.yaml index 03f0efa..d4be4e7 100644 --- a/test/integration/http/resources/service.yaml +++ b/test/integration/http/resources/service.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: targets + name: http-svc spec: selector: - app: targets-server + app: http-target-inv ports: - port: 80 targetPort: 80 \ No newline at end of file From efead83350041b874e39627d9e146ad83ef42f96 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:57:11 +0000 Subject: [PATCH 188/246] fixed http target url and profile --- test/integration/resources/targetsources/http.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/resources/targetsources/http.yaml b/test/integration/resources/targetsources/http.yaml index 199fcf3..422cfdc 100644 --- a/test/integration/resources/targetsources/http.yaml +++ b/test/integration/resources/targetsources/http.yaml @@ -5,7 +5,7 @@ metadata: spec: provider: http: - url: http://targets.default.svc/targets.json + url: http://http-svc.default.svc/targets.json targetLabels: - integration-test: http - targetProfile: http-default \ No newline at end of file + integrationtest: http + targetProfile: default \ No newline at end of file From 4d461a3cbaee9e4310288bf5ffa579b7b42a04ed Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:57:37 +0000 Subject: [PATCH 189/246] removed netbox targetsource for integration test --- test/integration/resources/targetsources/netbox.yaml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test/integration/resources/targetsources/netbox.yaml diff --git a/test/integration/resources/targetsources/netbox.yaml b/test/integration/resources/targetsources/netbox.yaml deleted file mode 100644 index 39ab922..0000000 --- a/test/integration/resources/targetsources/netbox.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: netbox-ts -spec: - provider: - http: - url: http://localhost:8082/targets - targetLabels: - integration-test: netbox - targetProfile: netbox-default \ No newline at end of file From f7c627a0b88ac15740e59a6ad154281b400bff26 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:58:09 +0000 Subject: [PATCH 190/246] mapped operator resources to new http test --- test/integration/resources/clusters/cluster1.yaml | 2 +- test/integration/resources/pipelines/pipeline1.yaml | 2 ++ test/integration/resources/pipelines/pipeline2.yaml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/resources/clusters/cluster1.yaml b/test/integration/resources/clusters/cluster1.yaml index 513b948..c01cc56 100644 --- a/test/integration/resources/clusters/cluster1.yaml +++ b/test/integration/resources/clusters/cluster1.yaml @@ -13,4 +13,4 @@ spec: memory: "500Mi" cpu: "1" targetDistribution: - podCapacity: 5 \ No newline at end of file + podCapacity: 10 \ No newline at end of file diff --git a/test/integration/resources/pipelines/pipeline1.yaml b/test/integration/resources/pipelines/pipeline1.yaml index 0dc67a3..82c0289 100644 --- a/test/integration/resources/pipelines/pipeline1.yaml +++ b/test/integration/resources/pipelines/pipeline1.yaml @@ -12,6 +12,8 @@ spec: - matchLabels: vendor: nokia_srlinux role: spine + - matchLabels: + operator.gnmic.dev/targetsource: http-ts subscriptionSelectors: - matchLabels: vendor: nokia_srlinux diff --git a/test/integration/resources/pipelines/pipeline2.yaml b/test/integration/resources/pipelines/pipeline2.yaml index 7420d7d..a361833 100644 --- a/test/integration/resources/pipelines/pipeline2.yaml +++ b/test/integration/resources/pipelines/pipeline2.yaml @@ -12,6 +12,8 @@ spec: - matchLabels: vendor: nokia_srlinux role: spine + - matchLabels: + operator.gnmic.dev/targetsource: http-ts subscriptionSelectors: - matchLabels: vendor: nokia_srlinux From f37cc4abc5a41018f8d5762c5faf7d1e2b839be8 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 21:00:08 +0000 Subject: [PATCH 191/246] removed netbox clab and changed address --- .../netbox/initializers/ip-addresses.yaml | 6 ++-- test/integration/netbox/netbox.clab.yaml | 28 ------------------- 2 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 test/integration/netbox/netbox.clab.yaml diff --git a/test/integration/netbox/initializers/ip-addresses.yaml b/test/integration/netbox/initializers/ip-addresses.yaml index de95cc8..73453a8 100644 --- a/test/integration/netbox/initializers/ip-addresses.yaml +++ b/test/integration/netbox/initializers/ip-addresses.yaml @@ -1,4 +1,4 @@ -- address: 172.18.1.10/32 +- address: clab-t1-spine1 assigned_object: device: name: spine1 @@ -6,7 +6,7 @@ status: active primary: true dns_name: t2-nodes-spine1 -- address: 172.18.1.11/32 +- address: clab-t1-leaf1 assigned_object: device: name: leaf1 @@ -14,7 +14,7 @@ status: active primary: true dns_name: t2-nodes-leaf1 -- address: 172.18.1.12/32 +- address: clab-t1-leaf2 assigned_object: device: name: leaf2 diff --git a/test/integration/netbox/netbox.clab.yaml b/test/integration/netbox/netbox.clab.yaml deleted file mode 100644 index ddd1705..0000000 --- a/test/integration/netbox/netbox.clab.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: netbox - -mgmt: - network: kind - -topology: - defaults: - kind: nokia_srlinux - image: ghcr.io/nokia/srlinux:25.10.1 - - kinds: - nokia_srlinux: - image: ghcr.io/nokia/srlinux:25.10.1 - type: ixr-d2l - - nodes: - spine1: - type: ixr-d3l - mgmt-ipv4: 172.18.1.10 - - leaf1: - mgmt-ipv4: 172.18.1.11 - leaf2: - mgmt-ipv4: 172.18.1.12 - - links: - - endpoints: ["spine1:e1-1", "leaf1:e1-49"] - - endpoints: ["spine1:e1-2", "leaf2:e1-49"] From 000d405b8413e545534f86fabed9906948ac775e Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:56:16 +0000 Subject: [PATCH 192/246] changed resource names + mapped target inventory to clab --- .../integration/http/resources/configmap.yaml | 24 +++++++++++++------ .../http/resources/deployment.yaml | 8 +++---- test/integration/http/resources/service.yaml | 4 ++-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/test/integration/http/resources/configmap.yaml b/test/integration/http/resources/configmap.yaml index 0e8b35c..f017566 100644 --- a/test/integration/http/resources/configmap.yaml +++ b/test/integration/http/resources/configmap.yaml @@ -1,22 +1,32 @@ apiVersion: v1 kind: ConfigMap metadata: - name: targets-config + name: http-target-cfg data: targets.json: | [ { - "address": "10.0.0.1:57000", - "name": "router1", + "address": "clab-t1-spine1:57400", + "name": "spine1", "labels": { - "label1": "test" + "vendor": "nokia_srlinux", + "role": "spine" } }, { - "address": "10.0.0.2:57000", - "name": "router2", + "address": "clab-t1-leaf1:57400", + "name": "leaf1", "labels": { - "label2": "test2" + "vendor": "nokia_srlinux", + "role": "leaf" + } + }, + { + "address": "clab-t1-leaf2:57400", + "name": "leaf2", + "labels": { + "vendor": "nokia_srlinux", + "role": "leaf" } } ] \ No newline at end of file diff --git a/test/integration/http/resources/deployment.yaml b/test/integration/http/resources/deployment.yaml index 3dc1f61..785c1e3 100644 --- a/test/integration/http/resources/deployment.yaml +++ b/test/integration/http/resources/deployment.yaml @@ -1,16 +1,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: targets-server + name: http-target-inv spec: replicas: 1 selector: matchLabels: - app: targets-server + app: http-target-inv template: metadata: labels: - app: targets-server + app: http-target-inv spec: containers: - name: nginx @@ -21,4 +21,4 @@ spec: volumes: - name: data configMap: - name: targets-config \ No newline at end of file + name: http-target-cfg \ No newline at end of file diff --git a/test/integration/http/resources/service.yaml b/test/integration/http/resources/service.yaml index 03f0efa..d4be4e7 100644 --- a/test/integration/http/resources/service.yaml +++ b/test/integration/http/resources/service.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: targets + name: http-svc spec: selector: - app: targets-server + app: http-target-inv ports: - port: 80 targetPort: 80 \ No newline at end of file From 858600b8bdcd1824ddde80eb397f3bdb26a5ff6b Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:57:11 +0000 Subject: [PATCH 193/246] fixed http target url and profile --- test/integration/resources/targetsources/http.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/resources/targetsources/http.yaml b/test/integration/resources/targetsources/http.yaml index 199fcf3..422cfdc 100644 --- a/test/integration/resources/targetsources/http.yaml +++ b/test/integration/resources/targetsources/http.yaml @@ -5,7 +5,7 @@ metadata: spec: provider: http: - url: http://targets.default.svc/targets.json + url: http://http-svc.default.svc/targets.json targetLabels: - integration-test: http - targetProfile: http-default \ No newline at end of file + integrationtest: http + targetProfile: default \ No newline at end of file From bbbceec02e540df17102ccd4503679ce5795993e Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:57:37 +0000 Subject: [PATCH 194/246] removed netbox targetsource for integration test --- test/integration/resources/targetsources/netbox.yaml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test/integration/resources/targetsources/netbox.yaml diff --git a/test/integration/resources/targetsources/netbox.yaml b/test/integration/resources/targetsources/netbox.yaml deleted file mode 100644 index 39ab922..0000000 --- a/test/integration/resources/targetsources/netbox.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: netbox-ts -spec: - provider: - http: - url: http://localhost:8082/targets - targetLabels: - integration-test: netbox - targetProfile: netbox-default \ No newline at end of file From bfaf0e3e2e32361ee97343cccfa4f4ad717e1256 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 20:58:09 +0000 Subject: [PATCH 195/246] mapped operator resources to new http test --- test/integration/resources/clusters/cluster1.yaml | 2 +- test/integration/resources/pipelines/pipeline1.yaml | 2 ++ test/integration/resources/pipelines/pipeline2.yaml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/resources/clusters/cluster1.yaml b/test/integration/resources/clusters/cluster1.yaml index 513b948..c01cc56 100644 --- a/test/integration/resources/clusters/cluster1.yaml +++ b/test/integration/resources/clusters/cluster1.yaml @@ -13,4 +13,4 @@ spec: memory: "500Mi" cpu: "1" targetDistribution: - podCapacity: 5 \ No newline at end of file + podCapacity: 10 \ No newline at end of file diff --git a/test/integration/resources/pipelines/pipeline1.yaml b/test/integration/resources/pipelines/pipeline1.yaml index 0dc67a3..82c0289 100644 --- a/test/integration/resources/pipelines/pipeline1.yaml +++ b/test/integration/resources/pipelines/pipeline1.yaml @@ -12,6 +12,8 @@ spec: - matchLabels: vendor: nokia_srlinux role: spine + - matchLabels: + operator.gnmic.dev/targetsource: http-ts subscriptionSelectors: - matchLabels: vendor: nokia_srlinux diff --git a/test/integration/resources/pipelines/pipeline2.yaml b/test/integration/resources/pipelines/pipeline2.yaml index 7420d7d..a361833 100644 --- a/test/integration/resources/pipelines/pipeline2.yaml +++ b/test/integration/resources/pipelines/pipeline2.yaml @@ -12,6 +12,8 @@ spec: - matchLabels: vendor: nokia_srlinux role: spine + - matchLabels: + operator.gnmic.dev/targetsource: http-ts subscriptionSelectors: - matchLabels: vendor: nokia_srlinux From f7e6ac0f340f8ef39cb220d94db2ded103088a97 Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 21:00:08 +0000 Subject: [PATCH 196/246] removed netbox clab and changed address --- .../netbox/initializers/ip-addresses.yaml | 6 ++-- test/integration/netbox/netbox.clab.yaml | 28 ------------------- 2 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 test/integration/netbox/netbox.clab.yaml diff --git a/test/integration/netbox/initializers/ip-addresses.yaml b/test/integration/netbox/initializers/ip-addresses.yaml index de95cc8..73453a8 100644 --- a/test/integration/netbox/initializers/ip-addresses.yaml +++ b/test/integration/netbox/initializers/ip-addresses.yaml @@ -1,4 +1,4 @@ -- address: 172.18.1.10/32 +- address: clab-t1-spine1 assigned_object: device: name: spine1 @@ -6,7 +6,7 @@ status: active primary: true dns_name: t2-nodes-spine1 -- address: 172.18.1.11/32 +- address: clab-t1-leaf1 assigned_object: device: name: leaf1 @@ -14,7 +14,7 @@ status: active primary: true dns_name: t2-nodes-leaf1 -- address: 172.18.1.12/32 +- address: clab-t1-leaf2 assigned_object: device: name: leaf2 diff --git a/test/integration/netbox/netbox.clab.yaml b/test/integration/netbox/netbox.clab.yaml deleted file mode 100644 index ddd1705..0000000 --- a/test/integration/netbox/netbox.clab.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: netbox - -mgmt: - network: kind - -topology: - defaults: - kind: nokia_srlinux - image: ghcr.io/nokia/srlinux:25.10.1 - - kinds: - nokia_srlinux: - image: ghcr.io/nokia/srlinux:25.10.1 - type: ixr-d2l - - nodes: - spine1: - type: ixr-d3l - mgmt-ipv4: 172.18.1.10 - - leaf1: - mgmt-ipv4: 172.18.1.11 - leaf2: - mgmt-ipv4: 172.18.1.12 - - links: - - endpoints: ["spine1:e1-1", "leaf1:e1-49"] - - endpoints: ["spine1:e1-2", "leaf2:e1-49"] From 61ef95dc3d5fa92ce952f50ad8067f03d1373b0e Mon Sep 17 00:00:00 2001 From: mcdillson Date: Fri, 8 May 2026 21:23:56 +0000 Subject: [PATCH 197/246] generated manifests --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3848412..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) From d64bf6a093a57fa1c5dfda996062821c26a70c15 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 8 May 2026 16:37:37 -0600 Subject: [PATCH 198/246] added tests for mapper.go --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- internal/controller/discovery/mapper_test.go | 205 +++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 internal/controller/discovery/mapper_test.go diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3848412..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go new file mode 100644 index 0000000..2475fff --- /dev/null +++ b/internal/controller/discovery/mapper_test.go @@ -0,0 +1,205 @@ +package discovery + +import ( + "fmt" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +func mockDiscoveredTargetList(len int) []core.DiscoveredTarget { + targets := make([]core.DiscoveredTarget, len) + + if len > 100 { + len = 100 + } + + for i := range len { + targets[i] = core.DiscoveredTarget{ + Address: fmt.Sprintf("192.168.1.%d", i+1), + Name: fmt.Sprintf("router%d", i+1), + } + } + + return targets +} + +func mockGnmicTargetList(len int) []gnmicv1alpha1.Target { + targets := make([]gnmicv1alpha1.Target, len) + + if len > 100 { + len = 100 + } + + for i := range len { + targets[i] = gnmicv1alpha1.Target{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("router%d", i+1), + Namespace: "default", + }, + Spec: gnmicv1alpha1.TargetSpec{ + Address: fmt.Sprintf("192.168.1.%d", i+1), + Profile: "default", + }, + } + } + + return targets +} + +func TestGenerateEventsEmptyList(t *testing.T) { + e := mockGnmicTargetList(0) + d := mockDiscoveredTargetList(0) + + if events := generateEvents(e, d); len(events) != 0 { + t.Errorf("Wanted 0 events, got: %d", len(events)) + } +} + +func TestGenerateEventsEmptyExisting(t *testing.T) { + len_e := 0 + len_d := 5 + + e := mockGnmicTargetList(len_e) + d := mockDiscoveredTargetList(len_d) + + events := generateEvents(e, d) + + if len(events) != len_d { + t.Errorf("Wanted %d events, got: %d", len_d, len(events)) + } + + for _, event := range events { + if event.Event != core.EventApply { + t.Errorf("Wanted event APPLY, got: %s", event.Event.ToString()) + } + } +} + +func TestGenerateEventsEmptyDiscovery(t *testing.T) { + len_e := 5 + len_d := 0 + + e := mockGnmicTargetList(len_e) + d := mockDiscoveredTargetList(len_d) + + events := generateEvents(e, d) + + if len(events) != len_e { + t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + } + + for _, event := range events { + if event.Event != core.EventDelete { + t.Errorf("Wanted event APPLY, got: %s", event.Event.ToString()) + } + } +} + +func TestGenerateEventsMoreExisting(t *testing.T) { + len_e := 5 + len_d := 3 + + e := mockGnmicTargetList(len_e) + d := mockDiscoveredTargetList(len_d) + + events := generateEvents(e, d) + + if len(events) != len_e { + t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + } + + seenApply := false + numApply := 0 + numDelete := 0 + + for _, event := range events { + if event.Event == core.EventDelete && seenApply == true { + t.Error("Want delete events before apply events, got inversed") + } else if event.Event == core.EventDelete { + numDelete++ + } else if event.Event == core.EventApply { + seenApply = true + numApply++ + } + } + + if numDelete != len_e-len_d { + t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) + } else if numApply != len_d { + t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + } +} + +func TestGenerateEventsMoreDiscovered(t *testing.T) { + len_e := 3 + len_d := 5 + + e := mockGnmicTargetList(len_e) + d := mockDiscoveredTargetList(len_d) + + events := generateEvents(e, d) + + if len(events) != len_d { + t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + } + + seenApply := false + numApply := 0 + numDelete := 0 + + for _, event := range events { + if event.Event == core.EventDelete && seenApply == true { + t.Error("Want delete events before apply events, got inversed") + } else if event.Event == core.EventDelete { + numDelete++ + } else if event.Event == core.EventApply { + seenApply = true + numApply++ + } + } + + if numDelete != 0 { + t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) + } else if numApply != len_d { + t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + } +} + +func TestGenerateEventsNonOverlappingLists(t *testing.T) { + len_e := 5 + len_d := 5 + + e := mockGnmicTargetList(len_e) + d := mockDiscoveredTargetList(len_e + len_d)[len_e:] + + events := generateEvents(e, d) + + if len(events) != len_e+len_d { + t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + } + + seenApply := false + numApply := 0 + numDelete := 0 + + for _, event := range events { + if event.Event == core.EventDelete && seenApply == true { + t.Error("Want delete events before apply events, got inversed") + } else if event.Event == core.EventDelete { + numDelete++ + } else if event.Event == core.EventApply { + seenApply = true + numApply++ + } + } + + if numDelete != len_e { + t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) + } else if numApply != len_d { + t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + } +} From 17bd490f683d9eddae42e034c2d15b74449c1e0b Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 8 May 2026 18:01:25 -0600 Subject: [PATCH 199/246] rewrote and added more tests for mapper.go --- internal/controller/discovery/mapper_test.go | 425 +++++++++++++++---- 1 file changed, 336 insertions(+), 89 deletions(-) diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go index 2475fff..31e43bb 100644 --- a/internal/controller/discovery/mapper_test.go +++ b/internal/controller/discovery/mapper_test.go @@ -27,6 +27,81 @@ func mockDiscoveredTargetList(len int) []core.DiscoveredTarget { return targets } +func mockDiscoveryTarget(opts ...func(*core.DiscoveredTarget)) core.DiscoveredTarget { + t := core.DiscoveredTarget{ + Name: "target1", + Address: "10.0.0.1", + Labels: map[string]string{}, + } + + for _, opt := range opts { + opt(&t) + } + + return t +} + +func withDiscoveredTargetName(name string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Name = name + } +} + +func withDiscoveredTargetAddress(address string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Address = address + } +} + +func withDiscoveredTargetLabels(labels map[string]string) func(*core.DiscoveredTarget) { + return func(t *core.DiscoveredTarget) { + t.Labels = labels + } +} + +func mockTargetSource(opts ...func(*gnmicv1alpha1.TargetSource)) gnmicv1alpha1.TargetSource { + ts := gnmicv1alpha1.TargetSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ts1", + Namespace: "default", + }, + Spec: gnmicv1alpha1.TargetSourceSpec{ + TargetProfile: "default", + TargetLabels: map[string]string{}, + }, + } + + for _, opt := range opts { + opt(&ts) + } + + return ts +} + +func withTargetSourceName(name string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.ObjectMeta.Name = name + } +} + +func withTargetSourceNamespace(namespace string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.ObjectMeta.Namespace = namespace + } +} + +func withTargetSourceTargetProfile(profile string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.Spec.TargetProfile = profile + } +} + +func withTargetSourceTargetLabels(labels map[string]string) func(*gnmicv1alpha1.TargetSource) { + return func(ts *gnmicv1alpha1.TargetSource) { + ts.Spec.TargetLabels = labels + } +} + func mockGnmicTargetList(len int) []gnmicv1alpha1.Target { targets := make([]gnmicv1alpha1.Target, len) @@ -50,156 +125,328 @@ func mockGnmicTargetList(len int) []gnmicv1alpha1.Target { return targets } -func TestGenerateEventsEmptyList(t *testing.T) { - e := mockGnmicTargetList(0) - d := mockDiscoveredTargetList(0) +func TestGenerateEvents_EmptyLists(t *testing.T) { + events := generateEvents( + mockGnmicTargetList(0), + mockDiscoveredTargetList(0), + ) - if events := generateEvents(e, d); len(events) != 0 { - t.Errorf("Wanted 0 events, got: %d", len(events)) + if len(events) != 0 { + t.Fatalf("expected 0 events, got %d", len(events)) } } -func TestGenerateEventsEmptyExisting(t *testing.T) { - len_e := 0 - len_d := 5 - - e := mockGnmicTargetList(len_e) - d := mockDiscoveredTargetList(len_d) +func TestGenerateEvents_AllDiscoveredTargetsBecomeApplyEvents(t *testing.T) { + discovered := mockDiscoveredTargetList(5) - events := generateEvents(e, d) + events := generateEvents( + mockGnmicTargetList(0), + discovered, + ) - if len(events) != len_d { - t.Errorf("Wanted %d events, got: %d", len_d, len(events)) + if len(events) != len(discovered) { + t.Fatalf("expected %d events, got %d", len(discovered), len(events)) } for _, event := range events { if event.Event != core.EventApply { - t.Errorf("Wanted event APPLY, got: %s", event.Event.ToString()) + t.Fatalf( + "expected all events to be %s, got %s", + core.EventApply.ToString(), + event.Event.ToString(), + ) } } } -func TestGenerateEventsEmptyDiscovery(t *testing.T) { - len_e := 5 - len_d := 0 +func TestGenerateEvents_AllExistingTargetsBecomeDeleteEvents(t *testing.T) { + existing := mockGnmicTargetList(5) - e := mockGnmicTargetList(len_e) - d := mockDiscoveredTargetList(len_d) + events := generateEvents( + existing, + mockDiscoveredTargetList(0), + ) - events := generateEvents(e, d) - - if len(events) != len_e { - t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + if len(events) != len(existing) { + t.Fatalf("expected %d events, got %d", len(existing), len(events)) } for _, event := range events { if event.Event != core.EventDelete { - t.Errorf("Wanted event APPLY, got: %s", event.Event.ToString()) + t.Fatalf( + "expected all events to be %s, got %s", + core.EventDelete.ToString(), + event.Event.ToString(), + ) } } } -func TestGenerateEventsMoreExisting(t *testing.T) { - len_e := 5 - len_d := 3 - - e := mockGnmicTargetList(len_e) - d := mockDiscoveredTargetList(len_d) - - events := generateEvents(e, d) +func TestGenerateEvents_GeneratesDeleteThenApplyEvents(t *testing.T) { + existing := mockGnmicTargetList(5) + discovered := mockDiscoveredTargetList(3) - if len(events) != len_e { - t.Errorf("Wanted %d events, got: %d", len_e, len(events)) - } + events := generateEvents(existing, discovered) - seenApply := false - numApply := 0 - numDelete := 0 + var ( + numDelete int + numApply int + seenApply bool + ) for _, event := range events { - if event.Event == core.EventDelete && seenApply == true { - t.Error("Want delete events before apply events, got inversed") - } else if event.Event == core.EventDelete { + switch event.Event { + case core.EventDelete: + if seenApply { + t.Fatalf("expected delete events before apply events") + } numDelete++ - } else if event.Event == core.EventApply { + + case core.EventApply: seenApply = true numApply++ } } - if numDelete != len_e-len_d { - t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) - } else if numApply != len_d { - t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + if numDelete != 2 { + t.Fatalf("expected 2 delete events, got %d", numDelete) + } + + if numApply != 3 { + t.Fatalf("expected 3 apply events, got %d", numApply) } } -func TestGenerateEventsMoreDiscovered(t *testing.T) { - len_e := 3 - len_d := 5 +func TestGenerateEvents_OnlyApplyEventsAreGeneratedForNewTargets(t *testing.T) { + existing := mockGnmicTargetList(3) + discovered := mockDiscoveredTargetList(5) + + events := generateEvents(existing, discovered) + + var ( + numDelete int + numApply int + ) - e := mockGnmicTargetList(len_e) - d := mockDiscoveredTargetList(len_d) + for _, event := range events { + switch event.Event { + case core.EventDelete: + numDelete++ + + case core.EventApply: + numApply++ + } + } - events := generateEvents(e, d) + if numDelete != 0 { + t.Fatalf("expected 0 delete events, got %d", numDelete) + } - if len(events) != len_d { - t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + if numApply != 5 { + t.Fatalf("expected 5 apply events, got %d", numApply) } +} + +func TestGenerateEvents_NonOverlappingListsGenerateDeleteAndApplyEvents(t *testing.T) { + existing := mockGnmicTargetList(5) - seenApply := false - numApply := 0 - numDelete := 0 + discovered := mockDiscoveredTargetList(10)[5:] + + events := generateEvents(existing, discovered) + + var ( + numDelete int + numApply int + seenApply bool + ) for _, event := range events { - if event.Event == core.EventDelete && seenApply == true { - t.Error("Want delete events before apply events, got inversed") - } else if event.Event == core.EventDelete { + switch event.Event { + case core.EventDelete: + if seenApply { + t.Fatalf("expected delete events before apply events") + } numDelete++ - } else if event.Event == core.EventApply { + + case core.EventApply: seenApply = true numApply++ } } - if numDelete != 0 { - t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) - } else if numApply != len_d { - t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + if numDelete != 5 { + t.Fatalf("expected 5 delete events, got %d", numDelete) + } + + if numApply != 5 { + t.Fatalf("expected 5 apply events, got %d", numApply) } } -func TestGenerateEventsNonOverlappingLists(t *testing.T) { - len_e := 5 - len_d := 5 +func TestGenerateTargetResource_SetsTargetSourceNameLabel(t *testing.T) { + ts := mockTargetSource() + d := mockDiscoveryTarget() - e := mockGnmicTargetList(len_e) - d := mockDiscoveredTargetList(len_e + len_d)[len_e:] + target := generateTargetResource(d, &ts) + + if got := target.Labels[LabelTargetSourceName]; got != ts.Name { + t.Fatalf( + "expected %s=%q, got %q", + LabelTargetSourceName, + ts.Name, + got, + ) + } +} - events := generateEvents(e, d) +func TestGenerateTargetResource_CopiesDiscoveredLabels(t *testing.T) { + d := mockDiscoveryTarget( + withDiscoveredTargetLabels(map[string]string{ + "discoveredLabel1": "discoveredValue1", + "discoveredLabel2": "discoveredValue2", + }), + ) - if len(events) != len_e+len_d { - t.Errorf("Wanted %d events, got: %d", len_e, len(events)) + ts := mockTargetSource() + + target := generateTargetResource(d, &ts) + + tests := map[string]string{ + "discoveredLabel1": "discoveredValue1", + "discoveredLabel2": "discoveredValue2", } - seenApply := false - numApply := 0 - numDelete := 0 + for k, want := range tests { + if got := target.Labels[k]; got != want { + t.Fatalf("expected label %s=%q, got %q", k, want, got) + } + } +} - for _, event := range events { - if event.Event == core.EventDelete && seenApply == true { - t.Error("Want delete events before apply events, got inversed") - } else if event.Event == core.EventDelete { - numDelete++ - } else if event.Event == core.EventApply { - seenApply = true - numApply++ +func TestGenerateTargetResource_CopiesTargetSourceLabels(t *testing.T) { + ts := mockTargetSource( + withTargetSourceTargetLabels(map[string]string{ + "targetSourceLabel1": "targetSourceValue1", + "targetSourceLabel2": "targetSourceValue2", + }), + ) + + d := mockDiscoveryTarget() + + target := generateTargetResource(d, &ts) + + tests := map[string]string{ + "targetSourceLabel1": "targetSourceValue1", + "targetSourceLabel2": "targetSourceValue2", + } + + for k, want := range tests { + if got := target.Labels[k]; got != want { + t.Fatalf("expected label %s=%q, got %q", k, want, got) } } +} + +func TestGenerateTargetResource_OverridesReservedTargetSourceNameLabel(t *testing.T) { + ts := mockTargetSource( + withTargetSourceTargetLabels(map[string]string{ + LabelTargetSourceName: "wrong-value", + }), + ) + + d := mockDiscoveryTarget( + withDiscoveredTargetLabels(map[string]string{ + LabelTargetSourceName: "another-wrong-value", + }), + ) + + target := generateTargetResource(d, &ts) + + if got := target.Labels[LabelTargetSourceName]; got != ts.Name { + t.Fatalf( + "expected reserved label %s=%q, got %q", + LabelTargetSourceName, + ts.Name, + got, + ) + } +} + +func TestGenerateTargetResource_TargetSourceLabelsOverrideDiscoveredLabels(t *testing.T) { + ts := mockTargetSource( + withTargetSourceTargetLabels(map[string]string{ + "sharedLabel": "targetSourceValue", + }), + ) + + d := mockDiscoveryTarget( + withDiscoveredTargetLabels(map[string]string{ + "sharedLabel": "discoveredValue", + }), + ) + + target := generateTargetResource(d, &ts) + + if got := target.Labels["sharedLabel"]; got != "discoveredValue" { + t.Fatalf( + "expected target source label to override discovered label, got %q", + got, + ) + } +} + +func TestNormalizeTarget_PrefixesTargetName(t *testing.T) { + target := mockDiscoveryTarget( + withDiscoveredTargetName("router1"), + ) + + normalized := normalizeTarget(target, "ts1") + + if got := normalized.Name; got != "ts1-router1" { + t.Fatalf( + "expected normalized name %q, got %q", + "ts1-router1", + got, + ) + } +} + +func TestNormalizeTarget_PreservesTargetAddress(t *testing.T) { + target := mockDiscoveryTarget( + withDiscoveredTargetAddress("192.168.1.10"), + ) + + normalized := normalizeTarget(target, "ts1") + + if got := normalized.Address; got != "192.168.1.10" { + t.Fatalf( + "expected address %q, got %q", + "192.168.1.10", + got, + ) + } +} + +func TestNormalizeTarget_PreservesTargetLabels(t *testing.T) { + labels := map[string]string{ + "env": "prod", + "role": "leaf", + } - if numDelete != len_e { - t.Errorf("Wanted %d delete events, got: %d", len_e-len_d, numDelete) - } else if numApply != len_d { - t.Errorf("Wanted %d apply events, got: %d", len_d, numApply) + target := mockDiscoveryTarget( + withDiscoveredTargetLabels(labels), + ) + + normalized := normalizeTarget(target, "ts1") + + for k, want := range labels { + if got := normalized.Labels[k]; got != want { + t.Fatalf( + "expected label %s=%q, got %q", + k, + want, + got, + ) + } } } From 9208766b0cc3a0eeeaf0e82270a818bd724e4cc5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 07:43:33 +0000 Subject: [PATCH 200/246] remove closeChannel and fix cleanup logic --- internal/controller/targetsource_controller.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c65e254..ccdf430 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -177,7 +177,6 @@ func (r *TargetSourceReconciler) startDiscovery( cleanup := func() { cancel() r.DiscoveryRegistry.Unregister(key) - close(targetChannel) } messageProcessor := discovery.NewMessageProcessor( @@ -225,9 +224,6 @@ func (r *TargetSourceReconciler) startDiscovery( } else { logger.Error(nil, "Target loader exited unexpectedly without error") } - - // Any exit is considered a bug that should stop the discovery runtime - cleanup() }() return nil From 209948e2e1c2cebb1bbc78d4183d6be6bcdd3805 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 07:54:23 +0000 Subject: [PATCH 201/246] fix: resolved pointer and returns smells --- internal/controller/discovery/loaders.go | 8 ++++---- internal/controller/targetsource_controller.go | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 7f2c656..f882aa2 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -9,16 +9,16 @@ import ( ) // NewLoader creates a loader by name -func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, core.CommonLoaderConfig, error) { +func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { switch { case spec.Provider.HTTP != nil: cfg.AcceptPush = spec.Provider.HTTP.AcceptPush - return http.New(cfg), cfg, nil + return http.New(*cfg), nil case spec.Provider.Consul != nil: - return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index ccdf430..522aabd 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -185,9 +185,7 @@ func (r *TargetSourceReconciler) startDiscovery( targetSource, targetChannel, ) - loader, loaderConfig, err := discovery.NewLoader(loaderConfig, - &targetSource.Spec, - ) + loader, err := discovery.NewLoader(&loaderConfig, targetSource.Spec) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() From 1a0f4476b6149efef34bd38f599b29c0faedd6e4 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:04:50 +0000 Subject: [PATCH 202/246] improved logging message --- internal/controller/discovery/loaders.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index f882aa2..c888c27 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -16,9 +16,9 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec cfg.AcceptPush = spec.Provider.HTTP.AcceptPush return http.New(*cfg), nil case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, fmt.Errorf("Unimplemented targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) } } From d94c23fb5f20c061e95ff148d0e8e4bfae6e98f5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:23:48 +0000 Subject: [PATCH 203/246] improved error handling --- .../controller/discovery/message_processor.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index ed66940..b16603e 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -127,8 +127,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { if m.activeSnapshot == nil { - m.startNewSnapshot(chunk, logger) - return nil + return m.startNewSnapshot(ctx, chunk, logger) } snapshot := m.activeSnapshot @@ -149,14 +148,13 @@ func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.Disco } // Start collecting the new snapshot - m.startNewSnapshot(chunk, logger) - return nil + return m.startNewSnapshot(ctx, chunk, logger) } - return m.collectSnapshot(chunk, logger) + return m.collectSnapshot(ctx, chunk, logger) } -func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { +func (m *MessageProcessor) startNewSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { m.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, @@ -166,10 +164,10 @@ func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger // Delete buffered events that will be current with new snapshot m.deferredEvents = nil - m.collectSnapshot(chunk, logger) + return m.collectSnapshot(ctx, chunk, logger) } -func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { +func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { snapshot := m.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { @@ -178,6 +176,7 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger "Snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID, ) + return fmt.Errorf("snapshot totalChunks mismatch") } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error( @@ -186,7 +185,7 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger "chunkIndex", chunk.ChunkIndex, ) m.activeSnapshot = nil - return nil + return fmt.Errorf("invalid chunk index") } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error( @@ -195,13 +194,14 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger "chunkIndex", chunk.ChunkIndex, ) m.activeSnapshot = nil - return nil + return fmt.Errorf("duplicate snapshot chunk") } snapshot.received[chunk.ChunkIndex] = chunk.Targets if len(snapshot.received) == snapshot.totalChunks { snapshot.complete = true + return m.applySnapshot(ctx, snapshot, logger) } return nil @@ -232,7 +232,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot "chunkIndex", i, ) m.activeSnapshot = nil - return nil + return fmt.Errorf("missing snapshot chunk %d", i) } allTargets = append(allTargets, chunk...) } @@ -243,7 +243,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot "targets", len(allTargets), ) - // apply all targets + // todo: apply all targets // a.applyTargets // Replay deferred events From e3f18d8a213be2afd81f5c617eaae1b7ad068652 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:26:05 +0000 Subject: [PATCH 204/246] refactor: ctx should flow not be stored --- internal/controller/discovery/message_processor.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index b16603e..a95a208 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -22,7 +22,6 @@ type snapshotBuffer struct { // MessageProcessor consumes discovery messages and applies them to Kubernetes type MessageProcessor struct { - ctx context.Context client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource @@ -46,8 +45,6 @@ func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly func (m *MessageProcessor) Run(ctx context.Context) error { - m.ctx = ctx - logger := log.FromContext(ctx).WithValues( "component", "message-processor", "targetsource", m.targetSource.Name, @@ -56,7 +53,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { logger.Info("Message processor started") - for m.ctx.Err() == nil { + for ctx.Err() == nil { select { case batch, ok := <-m.in: if !ok { @@ -79,7 +76,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { msg := m.queue[0] m.queue = m.queue[1:] - if err := m.processMessage(m.ctx, msg, logger); err != nil { + if err := m.processMessage(ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? From bc1b3508e7e34f2f5c199f448bf7baceb2966b6a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:30:48 +0000 Subject: [PATCH 205/246] refactor: resetSnapshot --- .../controller/discovery/message_processor.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index a95a208..bbefd24 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -181,7 +181,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco "Snapshot chunk index out of range", "chunkIndex", chunk.ChunkIndex, ) - m.activeSnapshot = nil + m.resetSnapshot() return fmt.Errorf("invalid chunk index") } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { @@ -190,7 +190,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco "Duplicate snapshot chunk received", "chunkIndex", chunk.ChunkIndex, ) - m.activeSnapshot = nil + m.resetSnapshot() return fmt.Errorf("duplicate snapshot chunk") } @@ -207,7 +207,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - m.activeSnapshot = nil + m.resetSnapshot() return nil default: } @@ -216,7 +216,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - m.activeSnapshot = nil + m.resetSnapshot() return nil default: } @@ -228,7 +228,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot "Missing snapshot chunk", "chunkIndex", i, ) - m.activeSnapshot = nil + m.resetSnapshot() return fmt.Errorf("missing snapshot chunk %d", i) } allTargets = append(allTargets, chunk...) @@ -255,7 +255,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot } } - m.activeSnapshot = nil + m.resetSnapshot() m.deferredEvents = nil return nil } @@ -290,3 +290,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE } return nil } + +func (m *MessageProcessor) resetSnapshot() { + m.activeSnapshot = nil +} From 020be5ae1d507b05051b37edc59bb5d6fc123dc0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:31:51 +0000 Subject: [PATCH 206/246] refactor: context cancellation --- internal/controller/discovery/message_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index bbefd24..b1d893e 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -53,7 +53,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { logger.Info("Message processor started") - for ctx.Err() == nil { + for { select { case batch, ok := <-m.in: if !ok { From 3280229ed3374583925b4c3600dea5f2115d219e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 08:48:30 +0000 Subject: [PATCH 207/246] refactor: default error handling now logs errors instead of terminating the message processor --- internal/controller/discovery/message_processor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index b1d893e..f7aafb1 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -79,15 +79,15 @@ func (m *MessageProcessor) Run(ctx context.Context) error { if err := m.processMessage(ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation - // Q: when to return an error vs just log and continue? - return err + logger.Info( + "Could not process the message", + "error", err, + ) + return nil } } } - - logger.Info("Message processor stopped") - return nil } func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { From 39f16502bd1e399cf30567818ee5b22a1104e24d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 09:21:34 +0000 Subject: [PATCH 208/246] refactor: pointer missuse --- internal/controller/discovery/loaders/http/loader.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index b55bcef..e791af9 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -19,12 +19,12 @@ import ( // Loader implements the HTTP pull discovery mechanism type Loader struct { loaderCfg core.CommonLoaderConfig - spec *gnmicv1alpha1.HTTPConfig + spec gnmicv1alpha1.HTTPConfig } // New instantiates the http loader with the provided config func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader { - return &Loader{loaderCfg: cfg, spec: &httpConfig} + return &Loader{loaderCfg: cfg, spec: httpConfig} } func (l *Loader) Name() string { @@ -74,8 +74,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er ctx, client, l.spec.URL, - l.spec.Authorization.Token.Scheme, - l.spec.Authorization.Token.Token, + *l.spec.Authorization.Token.Scheme, + *l.spec.Authorization.Token.Token, ) if err != nil { logger.Error( From f1d8c3165c9687107fa02a47f47ea92bff83c3be Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 09:35:05 +0000 Subject: [PATCH 209/246] move defaulting logic to kubebuilder:default --- api/v1alpha1/targetsource_types.go | 32 +++++---- api/v1alpha1/zz_generated.deepcopy.go | 72 +------------------ .../operator.gnmic.dev_targetsources.yaml | 3 + .../discovery/loaders/http/loader.go | 4 +- .../webhook/v1alpha1/targetsource_webhook.go | 17 +---- 5 files changed, 25 insertions(+), 103 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 78a2f74..0ca98ab 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -44,8 +44,11 @@ type HTTPConfig struct { URL string `json:"url,omitempty"` // +kubebuilder:validation:Optional Authorization *AuthorizationSpec `json:"authorization,omitempty"` + // TODO: increase default value + // +kubebuilder:default="30s" // +kubebuilder:validation:Optional PollInterval *metav1.Duration `json:"interval,omitempty"` + // +kubebuilder:default="10s" // +kubebuilder:validation:Optional Timeout *metav1.Duration `json:"timeout,omitempty"` // +kubebuilder:validation:Optional @@ -54,8 +57,9 @@ type HTTPConfig struct { Pagination *PaginationSpec `json:"pagination,omitempty"` // +kubebuilder:validation:Optional ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"` + // +kubebuilder:default=false // +kubebuilder:validation:Optional - AcceptPush *bool `json:"acceptPush,omitempty"` + AcceptPush bool `json:"acceptPush,omitempty"` } type ClientTLSConfig struct { @@ -73,16 +77,16 @@ type AuthorizationSpec struct { // Enforce EITHER inline creds OR secret ref // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix" type BasicAuthSpec struct { - Username *string `json:"username,omitempty"` - Password *string `json:"password,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"` } // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both" type TokenAuthSpec struct { // +kubebuilder:validation:MinLength=1 - Scheme *string `json:"scheme"` - Token *string `json:"token,omitempty"` + Scheme string `json:"scheme"` + Token string `json:"token,omitempty"` TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } @@ -91,35 +95,35 @@ type TokenAuthSpec struct { // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" type JWTAuthSpec struct { // Static pre-generated JWT - Token *string `json:"token,omitempty"` + Token string `json:"token,omitempty"` TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` // Optional: generate JWT dynamically Claims map[string]string `json:"claims,omitempty"` SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"` // HS256, RS256, ES256, etc. - Algorithm *string `json:"algorithm,omitempty"` + Algorithm string `json:"algorithm,omitempty"` TTL *metav1.Duration `json:"ttl,omitempty"` } type PaginationSpec struct { // Example: "results" - ItemsField *string `json:"itemsField,omitempty"` + ItemsField string `json:"itemsField,omitempty"` // Example: "next" - NextField *string `json:"nextField,omitempty"` + NextField string `json:"nextField,omitempty"` } // JSONPath-style expressions type ResponseMappingSpec struct { - Name *string `json:"name"` - Address *string `json:"address"` - Port *string `json:"port,omitempty"` + Name string `json:"name"` + Address string `json:"address"` + Port string `json:"port,omitempty"` Labels map[string]string `json:"labels,omitempty"` } // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { - Status *string `json:"status"` - TargetsCount *int32 `json:"targetsCount"` + Status string `json:"status"` + TargetsCount int32 `json:"targetsCount"` LastSync metav1.Time `json:"lastSync"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 453dcc9..d87d54d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -79,16 +79,6 @@ func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) { *out = *in - if in.Username != nil { - in, out := &in.Username, &out.Username - *out = new(string) - **out = **in - } - if in.Password != nil { - in, out := &in.Password, &out.Password - *out = new(string) - **out = **in - } if in.CredentialsSecretRef != nil { in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef *out = new(v1.SecretKeySelector) @@ -366,18 +356,13 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) { if in.Pagination != nil { in, out := &in.Pagination, &out.Pagination *out = new(PaginationSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.ResponseMapping != nil { in, out := &in.ResponseMapping, &out.ResponseMapping *out = new(ResponseMappingSpec) (*in).DeepCopyInto(*out) } - if in.AcceptPush != nil { - in, out := &in.AcceptPush, &out.AcceptPush - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig. @@ -522,11 +507,6 @@ func (in *InputStatus) DeepCopy() *InputStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { *out = *in - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(string) - **out = **in - } if in.TokenSecretRef != nil { in, out := &in.TokenSecretRef, &out.TokenSecretRef *out = new(v1.SecretKeySelector) @@ -544,11 +524,6 @@ func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } - if in.Algorithm != nil { - in, out := &in.Algorithm, &out.Algorithm - *out = new(string) - **out = **in - } if in.TTL != nil { in, out := &in.TTL, &out.TTL *out = new(metav1.Duration) @@ -742,16 +717,6 @@ func (in *OutputStatus) DeepCopy() *OutputStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) { *out = *in - if in.ItemsField != nil { - in, out := &in.ItemsField, &out.ItemsField - *out = new(string) - **out = **in - } - if in.NextField != nil { - in, out := &in.NextField, &out.NextField - *out = new(string) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec. @@ -1018,21 +983,6 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) { *out = *in - if in.Name != nil { - in, out := &in.Name, &out.Name - *out = new(string) - **out = **in - } - if in.Address != nil { - in, out := &in.Address, &out.Address - *out = new(string) - **out = **in - } - if in.Port != nil { - in, out := &in.Port, &out.Port - *out = new(string) - **out = **in - } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) @@ -1523,16 +1473,6 @@ func (in *TargetSourceSpec) DeepCopy() *TargetSourceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetSourceStatus) DeepCopyInto(out *TargetSourceStatus) { *out = *in - if in.Status != nil { - in, out := &in.Status, &out.Status - *out = new(string) - **out = **in - } - if in.TargetsCount != nil { - in, out := &in.TargetsCount, &out.TargetsCount - *out = new(int32) - **out = **in - } in.LastSync.DeepCopyInto(&out.LastSync) } @@ -1606,16 +1546,6 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) { *out = *in - if in.Scheme != nil { - in, out := &in.Scheme, &out.Scheme - *out = new(string) - **out = **in - } - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(string) - **out = **in - } if in.TokenSecretRef != nil { in, out := &in.TokenSecretRef, &out.TokenSecretRef *out = new(v1.SecretKeySelector) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 588bccf..2e6872a 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -44,6 +44,7 @@ spec: http: properties: acceptPush: + default: false type: boolean authorization: properties: @@ -211,6 +212,7 @@ spec: rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size() == 1' interval: + default: 30s type: string mapping: description: JSONPath-style expressions @@ -239,6 +241,7 @@ spec: type: string type: object timeout: + default: 10s type: string tls: properties: diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index e791af9..bb8a025 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -74,8 +74,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er ctx, client, l.spec.URL, - *l.spec.Authorization.Token.Scheme, - *l.spec.Authorization.Token.Token, + l.spec.Authorization.Token.Scheme, + l.spec.Authorization.Token.Token, ) if err != nil { logger.Error( diff --git a/internal/webhook/v1alpha1/targetsource_webhook.go b/internal/webhook/v1alpha1/targetsource_webhook.go index 992d67b..b3eb960 100644 --- a/internal/webhook/v1alpha1/targetsource_webhook.go +++ b/internal/webhook/v1alpha1/targetsource_webhook.go @@ -21,14 +21,12 @@ import ( "fmt" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - httpDefaults "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // nolint:unused @@ -67,20 +65,7 @@ func (d *TargetSourceCustomDefaulter) Default(_ context.Context, obj runtime.Obj } targetsourcelog.Info("Defaulting for TargetSource", "name", targetsource.GetName()) - // HTTP Config Defaulting - if targetsource.Spec.Provider.HTTP != nil { - http := targetsource.Spec.Provider.HTTP - - if http.PollInterval == nil { - http.PollInterval.Duration = httpDefaults.DefaultPollInterval - } - if http.Timeout == nil { - http.Timeout.Duration = httpDefaults.DefaultTimeout - } - if http.AcceptPush == nil { - http.AcceptPush = pointer.Bool(httpDefaults.DefaultAcceptPush) - } - } + // TODO(user): fill in your defaulting logic. return nil } From b0c63ff27f437524233e565ea27c262fbe721bd9 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 09:36:02 +0000 Subject: [PATCH 210/246] remove pointer from bool --- api/v1alpha1/targetsource_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 0ca98ab..7d0ca67 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -63,7 +63,7 @@ type HTTPConfig struct { } type ClientTLSConfig struct { - InsecureSkipVerify *bool `json:"insecureSkipVerify,omitempty"` + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` CASecretRef *corev1.SecretKeySelector `json:"caSecretRef,omitempty"` } From c422dff6a4a1c0b5affea49db7db00734a267479 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 09:38:55 +0000 Subject: [PATCH 211/246] update deepcopy --- api/v1alpha1/zz_generated.deepcopy.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d87d54d..f115201 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -99,11 +99,6 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) { *out = *in - if in.InsecureSkipVerify != nil { - in, out := &in.InsecureSkipVerify, &out.InsecureSkipVerify - *out = new(bool) - **out = **in - } if in.CASecretRef != nil { in, out := &in.CASecretRef, &out.CASecretRef *out = new(v1.SecretKeySelector) From abb718089d5e4b066df1eda0ba666c65ff375ba4 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 13 May 2026 09:41:34 +0000 Subject: [PATCH 212/246] fix: pointer issue --- internal/controller/discovery/loaders.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 57588f2..9143ae4 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -13,7 +13,7 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec switch { case spec.Provider.HTTP != nil: - cfg.AcceptPush = *spec.Provider.HTTP.AcceptPush + cfg.AcceptPush = spec.Provider.HTTP.AcceptPush return http.New(*cfg, *spec.Provider.HTTP), nil default: return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) From 35c18a2ee98a8861e7cf0d62003b957532705564 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 13 May 2026 11:07:12 +0000 Subject: [PATCH 213/246] add ip checker and default port (if empty) --- internal/apiserver/apiserver.go | 12 ++++++++++++ internal/apiserver/helpers.go | 19 ++++++++++++++++++- internal/apiserver/helpers_test.go | 30 ++++++++++++++++++++++++++++++ internal/apiserver/temp.md | 2 +- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index a46837f..2bf7b30 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,6 +6,18 @@ package apiserver // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +// kubectl get svc on different branch +// NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +// gnmic-c1 ClusterIP None 7890/TCP 30h +// gnmic-c1-prom-cpipe1-prom-output1 ClusterIP 10.96.202.243 9916/TCP 30h +// gnmic-c1-prom-p1-prom-output1 ClusterIP 10.96.191.84 10344/TCP 30h +// kubernetes ClusterIP 10.96.0.1 443/TCP 27d +// in this branch, the prometheus output is gone -> bug to +// kubectl get svc +// NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +// gnmic-c1 ClusterIP None 7890/TCP 5m14s +// kubernetes ClusterIP 10.96.0.1 443/TCP 29d + import ( "context" "fmt" diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 487e371..48614d0 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -2,6 +2,7 @@ package apiserver import ( "fmt" + "net" "net/http" "github.com/gin-gonic/gin" @@ -26,11 +27,15 @@ func createDiscoveryEvent(payloadTargets []target) ([]core.DiscoveryEvent, error if err != nil { return nil, err } + verifiedAddress, err := validateAddress(target.Address) + if err != nil { + return nil, err + } targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, - Address: target.Address, + Address: verifiedAddress, Labels: convertTargetLabelsToMap(target), }, Event: event, @@ -40,6 +45,18 @@ func createDiscoveryEvent(payloadTargets []target) ([]core.DiscoveryEvent, error return targets, nil } +// validateAddress checks if the address is of format IPv4 or IPv6 and adds default port 57400 if empty. +func validateAddress(address string) (string, error) { + address, port, err := net.SplitHostPort(address) + if err != nil { + return "", err + } + if port == "" { + port = "57400" + } + return address + ":" + port, nil +} + // getKey returns key for used to identify correct channel in DiscoveryRegistry func getKey(u urlStruct) types.NamespacedName { key := types.NamespacedName{ diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go index 9bfa41b..58ca283 100644 --- a/internal/apiserver/helpers_test.go +++ b/internal/apiserver/helpers_test.go @@ -263,3 +263,33 @@ func TestParseURIMissingName(t *testing.T) { t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusBadRequest) } } + +func TestVerifyAddress(t *testing.T) { + address := "10.10.10.10:57400" + expected := "10.10.10.10:57400" + convertedAddress, _ := validateAddress(address) + if !reflect.DeepEqual(convertedAddress, expected) { + t.Errorf("addDefaultPortIfEmpty(address) = %s; want %s", convertedAddress, expected) + } +} + +func TestVerifyAddressIPv6(t *testing.T) { + address := "[2345:0425:2CA1:0000:0000:0567:5673:23b5]:57400" + expected := "2345:0425:2CA1:0000:0000:0567:5673:23b5:57400" + convertedAddress, _ := validateAddress(address) + if !reflect.DeepEqual(convertedAddress, expected) { + t.Errorf("addDefaultPortIfEmpty(address) = %s; want %s", convertedAddress, expected) + } +} + +func TestVerifyAddressNoPort(t *testing.T) { + address := "10.10.10.10:" + expected := "10.10.10.10:57400" + convertedAddress, err := validateAddress(address) + if err != nil { + t.Errorf("addDefaultPortIfEmpty(address) threw unexpected error: %s", err) + } + if !reflect.DeepEqual(convertedAddress, expected) { + t.Errorf("addDefaultPortIfEmpty(address) = %s; want %s", convertedAddress, expected) + } +} diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index afc0eb9..1acdf6a 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -2,7 +2,7 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/ -H "Content-Type: application/json" \ -d '[ { - "address": "1.1.1.1", + "address": "1.1.1.1:123", "name": "Router1", "operation": "created", "profile": "defaultProfile", From 6ccf4ba292f1ff2d2737616ba95ffed3e1172dc0 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 13 May 2026 13:09:22 -0600 Subject: [PATCH 214/246] added handling for unknown operator labels --- internal/controller/discovery/mapper.go | 10 ++++++---- internal/controller/discovery/mapper_test.go | 10 +++++----- internal/controller/discovery/message_processor.go | 9 ++++++++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index 36ce541..ecbe23c 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -10,8 +10,9 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) -// generateTargetResource converts a DiscoveredTarget into a Kubernetes Target Object based on the TargetSource Spec -func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSource) *gnmicv1alpha1.Target { +// generateTargetResource converts a DiscoveredTarget into a Kubernetes Target Object based on the TargetSource Spec. +// Returns the Target Resource and a map of unknown operator labels. +func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSource) (*gnmicv1alpha1.Target, map[string]string) { // Create object instance t := &gnmicv1alpha1.Target{ ObjectMeta: metav1.ObjectMeta{ @@ -20,6 +21,7 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou Labels: make(map[string]string), }, } + unknownLabels := make(map[string]string) // Add Address from DiscoveredTarget t.Spec.Address = d.Address @@ -36,7 +38,7 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou case ExternalLabelTargetProfile: // Overwrite TargetProfile if specified by SoT t.Spec.Profile = v default: - // TODO: handle unknown label + unknownLabels[k] = v } } else { // Copy all other labels into the Target t.Labels[k] = v @@ -46,7 +48,7 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou // Add TargetSource Label to the Target (precedence over all labels) t.Labels[LabelTargetSourceName] = ts.Name - return t + return t, unknownLabels } // generateEvents returns a list of DiscoveryEvents. Needed for snapshot handling to determine which devices get deleted and which applied. diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go index 31e43bb..7cd9f77 100644 --- a/internal/controller/discovery/mapper_test.go +++ b/internal/controller/discovery/mapper_test.go @@ -287,7 +287,7 @@ func TestGenerateTargetResource_SetsTargetSourceNameLabel(t *testing.T) { ts := mockTargetSource() d := mockDiscoveryTarget() - target := generateTargetResource(d, &ts) + target, _ := generateTargetResource(d, &ts) if got := target.Labels[LabelTargetSourceName]; got != ts.Name { t.Fatalf( @@ -309,7 +309,7 @@ func TestGenerateTargetResource_CopiesDiscoveredLabels(t *testing.T) { ts := mockTargetSource() - target := generateTargetResource(d, &ts) + target, _ := generateTargetResource(d, &ts) tests := map[string]string{ "discoveredLabel1": "discoveredValue1", @@ -333,7 +333,7 @@ func TestGenerateTargetResource_CopiesTargetSourceLabels(t *testing.T) { d := mockDiscoveryTarget() - target := generateTargetResource(d, &ts) + target, _ := generateTargetResource(d, &ts) tests := map[string]string{ "targetSourceLabel1": "targetSourceValue1", @@ -360,7 +360,7 @@ func TestGenerateTargetResource_OverridesReservedTargetSourceNameLabel(t *testin }), ) - target := generateTargetResource(d, &ts) + target, _ := generateTargetResource(d, &ts) if got := target.Labels[LabelTargetSourceName]; got != ts.Name { t.Fatalf( @@ -385,7 +385,7 @@ func TestGenerateTargetResource_TargetSourceLabelsOverrideDiscoveredLabels(t *te }), ) - target := generateTargetResource(d, &ts) + target, _ := generateTargetResource(d, &ts) if got := target.Labels["sharedLabel"]; got != "discoveredValue" { t.Fatalf( diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index b495301..9b53390 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -347,7 +347,14 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE ) } case core.EventApply: - target := generateTargetResource(event.Target, m.targetSource) + target, unknownLabels := generateTargetResource(event.Target, m.targetSource) + for k, v := range unknownLabels { + logger.V(1).Info("unknown operator label for target", + "target", event.Target.Name, + "label", k, + "value", v, + ) + } if err := applyTarget(ctx, m.client, m.scheme, target, m.targetSource); err != nil { logger.Error(err, "error applying target", From 6b01748ab6f31fe4a0c82512b003505f25c2da97 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 13 May 2026 13:47:26 -0600 Subject: [PATCH 215/246] removed value for log verbosity --- internal/controller/discovery/message_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 9b53390..aa051b9 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -349,7 +349,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE case core.EventApply: target, unknownLabels := generateTargetResource(event.Target, m.targetSource) for k, v := range unknownLabels { - logger.V(1).Info("unknown operator label for target", + logger.Info("unknown operator label for target", "target", event.Target.Name, "label", k, "value", v, From 36cf9fddf31b5fbdbe704992971d920fbb43025b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 08:32:51 +0000 Subject: [PATCH 216/246] add helper to read secrets --- internal/controller/discovery/client.go | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index cb02161..9ccbb68 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -2,7 +2,9 @@ package discovery import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -30,3 +32,44 @@ func fetchExistingTargets( return targetList.Items, nil } + +// Helper: GetSecretValues returns values from a secret +// If keys are provided -> returns only those keys +// If keys is empty -> returns entire secret data +func GetSecretValues( + ctx context.Context, + c client.Client, + namespace string, + secretRef string, + keys ...string, +) (map[string]string, error) { + var secret corev1.Secret + if err := c.Get(ctx, + client.ObjectKey{ + Name: secretRef, + Namespace: namespace, + }, &secret); err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %w", namespace, secretRef, err) + } + + result := make(map[string]string) + + // Return full secret + if len(keys) == 0 { + for k, v := range secret.Data { + result[k] = string(v) + } + return result, nil + } + + // Return specific keys + for _, key := range keys { + val, ok := secret.Data[key] + if !ok { + return nil, fmt.Errorf("key %s missing in secret %s/%s", key, namespace, secretRef) + } + result[key] = string(val) + } + + return result, nil +} From 4f70c437aea340a8fd5d2c2a66bb839927e01489 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 08:33:04 +0000 Subject: [PATCH 217/246] implement basic and token authentication --- api/v1alpha1/targetsource_types.go | 39 ++++--- api/v1alpha1/zz_generated.deepcopy.go | 42 ------- .../operator.gnmic.dev_targetsources.yaml | 95 +++------------- internal/controller/discovery/loaders.go | 104 +++++++++++++++++- .../discovery/loaders/http/const.go | 9 -- .../discovery/loaders/http/loader.go | 48 +++++--- .../controller/targetsource_controller.go | 5 +- 7 files changed, 175 insertions(+), 167 deletions(-) delete mode 100644 internal/controller/discovery/loaders/http/const.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 7d0ca67..ee3838c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -63,7 +63,8 @@ type HTTPConfig struct { } type ClientTLSConfig struct { - InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + // +kubebuilder:default:=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` CASecretRef *corev1.SecretKeySelector `json:"caSecretRef,omitempty"` } @@ -71,14 +72,18 @@ type ClientTLSConfig struct { type AuthorizationSpec struct { Basic *BasicAuthSpec `json:"basic,omitempty"` Token *TokenAuthSpec `json:"token,omitempty"` - JWT *JWTAuthSpec `json:"jwt,omitempty"` + // JWT *JWTAuthSpec `json:"jwt,omitempty"` } // Enforce EITHER inline creds OR secret ref // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix" type BasicAuthSpec struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + // CredentialsSecretRef references a secret containing: + // - username + // - password + // NOTE: key field is ignored; fixed keys are used instead CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"` } @@ -90,20 +95,20 @@ type TokenAuthSpec struct { TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } -// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" -// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" +// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" -type JWTAuthSpec struct { - // Static pre-generated JWT - Token string `json:"token,omitempty"` - TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` - // Optional: generate JWT dynamically - Claims map[string]string `json:"claims,omitempty"` - SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"` - // HS256, RS256, ES256, etc. - Algorithm string `json:"algorithm,omitempty"` - TTL *metav1.Duration `json:"ttl,omitempty"` -} +// type JWTAuthSpec struct { +// // Static pre-generated JWT +// Token string `json:"token,omitempty"` +// TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` +// // Optional: generate JWT dynamically +// Claims map[string]string `json:"claims,omitempty"` +// Key string `json:"key,omitempty"` +// SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"` +// // HS256, RS256, ES256, etc. +// Algorithm string `json:"algorithm,omitempty"` +// TTL *metav1.Duration `json:"ttl,omitempty"` +// } type PaginationSpec struct { // Example: "results" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f115201..3844789 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -59,11 +59,6 @@ func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) { *out = new(TokenAuthSpec) (*in).DeepCopyInto(*out) } - if in.JWT != nil { - in, out := &in.JWT, &out.JWT - *out = new(JWTAuthSpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationSpec. @@ -499,43 +494,6 @@ func (in *InputStatus) DeepCopy() *InputStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) { - *out = *in - if in.TokenSecretRef != nil { - in, out := &in.TokenSecretRef, &out.TokenSecretRef - *out = new(v1.SecretKeySelector) - (*in).DeepCopyInto(*out) - } - if in.Claims != nil { - in, out := &in.Claims, &out.Claims - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.SigningKeySecretRef != nil { - in, out := &in.SigningKeySecretRef, &out.SigningKeySecretRef - *out = new(v1.SecretKeySelector) - (*in).DeepCopyInto(*out) - } - if in.TTL != nil { - in, out := &in.TTL, &out.TTL - *out = new(metav1.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthSpec. -func (in *JWTAuthSpec) DeepCopy() *JWTAuthSpec { - if in == nil { - return nil - } - out := new(JWTAuthSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Output) DeepCopyInto(out *Output) { *out = *in diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 2e6872a..aab3917 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -52,8 +52,11 @@ spec: description: Enforce EITHER inline creds OR secret ref properties: credentialsSecretRef: - description: SecretKeySelector selects a key of a - Secret. + description: |- + CredentialsSecretRef references a secret containing: + - username + - password + NOTE: key field is ignored; fixed keys are used instead properties: key: description: The key of the secret to select from. Must @@ -87,85 +90,6 @@ spec: rule: (has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password)) - jwt: - properties: - algorithm: - description: HS256, RS256, ES256, etc. - type: string - claims: - additionalProperties: - type: string - description: 'Optional: generate JWT dynamically' - type: object - signingKeySecretRef: - description: SecretKeySelector selects a key of a - Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - token: - description: Static pre-generated JWT - type: string - tokenSecretRef: - description: SecretKeySelector selects a key of a - Secret. - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its - key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - ttl: - type: string - type: object - x-kubernetes-validations: - - message: static JWT token and generated JWT configuration - cannot be combined - rule: '!((has(self.token) || has(self.tokenSecretRef)) - && (has(self.signingKeySecretRef) || has(self.claims)))' - - message: static JWT token and generated JWT configuration - cannot be combined - rule: '!((has(self.token) || has(self.tokenSecretRef)) - && (has(self.signingKeySecretRef) || has(self.claims)))' - - message: algorithm must be specified when generating - a JWT - rule: '!has(self.signingKeySecretRef) || self.algorithm - != ""' token: properties: scheme: @@ -240,6 +164,14 @@ spec: description: 'Example: "next"' type: string type: object + x-kubernetes-validations: + - message: static JWT token and generated JWT configuration + cannot be combined + rule: '!((has(self.token) || has(self.tokenSecretRef)) && + ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))' + - message: algorithm must be specified when generating a JWT + rule: '!has(self.signingKeySecretRef) || self.algorithm + != ""' timeout: default: 10s type: string @@ -270,6 +202,7 @@ spec: type: object x-kubernetes-map-type: atomic insecureSkipVerify: + default: false type: boolean type: object url: diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 9143ae4..65888b8 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -1,22 +1,120 @@ package discovery import ( + "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/client" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name -func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { switch { case spec.Provider.HTTP != nil: - cfg.AcceptPush = spec.Provider.HTTP.AcceptPush - return http.New(*cfg, *spec.Provider.HTTP), nil + httpSpec := *spec.Provider.HTTP + cfg.AcceptPush = httpSpec.AcceptPush + + // TODO: watch secrets -> if secret changes reconcile has to be executed + if httpSpec.Authorization != nil { + if err := resolveAuthorizationIntoSpec( + ctx, + c, + cfg.TargetsourceNN.Namespace, + httpSpec.Authorization, + ); err != nil { + return nil, err + } + } + + return http.New(*cfg, httpSpec), nil default: return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) } } + +func resolveAuthorizationIntoSpec( + ctx context.Context, + c client.Client, + namespace string, + authSpec *gnmicv1alpha1.AuthorizationSpec, +) error { + if authSpec == nil { + return nil + } + auth := authSpec + + switch { + case auth.Basic != nil: + b := auth.Basic + + if b.CredentialsSecretRef != nil { + values, err := GetSecretValues( + ctx, + c, + namespace, + b.CredentialsSecretRef.Name, + "username", + "password", + ) + if err != nil { + return err + } + b.Username = values["username"] + b.Password = values["password"] + } + + case auth.Token != nil: + t := auth.Token + if t.TokenSecretRef != nil { + values, err := GetSecretValues( + ctx, + c, + namespace, + t.TokenSecretRef.Name, + "token", + ) + if err != nil { + return err + } + t.Token = values["token"] + } + + // case auth.JWT != nil: + // jwt := auth.JWT + // if jwt.TokenSecretRef != nil { + // values, err := GetSecretValues( + // ctx, + // c, + // namespaceName, + // jwt.TokenSecretRef.Name, + // "token", + // ) + // if err != nil { + // return err + // } + // jwt.Token = values[jwt.TokenSecretRef.Key] + // } + // if jwt.SigningKeySecretRef != nil { + // values, err := GetSecretValues( + // ctx, + // c, + // namespaceName, + // jwt.SigningKeySecretRef.Name, + // "key", + // ) + // if err != nil { + // return err + // } + // jwt.Key = values[jwt.SigningKeySecretRef.Key] + + // } + } + + return nil +} diff --git a/internal/controller/discovery/loaders/http/const.go b/internal/controller/discovery/loaders/http/const.go deleted file mode 100644 index 4b18f6c..0000000 --- a/internal/controller/discovery/loaders/http/const.go +++ /dev/null @@ -1,9 +0,0 @@ -package http - -import "time" - -const ( - DefaultPollInterval = 1 * time.Hour - DefaultTimeout = 30 * time.Second - DefaultAcceptPush = false -) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index bb8a025..356a911 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -70,13 +70,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er // helper function to fetch targets and emit discovery messages fetchAndEmit := func() { - targets, err := l.fetchTargetsFromHTTPEndpoint( - ctx, - client, - l.spec.URL, - l.spec.Authorization.Token.Scheme, - l.spec.Authorization.Token.Token, - ) + targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client) if err != nil { logger.Error( err, @@ -123,18 +117,14 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er func (l *Loader) fetchTargetsFromHTTPEndpoint( ctx context.Context, client *http.Client, - url string, - scheme string, - token string, ) ([]core.DiscoveredTarget, error) { - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.spec.URL, nil) if err != nil { return nil, fmt.Errorf("creating HTTP request failed: %w", err) } req.Header.Set("Accept", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("%s %s", scheme, token)) + l.applyAuthorization(req) resp, err := client.Do(req) if err != nil { @@ -153,3 +143,35 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( return targets, nil } + +func (l *Loader) applyAuthorization(req *http.Request) { + auth := l.spec.Authorization + if auth == nil { + return + } + + switch { + case auth.Basic != nil: + req.SetBasicAuth( + auth.Basic.Username, + auth.Basic.Password, + ) + + case auth.Token != nil: + req.Header.Set( + "Authorization", + fmt.Sprintf("%s %s", + auth.Token.Scheme, + auth.Token.Token, + ), + ) + + // case auth.JWT != nil: + // if auth.JWT.Token != "" { + // req.Header.Set( + // "Authorization", + // fmt.Sprintf("Bearer %s", auth.JWT.Token), + // ) + // } + } +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 522aabd..7cf135e 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -94,7 +94,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscovery(ctx, req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } @@ -162,6 +162,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // - MessageProcessor and Loader must run for the lifetime of the TargetSource // - Any unexpected exit is treated as a bug and triggers full shutdown func (r *TargetSourceReconciler) startDiscovery( + reconcileCtx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, @@ -185,7 +186,7 @@ func (r *TargetSourceReconciler) startDiscovery( targetSource, targetChannel, ) - loader, err := discovery.NewLoader(&loaderConfig, targetSource.Spec) + loader, err := discovery.NewLoader(reconcileCtx, r.Client, &loaderConfig, targetSource.Spec) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() From 966cd59f6e9d73c65a1c0b7b7f6341f6aa4db370 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 08:43:18 +0000 Subject: [PATCH 218/246] support .Key for TokenSecretRef --- internal/controller/discovery/loaders.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 65888b8..98e63d2 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -72,17 +72,21 @@ func resolveAuthorizationIntoSpec( case auth.Token != nil: t := auth.Token if t.TokenSecretRef != nil { + key := "token" + if t.TokenSecretRef.Key != "" { + key = t.TokenSecretRef.Key + } values, err := GetSecretValues( ctx, c, namespace, t.TokenSecretRef.Name, - "token", + key, ) if err != nil { return err } - t.Token = values["token"] + t.Token = values[key] } // case auth.JWT != nil: From 862e28d6f7d0139ddd4ff163d11c3d4e0b828dc0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 09:42:23 +0000 Subject: [PATCH 219/246] update targetsource --- api/v1alpha1/targetsource_types.go | 113 +++++++++++++++--- api/v1alpha1/zz_generated.deepcopy.go | 9 +- .../operator.gnmic.dev_targetsources.yaml | 94 +++++++++++++-- 3 files changed, 185 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index ee3838c..143da3c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -24,74 +24,135 @@ import ( // TargetSourceSpec defines the desired state of TargetSource // +kubebuilder:validation:Required type TargetSourceSpec struct { + // Provider defines the source of targets for this TargetSource + // Only one provider can be specified per TargetSource + // +kubebuilder:validation:Required Provider *ProviderSpec `json:"provider"` + // TODO: implement in message processor + // Optional port to use for discovered targets if not specified by the provider + // +kubebuilder:validation:Optional + TargetPort int32 `json:"targetPort,omitempty"` + + // Optional labels to apply to all targets discovered by this TargetSource // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` + // The TargetProfile to use for targets discovered by this TargetSource + // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 TargetProfile string `json:"targetProfile"` } +// ProviderSpec defines the source of targets for a TargetSource +// Only one provider can be specified per TargetSource // +kubebuilder:validation:ExactlyOneOf=http type ProviderSpec struct { + // HTTP defines the configuration for a HTTP provider HTTP *HTTPConfig `json:"http,omitempty"` } +// HTTPConfig defines the configuration for the HTTP provider // +kubebuilder:validation:AtLeastOneOf=url;acceptPush type HTTPConfig struct { + // URL of the HTTP endpoint to pull targets from + // If defined, the loader will periodically poll this endpoint for targets // +kubebuilder:validation:Optional URL string `json:"url,omitempty"` + + // If true, the loader will accept pushed target updates to the controller endpoint + // The endpoint will be /{namespace}/{targetsource}/ + // +kubebuilder:default=false + // +kubebuilder:validation:Optional + AcceptPush bool `json:"acceptPush,omitempty"` + + // Optional authorization configuration for accessing the HTTP endpoint // +kubebuilder:validation:Optional Authorization *AuthorizationSpec `json:"authorization,omitempty"` + + // Optional interval for polling the HTTP endpoint for targets // TODO: increase default value // +kubebuilder:default="30s" // +kubebuilder:validation:Optional PollInterval *metav1.Duration `json:"interval,omitempty"` + + // Optional timeout for HTTP requests to the endpoint // +kubebuilder:default="10s" // +kubebuilder:validation:Optional Timeout *metav1.Duration `json:"timeout,omitempty"` + + // Optional TLS configuration for connecting to the HTTP endpoint // +kubebuilder:validation:Optional TLS *ClientTLSConfig `json:"tls,omitempty"` + + // Optional pagination configuration for parsing responses from the HTTP endpoint // +kubebuilder:validation:Optional Pagination *PaginationSpec `json:"pagination,omitempty"` + + // Optional mapping configuration for parsing responses from the HTTP endpoint // +kubebuilder:validation:Optional ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"` - // +kubebuilder:default=false - // +kubebuilder:validation:Optional - AcceptPush bool `json:"acceptPush,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive" type ClientTLSConfig struct { + // Skip TLS verification of the Provider's certificate. // +kubebuilder:default:=false - InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` - CASecretRef *corev1.SecretKeySelector `json:"caSecretRef,omitempty"` + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + + // Base64-encoded bundle of PEM CAs which will be used to validate the certificate + // chain presented by the Provider. Only used if using HTTPS to connect to Provider and + // ignored for HTTP connections. + // Mutually exclusive with CABundleSecretRef. + // +optional + CABundle []byte `json:"caBundle,omitempty"` + + // Reference to a Secret containing a bundle of PEM-encoded CAs to use when + // verifying the certificate chain presented by the Provider when using HTTPS. + // Mutually exclusive with CABundle. + CABundleSecretRef *corev1.SecretKeySelector `json:"caBundleSecretRef,omitempty"` } -// +kubebuilder:validation:ExactlyOneOf=basic;jwt;token +// AuthorizationSpec defines the configuration for authentication +// +kubebuilder:validation:ExactlyOneOf=basic;token type AuthorizationSpec struct { + // Basic authentication configuration Basic *BasicAuthSpec `json:"basic,omitempty"` + // Token-based authentication configuration Token *TokenAuthSpec `json:"token,omitempty"` // JWT *JWTAuthSpec `json:"jwt,omitempty"` + // MTLS } +// BasicAuthSpec defines the configuration for basic authentication // Enforce EITHER inline creds OR secret ref // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix" type BasicAuthSpec struct { + // Username for basic auth + // Mutually exclusive with CredentialsSecretRef. Username string `json:"username,omitempty"` + // Password for basic auth + // Mutually exclusive with CredentialsSecretRef. Password string `json:"password,omitempty"` - // CredentialsSecretRef references a secret containing: - // - username - // - password - // NOTE: key field is ignored; fixed keys are used instead + + // Reference to a Secret containing "username" and "password" keys to use for + // basic authentication when connecting to the Provider. + // Mutually exclusive with Username and Password. CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"` } +// TokenAuthSpec defines the configuration for token-based authentication // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both" type TokenAuthSpec struct { + // Scheme for the token, e.g. "Bearer" // +kubebuilder:validation:MinLength=1 - Scheme string `json:"scheme"` - Token string `json:"token,omitempty"` + Scheme string `json:"scheme"` + // Token value for authentication + // Mutually exclusive with TokenSecretRef. + Token string `json:"token,omitempty"` + // Reference to a Secret containing a key with the token value to use for + // authentication when connecting to the Provider. + // Mutually exclusive with Token. TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } @@ -110,19 +171,37 @@ type TokenAuthSpec struct { // TTL *metav1.Duration `json:"ttl,omitempty"` // } +// PaginationSpec defines the configuration for paginating through responses from providers type PaginationSpec struct { + // JSONPath-style expression to extract the list of targets from the response // Example: "results" ItemsField string `json:"itemsField,omitempty"` + + // JSONPath-style expression to extract the next page token or URL from the response for pagination // Example: "next" NextField string `json:"nextField,omitempty"` } -// JSONPath-style expressions +// JSONPath-style expressions to extract target fields from the response +// and map them to the corresponding Target fields. type ResponseMappingSpec struct { - Name string `json:"name"` - Address string `json:"address"` - Port string `json:"port,omitempty"` - Labels map[string]string `json:"labels,omitempty"` + // JSONPath expression to extract the target name from the response + // +kubebuilder:validation:Required + Name string `json:"name"` + + // JSONPath expression to extract the target address from the response + // +kubebuilder:validation:Required + Address string `json:"address"` + + // JSONPath expression to extract the target port from the response + // +kubebuilder:validation:Optional + Port string `json:"port,omitempty"` + + // JSONPath expression to extract the target labels from the response + // The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec, + // with values from the response taking precedence in case of conflicts. + // +kubebuilder:validation:Optional + Labels map[string]string `json:"labels,omitempty"` } // TargetSourceStatus defines the observed state of TargetSource diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3844789..dc4b784 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -94,8 +94,13 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) { *out = *in - if in.CASecretRef != nil { - in, out := &in.CASecretRef, &out.CASecretRef + if in.CABundle != nil { + in, out := &in.CABundle, &out.CABundle + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.CABundleSecretRef != nil { + in, out := &in.CABundleSecretRef, &out.CABundleSecretRef *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index aab3917..a6f1e1d 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -40,23 +40,31 @@ spec: description: TargetSourceSpec defines the desired state of TargetSource properties: provider: + description: |- + Provider defines the source of targets for this TargetSource + Only one provider can be specified per TargetSource properties: http: + description: HTTP defines the configuration for a HTTP provider properties: acceptPush: default: false + description: |- + If true, the loader will accept pushed target updates to the controller endpoint + The endpoint will be /{namespace}/{targetsource}/ type: boolean authorization: + description: Optional authorization configuration for accessing + the HTTP endpoint properties: basic: - description: Enforce EITHER inline creds OR secret ref + description: Basic authentication configuration properties: credentialsSecretRef: description: |- - CredentialsSecretRef references a secret containing: - - username - - password - NOTE: key field is ignored; fixed keys are used instead + Reference to a Secret containing "username" and "password" keys to use for + basic authentication when connecting to the Provider. + Mutually exclusive with Username and Password. properties: key: description: The key of the secret to select from. Must @@ -80,8 +88,14 @@ spec: type: object x-kubernetes-map-type: atomic password: + description: |- + Password for basic auth + Mutually exclusive with CredentialsSecretRef. type: string username: + description: |- + Username for basic auth + Mutually exclusive with CredentialsSecretRef. type: string type: object x-kubernetes-validations: @@ -91,15 +105,22 @@ spec: && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password)) token: + description: Token-based authentication configuration properties: scheme: + description: Scheme for the token, e.g. "Bearer" minLength: 1 type: string token: + description: |- + Token value for authentication + Mutually exclusive with TokenSecretRef. type: string tokenSecretRef: - description: SecretKeySelector selects a key of a - Secret. + description: |- + Reference to a Secret containing a key with the token value to use for + authentication when connecting to the Provider. + Mutually exclusive with Token. properties: key: description: The key of the secret to select from. Must @@ -137,31 +158,50 @@ spec: == 1' interval: default: 30s + description: Optional interval for polling the HTTP endpoint + for targets type: string mapping: - description: JSONPath-style expressions + description: Optional mapping configuration for parsing responses + from the HTTP endpoint properties: address: + description: JSONPath expression to extract the target + address from the response type: string labels: additionalProperties: type: string + description: |- + JSONPath expression to extract the target labels from the response + The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec, + with values from the response taking precedence in case of conflicts. type: object name: + description: JSONPath expression to extract the target + name from the response type: string port: + description: JSONPath expression to extract the target + port from the response type: string required: - address - name type: object pagination: + description: Optional pagination configuration for parsing + responses from the HTTP endpoint properties: itemsField: - description: 'Example: "results"' + description: |- + JSONPath-style expression to extract the list of targets from the response + Example: "results" type: string nextField: - description: 'Example: "next"' + description: |- + JSONPath-style expression to extract the next page token or URL from the response for pagination + Example: "next" type: string type: object x-kubernetes-validations: @@ -174,11 +214,25 @@ spec: != ""' timeout: default: 10s + description: Optional timeout for HTTP requests to the endpoint type: string tls: + description: Optional TLS configuration for connecting to + the HTTP endpoint properties: - caSecretRef: - description: SecretKeySelector selects a key of a Secret. + caBundle: + description: |- + Base64-encoded bundle of PEM CAs which will be used to validate the certificate + chain presented by the Provider. Only used if using HTTPS to connect to Provider and + ignored for HTTP connections. + Mutually exclusive with CABundleSecretRef. + format: byte + type: string + caBundleSecretRef: + description: |- + Reference to a Secret containing a bundle of PEM-encoded CAs to use when + verifying the certificate chain presented by the Provider when using HTTPS. + Mutually exclusive with CABundle. properties: key: description: The key of the secret to select from. Must @@ -203,9 +257,16 @@ spec: x-kubernetes-map-type: atomic insecureSkipVerify: default: false + description: Skip TLS verification of the Provider's certificate. type: boolean type: object + x-kubernetes-validations: + - message: caBundle and caBundleSecretRef are mutually exclusive + rule: '!(has(self.caBundle) && has(self.caBundleSecretRef))' url: + description: |- + URL of the HTTP endpoint to pull targets from + If defined, the loader will periodically poll this endpoint for targets type: string type: object x-kubernetes-validations: @@ -220,8 +281,17 @@ spec: targetLabels: additionalProperties: type: string + description: Optional labels to apply to all targets discovered by + this TargetSource type: object + targetPort: + description: Optional port to use for discovered targets if not specified + by the provider + format: int32 + type: integer targetProfile: + description: The TargetProfile to use for targets discovered by this + TargetSource minLength: 1 type: string required: From e6e9439be84116136e6c2c9e0230056a92071866 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 10:04:38 +0000 Subject: [PATCH 220/246] support TLS verification --- internal/controller/discovery/loaders.go | 44 ++++++++++++++++++- .../discovery/loaders/http/loader.go | 28 +++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 98e63d2..1e5ea46 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -30,12 +30,21 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi return nil, err } } + if httpSpec.TLS != nil { + if err := resolveTLSIntoSpec( + ctx, + c, + cfg.TargetsourceNN.Namespace, + httpSpec.TLS, + ); err != nil { + return nil, err + } + } return http.New(*cfg, httpSpec), nil default: return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN) } - } func resolveAuthorizationIntoSpec( @@ -122,3 +131,36 @@ func resolveAuthorizationIntoSpec( return nil } + +func resolveTLSIntoSpec( + ctx context.Context, + c client.Client, + namespace string, + tlsSpec *gnmicv1alpha1.ClientTLSConfig, +) error { + if tlsSpec == nil { + return nil + } + tls := tlsSpec + + if tls.CABundleSecretRef != nil { + key := "ca.crt" + if tls.CABundleSecretRef.Key != "" { + key = tls.CABundleSecretRef.Key + } + values, err := GetSecretValues( + ctx, + c, + namespace, + tls.CABundleSecretRef.Name, + key, + ) + if err != nil { + return err + } + // convert string to []byte + tls.CABundle = []byte(values[key]) + } + + return nil +} diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 356a911..d956799 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -2,6 +2,8 @@ package http import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "net/http" @@ -55,8 +57,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er // return errors.New("HTTP loader requires spec.provider.http to be set") // } - client := &http.Client{ - Timeout: l.spec.Timeout.Duration, + client, err := l.buildHTTPClient() + if err != nil { + return fmt.Errorf("failed to build HTTP client: %w", err) } interval := l.spec.PollInterval.Duration ticker := time.NewTicker(interval) @@ -114,6 +117,27 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er } } +func (l *Loader) buildHTTPClient() (*http.Client, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: l.spec.TLS != nil && l.spec.TLS.InsecureSkipVerify, + } + + if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 { + certPool := x509.NewCertPool() + if ok := certPool.AppendCertsFromPEM(l.spec.TLS.CABundle); !ok { + return nil, fmt.Errorf("Failed to parse CA bundle for TargetSource %s/%s\n", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name) + } + tlsConfig.RootCAs = certPool + } + + return &http.Client{ + Timeout: l.spec.Timeout.Duration, + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, nil +} + func (l *Loader) fetchTargetsFromHTTPEndpoint( ctx context.Context, client *http.Client, From 055bfb05e96b204456f65d4f47180342bbd03270 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 14:11:14 +0000 Subject: [PATCH 221/246] make manifest and generate --- config/crd/bases/operator.gnmic.dev_targetsources.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index a6f1e1d..d603546 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -152,9 +152,9 @@ spec: rule: has(self.token) != has(self.tokenSecretRef) type: object x-kubernetes-validations: - - message: exactly one of the fields in [basic jwt token] - must be set - rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size() + - message: exactly one of the fields in [basic token] must + be set + rule: '[has(self.basic),has(self.token)].filter(x,x==true).size() == 1' interval: default: 30s From 1deb8ccbaac61ef238579a11a4bfcec89fd1ba3e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 14 May 2026 14:21:08 +0000 Subject: [PATCH 222/246] fix CRD issues --- api/v1alpha1/targetsource_types.go | 4 ++-- config/crd/bases/operator.gnmic.dev_targetsources.yaml | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 143da3c..aa5b8ce 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -156,8 +156,8 @@ type TokenAuthSpec struct { TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"` } -// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" -// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" +// +kubebuilder(disabled):validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined" +// +kubebuilder(disabled):validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT" // type JWTAuthSpec struct { // // Static pre-generated JWT // Token string `json:"token,omitempty"` diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index d603546..6851ad7 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -204,14 +204,6 @@ spec: Example: "next" type: string type: object - x-kubernetes-validations: - - message: static JWT token and generated JWT configuration - cannot be combined - rule: '!((has(self.token) || has(self.tokenSecretRef)) && - ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))' - - message: algorithm must be specified when generating a JWT - rule: '!has(self.signingKeySecretRef) || self.algorithm - != ""' timeout: default: 10s description: Optional timeout for HTTP requests to the endpoint From 816b04f50aa77b5cab0b945ea08c4f9337840dca Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 08:34:40 +0000 Subject: [PATCH 223/246] add support for pagination --- api/v1alpha1/targetsource_types.go | 12 +- .../operator.gnmic.dev_targetsources.yaml | 12 +- .../discovery/loaders/http/loader.go | 134 +++++++++++++++--- 3 files changed, 134 insertions(+), 24 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index aa5b8ce..c4d3382 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -173,12 +173,18 @@ type TokenAuthSpec struct { // PaginationSpec defines the configuration for paginating through responses from providers type PaginationSpec struct { - // JSONPath-style expression to extract the list of targets from the response + // Field name in the JSON response that contains the list of items (targets). + // Must refer to a top-level key in the response object. // Example: "results" ItemsField string `json:"itemsField,omitempty"` - // JSONPath-style expression to extract the next page token or URL from the response for pagination - // Example: "next" + // Field name in the JSON response that contains the next page reference. + // The value can be either: + // - a full URL (used directly for the next request), or + // - a pagination token (appended as a query parameter using this field name as the key). + // + // Must refer to a top-level key in the response object. + // Example: "next" or "nextToken" NextField string `json:"nextField,omitempty"` } diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 6851ad7..3e7f89f 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -195,13 +195,19 @@ spec: properties: itemsField: description: |- - JSONPath-style expression to extract the list of targets from the response + Field name in the JSON response that contains the list of items (targets). + Must refer to a top-level key in the response object. Example: "results" type: string nextField: description: |- - JSONPath-style expression to extract the next page token or URL from the response for pagination - Example: "next" + Field name in the JSON response that contains the next page reference. + The value can be either: + - a full URL (used directly for the next request), or + - a pagination token (appended as a query parameter using this field name as the key). + + Must refer to a top-level key in the response object. + Example: "next" or "nextToken" type: string type: object timeout: diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index d956799..c064a7b 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "time" "github.com/google/uuid" @@ -142,30 +143,56 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( ctx context.Context, client *http.Client, ) ([]core.DiscoveredTarget, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.spec.URL, nil) - if err != nil { - return nil, fmt.Errorf("creating HTTP request failed: %w", err) - } + var allTargets []core.DiscoveredTarget + currentUrl := l.spec.URL - req.Header.Set("Accept", "application/json") - l.applyAuthorization(req) + for { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentUrl, nil) + if err != nil { + return nil, fmt.Errorf("creating HTTP request failed: %w", err) + } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) - } - defer resp.Body.Close() + req.Header.Set("Accept", "application/json") + l.applyAuthorization(req) - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) - } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) + } - var targets []core.DiscoveredTarget - if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil { - return nil, fmt.Errorf("failed to decode HTTP response: %w", err) + // Decode response into raw map for pagination support + var raw map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, fmt.Errorf("failed to decode HTTP response: %w", err) + } + + // Extract targets from response + targets, err := l.extractTargetsFromResponse(raw) + if err != nil { + return nil, err + } + allTargets = append(allTargets, targets...) + + // Check for pagination + nextPageInfo, err := l.extractNextPageInfo(raw) + if err != nil { + return nil, err + } + if nextPageInfo == "" { + break + } + nextURL, err := l.buildNextURL(currentUrl, nextPageInfo) + if err != nil { + return nil, err + } + currentUrl = nextURL } - return targets, nil + return allTargets, nil } func (l *Loader) applyAuthorization(req *http.Request) { @@ -199,3 +226,74 @@ func (l *Loader) applyAuthorization(req *http.Request) { // } } } + +func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) { + var targets []core.DiscoveredTarget + + if l.spec.Pagination == nil || l.spec.Pagination.ItemsField == "" { + // No pagination config, assume entire response is the target list + data, err := json.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("failed to marshal response: %w", err) + } + + if err := json.Unmarshal(data, &targets); err != nil { + return nil, fmt.Errorf("failed to decode targets: %w", err) + } + + return targets, nil + } + + // Extract from field + items, ok := raw[l.spec.Pagination.ItemsField] + if !ok { + return nil, fmt.Errorf("itemsField '%s' not found in response", l.spec.Pagination.ItemsField) + } + + data, err := json.Marshal(items) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(data, &targets); err != nil { + return nil, fmt.Errorf("failed to decode targets from itemsField: %w", err) + } + + return targets, nil +} + +func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) { + if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" { + return "", nil + } + + val, ok := raw[l.spec.Pagination.NextField] + if !ok { + return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField) + } + + next, ok := val.(string) + if !ok { + return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField) + } + + return next, nil +} + +func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) { + // nextVal is a full URL -> return as is + if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" { + return nextVal, nil + } + + // nextVal is a token -> append as query parameter + parsedURL, err := url.Parse(currentURL) + if err != nil { + return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err) + } + q := parsedURL.Query() + q.Set(l.spec.Pagination.NextField, nextVal) + parsedURL.RawQuery = q.Encode() + + return parsedURL.String(), nil +} From c01b199d8bb9605956b4ec17ec4f2e900dbd19f5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 08:37:33 +0000 Subject: [PATCH 224/246] refactor --- .../controller/discovery/loaders/http/auth.go | 38 ++++++++++ .../discovery/loaders/http/loader.go | 69 ------------------- .../discovery/loaders/http/pagination.go | 42 +++++++++++ 3 files changed, 80 insertions(+), 69 deletions(-) create mode 100644 internal/controller/discovery/loaders/http/auth.go create mode 100644 internal/controller/discovery/loaders/http/pagination.go diff --git a/internal/controller/discovery/loaders/http/auth.go b/internal/controller/discovery/loaders/http/auth.go new file mode 100644 index 0000000..0af0556 --- /dev/null +++ b/internal/controller/discovery/loaders/http/auth.go @@ -0,0 +1,38 @@ +package http + +import ( + "fmt" + "net/http" +) + +func (l *Loader) applyAuthorization(req *http.Request) { + auth := l.spec.Authorization + if auth == nil { + return + } + + switch { + case auth.Basic != nil: + req.SetBasicAuth( + auth.Basic.Username, + auth.Basic.Password, + ) + + case auth.Token != nil: + req.Header.Set( + "Authorization", + fmt.Sprintf("%s %s", + auth.Token.Scheme, + auth.Token.Token, + ), + ) + + // case auth.JWT != nil: + // if auth.JWT.Token != "" { + // req.Header.Set( + // "Authorization", + // fmt.Sprintf("Bearer %s", auth.JWT.Token), + // ) + // } + } +} diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index c064a7b..55689df 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "time" "github.com/google/uuid" @@ -195,38 +194,6 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( return allTargets, nil } -func (l *Loader) applyAuthorization(req *http.Request) { - auth := l.spec.Authorization - if auth == nil { - return - } - - switch { - case auth.Basic != nil: - req.SetBasicAuth( - auth.Basic.Username, - auth.Basic.Password, - ) - - case auth.Token != nil: - req.Header.Set( - "Authorization", - fmt.Sprintf("%s %s", - auth.Token.Scheme, - auth.Token.Token, - ), - ) - - // case auth.JWT != nil: - // if auth.JWT.Token != "" { - // req.Header.Set( - // "Authorization", - // fmt.Sprintf("Bearer %s", auth.JWT.Token), - // ) - // } - } -} - func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) { var targets []core.DiscoveredTarget @@ -261,39 +228,3 @@ func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core. return targets, nil } - -func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) { - if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" { - return "", nil - } - - val, ok := raw[l.spec.Pagination.NextField] - if !ok { - return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField) - } - - next, ok := val.(string) - if !ok { - return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField) - } - - return next, nil -} - -func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) { - // nextVal is a full URL -> return as is - if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" { - return nextVal, nil - } - - // nextVal is a token -> append as query parameter - parsedURL, err := url.Parse(currentURL) - if err != nil { - return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err) - } - q := parsedURL.Query() - q.Set(l.spec.Pagination.NextField, nextVal) - parsedURL.RawQuery = q.Encode() - - return parsedURL.String(), nil -} diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go new file mode 100644 index 0000000..6a4a3ec --- /dev/null +++ b/internal/controller/discovery/loaders/http/pagination.go @@ -0,0 +1,42 @@ +package http + +import ( + "fmt" + "net/url" +) + +func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) { + if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" { + return "", nil + } + + val, ok := raw[l.spec.Pagination.NextField] + if !ok { + return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField) + } + + next, ok := val.(string) + if !ok { + return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField) + } + + return next, nil +} + +func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) { + // nextVal is a full URL -> return as is + if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" { + return nextVal, nil + } + + // nextVal is a token -> append as query parameter + parsedURL, err := url.Parse(currentURL) + if err != nil { + return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err) + } + q := parsedURL.Query() + q.Set(l.spec.Pagination.NextField, nextVal) + parsedURL.RawQuery = q.Encode() + + return parsedURL.String(), nil +} From 5d95c9028fb63d0a600f37a4e5ecc3f573092b3d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 12:22:36 +0000 Subject: [PATCH 225/246] add support for JSONPath mapping --- api/v1alpha1/targetsource_types.go | 4 +- .../operator.gnmic.dev_targetsources.yaml | 6 +- go.mod | 2 + go.sum | 5 ++ internal/controller/discovery/core/types.go | 7 +- .../discovery/loaders/http/loader.go | 50 ++++++----- .../discovery/loaders/http/mapper.go | 78 +++++++++++++++++ .../discovery/loaders/http/mapper_direct.go | 70 +++++++++++++++ .../discovery/loaders/http/mapper_jsonpath.go | 87 +++++++++++++++++++ .../controller/discovery/message_processor.go | 3 +- 10 files changed, 283 insertions(+), 29 deletions(-) create mode 100644 internal/controller/discovery/loaders/http/mapper.go create mode 100644 internal/controller/discovery/loaders/http/mapper_direct.go create mode 100644 internal/controller/discovery/loaders/http/mapper_jsonpath.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index c4d3382..fae55cf 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -195,9 +195,9 @@ type ResponseMappingSpec struct { // +kubebuilder:validation:Required Name string `json:"name"` - // JSONPath expression to extract the target address from the response + // JSONPath expression to extract the target IP from the response // +kubebuilder:validation:Required - Address string `json:"address"` + IP string `json:"ip"` // JSONPath expression to extract the target port from the response // +kubebuilder:validation:Optional diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 3e7f89f..adc55ee 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -165,9 +165,9 @@ spec: description: Optional mapping configuration for parsing responses from the HTTP endpoint properties: - address: + ip: description: JSONPath expression to extract the target - address from the response + IP from the response type: string labels: additionalProperties: @@ -186,7 +186,7 @@ spec: port from the response type: string required: - - address + - ip - name type: object pagination: diff --git a/go.mod b/go.mod index 827da2a..782f99d 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,8 @@ require ( require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/PaesslerAG/gval v1.0.0 // indirect + github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go.sum b/go.sum index 8a613b4..f38b648 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,11 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8= +github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= +github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs= diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 99605b9..51a3477 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -37,9 +37,10 @@ const ( // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { - Name string - Address string - Labels map[string]string + Name string + IP string + Port int32 + Labels map[string]string } type DiscoveryEvent struct { diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 55689df..27eb4c5 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -195,35 +195,45 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( } func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) { - var targets []core.DiscoveredTarget + var items []interface{} - if l.spec.Pagination == nil || l.spec.Pagination.ItemsField == "" { - // No pagination config, assume entire response is the target list - data, err := json.Marshal(raw) - if err != nil { - return nil, fmt.Errorf("failed to marshal response: %w", err) + if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" { + // Extract items array from response using itemsField + val, ok := raw[l.spec.Pagination.ItemsField] + if !ok { + return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField) } - if err := json.Unmarshal(data, &targets); err != nil { - return nil, fmt.Errorf("failed to decode targets: %w", err) + arr, ok := val.([]interface{}) + if !ok { + return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField) } - return targets, nil + items = arr + } else { + // fallback: whole response is array + data, _ := json.Marshal(raw) + var out []interface{} + if err := json.Unmarshal(data, &out); err != nil { + return nil, fmt.Errorf("failed to interpret response as list") + } + items = out } - // Extract from field - items, ok := raw[l.spec.Pagination.ItemsField] - if !ok { - return nil, fmt.Errorf("itemsField '%s' not found in response", l.spec.Pagination.ItemsField) - } + // Map items to targets + var targets []core.DiscoveredTarget + for _, item := range items { + obj, ok := item.(map[string]interface{}) + if !ok { + continue + } - data, err := json.Marshal(items) - if err != nil { - return nil, err - } + target, err := l.mapItem(obj) + if err != nil { + return nil, err + } - if err := json.Unmarshal(data, &targets); err != nil { - return nil, fmt.Errorf("failed to decode targets from itemsField: %w", err) + targets = append(targets, target) } return targets, nil diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go new file mode 100644 index 0000000..bb36113 --- /dev/null +++ b/internal/controller/discovery/loaders/http/mapper.go @@ -0,0 +1,78 @@ +package http + +import ( + "strconv" + + "github.com/gnmic/operator/internal/controller/discovery/core" +) + +// valueGetter defines the contract for extracting values from a response item +type valueGetter interface { + GetName() (string, error) + GetIP() (string, error) + GetPort() int32 + GetLabels() map[string]string +} + +// getGetter selects the extraction strategy based on the spec +// If no ResponseMapping is defined -> use direct mapping +func (l *Loader) getGetter(item map[string]interface{}) valueGetter { + if l.spec.ResponseMapping == nil { + return &directGetter{ + item: item, + } + } + + return &jsonPathGetter{ + item: item, + spec: l.spec.ResponseMapping, + } +} + +// mapItem is the mapping entrypoint used by the loader +// It uses the selected valueGetter and produces a DiscoveredTarget +func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, error) { + getter := l.getGetter(item) + + name, err := getter.GetName() + if err != nil { + return core.DiscoveredTarget{}, err + } + + ip, err := getter.GetIP() + if err != nil { + return core.DiscoveredTarget{}, err + } + + port := getter.GetPort() + labels := getter.GetLabels() + + return core.DiscoveredTarget{ + Name: name, + IP: ip, + Port: port, + Labels: labels, + }, nil +} + +// extractPort attempts to normalize different JSON types into int32 +// +// Supports: +// - float64 (default JSON number type) +// - string ("1234") +// +// Returns 0 if conversion fails (treated as "no port specified"). +func extractPort(val interface{}) int32 { + switch v := val.(type) { + case float64: + return int32(v) + case string: + p, err := strconv.Atoi(v) + if err != nil { + return 0 + } + return int32(p) + default: + return 0 + } +} diff --git a/internal/controller/discovery/loaders/http/mapper_direct.go b/internal/controller/discovery/loaders/http/mapper_direct.go new file mode 100644 index 0000000..1d135f8 --- /dev/null +++ b/internal/controller/discovery/loaders/http/mapper_direct.go @@ -0,0 +1,70 @@ +package http + +import ( + "fmt" +) + +// directGetter extracts values via direct map access +// Example input: +// +// { +// "name": "router1", +// "ip": "10.0.0.1", +// "port": 57400, +// "labels": { ... } +// } +type directGetter struct { + item map[string]interface{} +} + +// GetName extracts the "name" field directly +func (g *directGetter) GetName() (string, error) { + val, ok := g.item["name"].(string) + if !ok || val == "" { + return "", fmt.Errorf("name must be a non-empty string") + } + return val, nil +} + +// GetIP extracts the "ip" field directly. +func (g *directGetter) GetIP() (string, error) { + val, ok := g.item["ip"].(string) + if !ok || val == "" { + return "", fmt.Errorf("ip must be a non-empty string") + } + return val, nil +} + +// GetPort extracts and normalizes the "port" field +// +// Behavior: +// - supports int, float64, string +// - returns 0 if value is missing or invalid +func (g *directGetter) GetPort() int32 { + if val, ok := g.item["port"]; ok { + return extractPort(val) + } + return 0 +} + +// GetLabels extracts labels from the "labels" field +// Expected format: +// +// "labels": { +// "key": "value" +// } +// +// Non-string values are converted to string +func (g *directGetter) GetLabels() map[string]string { + labels := make(map[string]string) + + if val, ok := g.item["labels"]; ok { + if m, ok := val.(map[string]interface{}); ok { + for k, v := range m { + labels[k] = fmt.Sprintf("%v", v) + } + } + } + + return labels +} diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go new file mode 100644 index 0000000..194c79b --- /dev/null +++ b/internal/controller/discovery/loaders/http/mapper_jsonpath.go @@ -0,0 +1,87 @@ +package http + +import ( + "fmt" + + "github.com/PaesslerAG/jsonpath" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +// jsonPathGetter extracts values using JSONPath expressions defined in the CR +// Example mapping: +// +// name: "$.hostname" +// ip: "$.ip" +// port: "$.port" +// labels: +// rack: "$.meta.rack" +type jsonPathGetter struct { + item map[string]interface{} + spec *gnmicv1alpha1.ResponseMappingSpec +} + +// helper function to execute JSONPath queries +func (g *jsonPathGetter) get(expr string) (interface{}, error) { + return jsonpath.Get(expr, g.item) +} + +// GetName extracts the target name using JSONPath +func (g *jsonPathGetter) GetName() (string, error) { + val, err := g.get(g.spec.Name) + if err != nil { + return "", fmt.Errorf("name mapping failed: %w", err) + } + + str, ok := val.(string) + if !ok || str == "" { + return "", fmt.Errorf("name must be a non-empty string") + } + + return str, nil +} + +// GetIP extracts the IP using JSONPath +func (g *jsonPathGetter) GetIP() (string, error) { + val, err := g.get(g.spec.IP) + if err != nil { + return "", fmt.Errorf("IP mapping failed: %w", err) + } + + str, ok := val.(string) + if !ok || str == "" { + return "", fmt.Errorf("IP must be a non-empty string") + } + + return str, nil +} + +// GetPort extracts the port using JSONPath +// +// Behavior: +// - returns 0 if no port mapping defined +// - returns 0 if extraction fails or value invalid +func (g *jsonPathGetter) GetPort() int32 { + if g.spec.Port == "" { + return 0 + } + + val, err := g.get(g.spec.Port) + if err != nil { + return 0 + } + + return extractPort(val) +} + +// GetLabels extracts labels using JSONPath expressions defined per label key +func (g *jsonPathGetter) GetLabels() map[string]string { + labels := make(map[string]string) + + for key, expr := range g.spec.Labels { + if val, err := jsonpath.Get(expr, g.item); err == nil { + labels[key] = fmt.Sprintf("%v", val) + } + } + + return labels +} diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index f7aafb1..cb1e068 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -283,7 +283,8 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Info( "Applying Target", "target", event.Target.Name, - "address", event.Target.Address, + "port", event.Target.Port, + "ip", event.Target.IP, "labels", event.Target.Labels, "targetsource", m.targetSource.Name, ) From 6a83f49beee0905a996dae1164b9553f044c80e7 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 12:35:33 +0000 Subject: [PATCH 226/246] add support for TargetProfile supplied by provider --- api/v1alpha1/targetsource_types.go | 4 ++++ .../operator.gnmic.dev_targetsources.yaml | 4 ++++ internal/controller/discovery/core/types.go | 9 ++++---- .../discovery/loaders/http/mapper.go | 11 ++++++---- .../discovery/loaders/http/mapper_direct.go | 14 ++++++++++++- .../discovery/loaders/http/mapper_jsonpath.go | 21 ++++++++++++++++++- 6 files changed, 53 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index fae55cf..c83e5e2 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -208,6 +208,10 @@ type ResponseMappingSpec struct { // with values from the response taking precedence in case of conflicts. // +kubebuilder:validation:Optional Labels map[string]string `json:"labels,omitempty"` + + // JSONPath expression to extract the target profile from the response + // +kubebuilder:validation:Optional + TargetProfile string `json:"targetProfile,omitempty"` } // TargetSourceStatus defines the observed state of TargetSource diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index adc55ee..b4ecb44 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -185,6 +185,10 @@ spec: description: JSONPath expression to extract the target port from the response type: string + targetProfile: + description: JSONPath expression to extract the target + profile from the response + type: string required: - ip - name diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 51a3477..1b40897 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -37,10 +37,11 @@ const ( // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { - Name string - IP string - Port int32 - Labels map[string]string + Name string + IP string + Port int32 + Labels map[string]string + TargetProfile string } type DiscoveryEvent struct { diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go index bb36113..95aa557 100644 --- a/internal/controller/discovery/loaders/http/mapper.go +++ b/internal/controller/discovery/loaders/http/mapper.go @@ -12,6 +12,7 @@ type valueGetter interface { GetIP() (string, error) GetPort() int32 GetLabels() map[string]string + GetTargetProfile() string } // getGetter selects the extraction strategy based on the spec @@ -46,12 +47,14 @@ func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, er port := getter.GetPort() labels := getter.GetLabels() + targetProfile := getter.GetTargetProfile() return core.DiscoveredTarget{ - Name: name, - IP: ip, - Port: port, - Labels: labels, + Name: name, + IP: ip, + Port: port, + Labels: labels, + TargetProfile: targetProfile, }, nil } diff --git a/internal/controller/discovery/loaders/http/mapper_direct.go b/internal/controller/discovery/loaders/http/mapper_direct.go index 1d135f8..185e1cb 100644 --- a/internal/controller/discovery/loaders/http/mapper_direct.go +++ b/internal/controller/discovery/loaders/http/mapper_direct.go @@ -11,7 +11,8 @@ import ( // "name": "router1", // "ip": "10.0.0.1", // "port": 57400, -// "labels": { ... } +// "labels": { ... }, +// "targetProfile": "profile1" // } type directGetter struct { item map[string]interface{} @@ -68,3 +69,14 @@ func (g *directGetter) GetLabels() map[string]string { return labels } + +// GetTargetProfile extracts the "targetProfile" field directly +// +// Behavior: +// - returns "" if value is missing or invalid +func (g *directGetter) GetTargetProfile() string { + if val, ok := g.item["targetProfile"].(string); ok { + return val + } + return "" +} diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go index 194c79b..85bf00a 100644 --- a/internal/controller/discovery/loaders/http/mapper_jsonpath.go +++ b/internal/controller/discovery/loaders/http/mapper_jsonpath.go @@ -78,10 +78,29 @@ func (g *jsonPathGetter) GetLabels() map[string]string { labels := make(map[string]string) for key, expr := range g.spec.Labels { - if val, err := jsonpath.Get(expr, g.item); err == nil { + if val, err := g.get(expr); err == nil { labels[key] = fmt.Sprintf("%v", val) } } return labels } + +// GetTargetProfile extracts the target profile using JSONPath +// +// Behavior: +// - returns "" if no target profile mapping defined +// - returns "" if extraction fails or value invalid +func (g *jsonPathGetter) GetTargetProfile() string { + val, err := g.get(g.spec.TargetProfile) + if err != nil { + return "" + } + + str, ok := val.(string) + if !ok { + return "" + } + + return str +} From 1e9feb6f92fe6feb2266562aee93d39e3793fe35 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 12:44:19 +0000 Subject: [PATCH 227/246] refactor --- internal/controller/discovery/loaders.go | 4 ++++ .../discovery/loaders/http/loader.go | 24 +++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 1e5ea46..42ab588 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -47,6 +47,8 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi } } +// resolveAuthorizationIntoSpec fetches credentials from Kubernetes Secrets +// and populates the AuthorizationSpec accordingly func resolveAuthorizationIntoSpec( ctx context.Context, c client.Client, @@ -132,6 +134,8 @@ func resolveAuthorizationIntoSpec( return nil } +// resolveTLSIntoSpec fetches TLS credentials from Kubernetes Secrets +// and populates the ClientTLSConfig accordingly func resolveTLSIntoSpec( ctx context.Context, c client.Client, diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 27eb4c5..5742092 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -19,20 +19,26 @@ import ( ) // Loader implements the HTTP pull discovery mechanism +// It periodically polls an HTTP endpoint, extracts targets from the response, +// and emits discovery snapshots downstream type Loader struct { loaderCfg core.CommonLoaderConfig spec gnmicv1alpha1.HTTPConfig } -// New instantiates the http loader with the provided config +// New creates a new HTTP loader instance with the provided configuration. +// The loader is stateless apart from its config and spec func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader { return &Loader{loaderCfg: cfg, spec: httpConfig} } +// Name returns the loader's name, used for logging and metrics func (l *Loader) Name() string { return "http" } +// Run starts the HTTP discovery loop +// It performs an immediate fetch and then continues polling at a fixed interval func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error { if l.spec.URL == "" { return nil @@ -52,11 +58,6 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger.Info("HTTP loader started") - // Input Validation of spec - // if l.spec.URL == "nil" { - // return errors.New("HTTP loader requires spec.provider.http to be set") - // } - client, err := l.buildHTTPClient() if err != nil { return fmt.Errorf("failed to build HTTP client: %w", err) @@ -73,6 +74,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er // helper function to fetch targets and emit discovery messages fetchAndEmit := func() { + // Fetch targets from HTTP endpoint targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client) if err != nil { logger.Error( @@ -83,6 +85,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er return } + // Emit discovery snapshot downstream snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString()) if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.loaderCfg.ChunkSize); err != nil { logger.Error( @@ -117,11 +120,13 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er } } +// buildHTTPClient constructs an HTTP client with optional configuration func (l *Loader) buildHTTPClient() (*http.Client, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: l.spec.TLS != nil && l.spec.TLS.InsecureSkipVerify, } + // If a CA bundle is provided, add it to the TLS config if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 { certPool := x509.NewCertPool() if ok := certPool.AppendCertsFromPEM(l.spec.TLS.CABundle); !ok { @@ -130,6 +135,7 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) { tlsConfig.RootCAs = certPool } + // Build the HTTP client with the specified timeout and TLS config return &http.Client{ Timeout: l.spec.Timeout.Duration, Transport: &http.Transport{ @@ -138,6 +144,7 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) { }, nil } +// fetchTargetsFromHTTPEndpoint retrieves targets from the configured HTTP endpoint func (l *Loader) fetchTargetsFromHTTPEndpoint( ctx context.Context, client *http.Client, @@ -146,14 +153,15 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( currentUrl := l.spec.URL for { + // Create HTTP request with context req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentUrl, nil) if err != nil { return nil, fmt.Errorf("creating HTTP request failed: %w", err) } - req.Header.Set("Accept", "application/json") l.applyAuthorization(req) + // Execute HTTP request resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) @@ -194,6 +202,8 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( return allTargets, nil } +// extractTargetsFromResponse extracts items from the response +// and maps each item into a DiscoveredTarget func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) { var items []interface{} From 0e8ea1a9cea0bb801598ea87fe6d2378ddf0f947 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 13:13:21 +0000 Subject: [PATCH 228/246] fix interfaces --- .../discovery/loaders/http/loader.go | 49 ++++++++++--------- .../discovery/loaders/http/pagination.go | 13 ++++- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 5742092..eef202b 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -172,7 +172,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( } // Decode response into raw map for pagination support - var raw map[string]interface{} + var raw interface{} if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { return nil, fmt.Errorf("failed to decode HTTP response: %w", err) } @@ -204,30 +204,33 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint( // extractTargetsFromResponse extracts items from the response // and maps each item into a DiscoveredTarget -func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) { +func (l *Loader) extractTargetsFromResponse(raw interface{}) ([]core.DiscoveredTarget, error) { var items []interface{} - if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" { - // Extract items array from response using itemsField - val, ok := raw[l.spec.Pagination.ItemsField] - if !ok { - return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField) - } - - arr, ok := val.([]interface{}) - if !ok { - return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField) - } - - items = arr - } else { - // fallback: whole response is array - data, _ := json.Marshal(raw) - var out []interface{} - if err := json.Unmarshal(data, &out); err != nil { - return nil, fmt.Errorf("failed to interpret response as list") - } - items = out + switch v := raw.(type) { + // Top-level array response + case []interface{}: + items = v + // Object with itemsField containing the array + case map[string]interface{}: + if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" { + // Extract items array from response using itemsField + val, ok := v[l.spec.Pagination.ItemsField] + if !ok { + return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField) + } + + arr, ok := val.([]interface{}) + if !ok { + return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField) + } + + items = arr + } else { + return nil, fmt.Errorf("response is an object but no itemsField specified for TargetSource %s/%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name) + } + default: + return nil, fmt.Errorf("unexpected response format") } // Map items to targets diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go index 6a4a3ec..9fef778 100644 --- a/internal/controller/discovery/loaders/http/pagination.go +++ b/internal/controller/discovery/loaders/http/pagination.go @@ -5,12 +5,20 @@ import ( "net/url" ) -func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) { +// extractNextPageInfo extracts pagination information from a response +func (l *Loader) extractNextPageInfo(raw interface{}) (string, error) { if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" { return "", nil } - val, ok := raw[l.spec.Pagination.NextField] + // Only objects can have "next" fields + obj, ok := raw.(map[string]interface{}) + if !ok { + // array case -> no pagination + return "", nil + } + + val, ok := obj[l.spec.Pagination.NextField] if !ok { return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField) } @@ -23,6 +31,7 @@ func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) return next, nil } +// buildNextURL constructs the URL for the next page based on the current URL and pagination info func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) { // nextVal is a full URL -> return as is if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" { From 262760c43a78cd62a6bd01681b1b575e11756531 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 15 May 2026 13:57:56 +0000 Subject: [PATCH 229/246] fix incorrect conversion between integer types --- internal/controller/discovery/loaders/http/mapper.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go index 95aa557..de618fa 100644 --- a/internal/controller/discovery/loaders/http/mapper.go +++ b/internal/controller/discovery/loaders/http/mapper.go @@ -1,6 +1,7 @@ package http import ( + "math" "strconv" "github.com/gnmic/operator/internal/controller/discovery/core" @@ -68,9 +69,12 @@ func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, er func extractPort(val interface{}) int32 { switch v := val.(type) { case float64: + if v < 0 || v > math.MaxInt32 { + return 0 + } return int32(v) case string: - p, err := strconv.Atoi(v) + p, err := strconv.ParseInt(v, 10, 32) if err != nil { return 0 } From 38c4ec694ce24a49f37fb5b725875032f87bfc53 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 17 May 2026 08:19:19 +0000 Subject: [PATCH 230/246] auth with bearer token --- cmd/main.go | 8 ++++- config/manager/manager.yaml | 5 +++ internal/apiserver/apiserver.go | 54 +++++++++++++++++++++--------- internal/apiserver/gen.go | 29 ++++++++++------ internal/apiserver/helpers.go | 6 ++-- internal/apiserver/helpers_test.go | 24 ++++++------- internal/apiserver/openapi.yaml | 11 ++++-- internal/apiserver/temp.md | 30 ++++++++--------- 8 files changed, 108 insertions(+), 59 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eadc12d..9701cb3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -130,7 +130,13 @@ func main() { var api *apiserver.APIServer if apiAddr != "" { - api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize) + apiBearerToken := os.Getenv("API_BEARER_TOKEN") + if apiBearerToken == "" { + setupLog.Error(errors.New("missing API_BEARER_TOKEN"), "unable to initialize API server") + os.Exit(1) + } + + api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize, apiBearerToken) if err != nil { setupLog.Error(err, "unable to initialize API server") os.Exit(1) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 900f373..b16cd6d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -82,6 +82,11 @@ spec: valueFrom: fieldRef: fieldPath: metadata.labels['app.kubernetes.io/name'] + - name: API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: gnmic-api-auth + key: bearer-token securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 2bf7b30..1af6129 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,22 +6,12 @@ package apiserver // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 -// kubectl get svc on different branch -// NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -// gnmic-c1 ClusterIP None 7890/TCP 30h -// gnmic-c1-prom-cpipe1-prom-output1 ClusterIP 10.96.202.243 9916/TCP 30h -// gnmic-c1-prom-p1-prom-output1 ClusterIP 10.96.191.84 10344/TCP 30h -// kubernetes ClusterIP 10.96.0.1 443/TCP 27d -// in this branch, the prometheus output is gone -> bug to -// kubectl get svc -// NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -// gnmic-c1 ClusterIP None 7890/TCP 5m14s -// kubernetes ClusterIP 10.96.0.1 443/TCP 29d - import ( "context" + "crypto/subtle" "fmt" "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" @@ -41,8 +31,9 @@ type APIServer struct { types.NamespacedName, core.DiscoveryRegistryValue, ] - chunzSize int - logger logr.Logger + chunzSize int + logger logr.Logger + bearerToken string } type urlStruct struct { @@ -58,10 +49,15 @@ func New( core.DiscoveryRegistryValue, ], discoveryChunksize int, + bearerToken string, ) (*APIServer, error) { - gin.SetMode(gin.ReleaseMode) // To double-check + gin.SetMode(gin.ReleaseMode) // To double-check router := gin.Default() logger := log.Log.WithValues("component", "api-server") + if bearerToken == "" { + return nil, fmt.Errorf("api bearer token cannot be empty") + } + a := &APIServer{ Server: &http.Server{ Addr: addr, @@ -72,6 +68,7 @@ func New( DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, logger: logger, + bearerToken: bearerToken, } logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() @@ -116,6 +113,11 @@ func (a *APIServer) CreateTargets(c *gin.Context) { ) logger.Info("Received POST request for CreateTargets") + if !a.checkBearerToken(c) { + logger.Info("Unauthorized request for CreateTargets") + return + } + var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { logger.Error(err, "Failed to bind request payload") @@ -140,3 +142,25 @@ func (a *APIServer) CreateTargets(c *gin.Context) { utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) c.JSON(http.StatusOK, payloadTargets) } + +func (a *APIServer) checkBearerToken(ctx *gin.Context) bool { + const bearerPrefix = "Bearer " + authHeader := strings.TrimSpace(ctx.GetHeader("Authorization")) + if !strings.HasPrefix(authHeader, bearerPrefix) { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing or invalid authorization header"}) + return false + } + + token := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) + if token == "" { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"}) + return false + } + + if subtle.ConstantTimeCompare([]byte(token), []byte(a.bearerToken)) != 1 { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"}) + return false + } + + return true +} diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 80f2c5f..e132343 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -16,6 +16,10 @@ import ( "github.com/gin-gonic/gin" ) +const ( + BearerAuthScopes = "bearerAuth.Scopes" +) + // Defines values for TargetOperation. const ( Created TargetOperation = "created" @@ -43,8 +47,8 @@ type Label struct { Value *string `json:"value,omitempty"` } -// target defines model for target. -type target struct { +// Target defines model for Target. +type Target struct { Address string `json:"address"` Labels *[]Label `json:"labels,omitempty"` Name string `json:"name"` @@ -56,7 +60,7 @@ type target struct { type TargetOperation string // Targets defines model for Targets. -type Targets = []target +type Targets = []Target // CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. type CreateTargetsJSONRequestBody = Targets @@ -83,6 +87,8 @@ type MiddlewareFunc func(c *gin.Context) // CreateTargets operation middleware func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { + c.Set(BearerAuthScopes, []string{}) + for _, middleware := range siw.HandlerMiddlewares { middleware(c) if c.IsAborted() { @@ -140,14 +146,15 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/7RTO2/cMAz+KwLb0Tg76eatDYogQB9Bm63IoEi8O6WypFJUACPwfy9EO/foXYAsnURL", - "JL8H6WcwcUgxYOAM/TNks8VBS/hFP6CvQaKYkNihXP/GsR48JoQeMpMLG5gaeNK+4JmXqXm5iQ+PaLjm", - "3mnaIJ/21tYS5ny2v6905MkxDhK8J1xDD+/avYZ2EdDO7PfgmkiP9TvoAc8CVCKaXQz1FUMZoP8FhlAz", - "WmigJLtEFj3W6L45bZIorp1/xQfCP8UR2tpYaDQ7xYfw96869nb5i8Mn+isNF9ZRCDquTGHz7euNUd8F", - "P5L68fnnnfp4ewMNPCFlMQS6Vbe6WFwKOjno4cPqctVBA0nzVvi0s1kHVFPMMuWdthsLPVwdpc22YOZP", - "0cpmmRgYg9TplLwzUtk+5nkys8K36c+z4L3vTAXlIqcY8rx0l93F/4G1mA25NK/UywTVslEqF2Mw53Xx", - "fpTtyGUYNI07gxQvFS4o3qI6HpOUtMlrYbf8Tcc+XyNf+ZIZ6bamncju6nFM8iBfEXKhgPYfctfIysxp", - "SuCnaZr+BgAA//+GH+FbRwQAAA==", + "H4sIAAAAAAAC/7RTTW8aMRD9K9a0xxVL0p72RqMqQupH1OSGOBjvAE68tjseI6Fo/3tle7Owgki59MRg", + "z7x57/ntKyjXeWfRcoDmFYLaYydz+UNu0KTCk/NIrDEfv+Ax/fDRIzQQmLTdQV/BQZqIV2766u3EbZ5R", + "cep9krRDvsSWbUsYwlV8k+jkK83Y5eIz4RYa+FSfNNSDgLqwPy2XRPKY/lvZ4dUFiYhk7Wy6RRs7aFag", + "CCVjCxVE3w5ViwZTta4uQTy5rTbv+ED4N2rCNgFnGtWo+Hz9+l3HPi5/cPhCf19BQBVJ8/ExtRbfNygJ", + "aRF5P2YgzZRjGDH2zB76hKHt1mWNmpNY2P36uVTid5bgSPz5/vgkFg9LqOCAFLKnMJ/NZzeD0VZ6DQ18", + "md3O5lCBl7zPROri95la70IOymjPsoUG7iZtxVkM/M21OZzKWUab56T3Rqs8WT+H8rjFpI9ZGIrg09Mx", + "RcwHwTsbin+385v/s7bFoEj7ksq3EIghlCJEpTCEbTQmR/troTEdWuQewe4FrdBBdDoEbXfCkdD2II1u", + "J6mAZjXNw2rdrysIseskHUfrBQ9ctBW8RzENQEasvZFZ9/CpT1/wHvnOxMBID6ntwtD5pZKzfkHIkSwO", + "3Edy98hClTaR1/d93/8LAAD//+4Fc3XkBAAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 48614d0..6b116f4 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -12,7 +12,7 @@ import ( ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets []target) ([]core.DiscoveryEvent, error) { +func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { targets := []core.DiscoveryEvent{} if len(payloadTargets) > 0 { @@ -67,7 +67,7 @@ func getKey(u urlStruct) types.NamespacedName { } // convertTargetLabelsToMap converts target.Labels to map. -func convertTargetLabelsToMap(target target) map[string]string { +func convertTargetLabelsToMap(target Target) map[string]string { labelToMap := make(map[string]string) if target.Labels != nil { for _, tag := range *target.Labels { @@ -81,7 +81,7 @@ func convertTargetLabelsToMap(target target) map[string]string { } // getEvent converts target.Operation to core.Operation. -func getEvent(target target, index int) (core.EventAction, error) { +func getEvent(target Target, index int) (core.EventAction, error) { event := core.EventApply switch target.Operation { case Created: diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go index 58ca283..af6e33d 100644 --- a/internal/apiserver/helpers_test.go +++ b/internal/apiserver/helpers_test.go @@ -13,7 +13,7 @@ import ( ) func TestGetEventApply(t *testing.T) { - target := target{ + target := Target{ Address: "1.1.1.1", Name: "routername", Labels: &[]Label{}, @@ -29,7 +29,7 @@ func TestGetEventApply(t *testing.T) { } func TestGetEventDelete(t *testing.T) { - target := target{ + target := Target{ Address: "1.1.1.1", Name: "routername", Labels: &[]Label{}, @@ -45,7 +45,7 @@ func TestGetEventDelete(t *testing.T) { } func TestGetEventEmptyOperation(t *testing.T) { - target := target{ + target := Target{ Address: "1.1.1.1", Name: "routername", Labels: &[]Label{}, @@ -58,7 +58,7 @@ func TestGetEventEmptyOperation(t *testing.T) { } func TestGetEventUpdate(t *testing.T) { - target := target{ + target := Target{ Address: "1.1.1.1", Name: "routername", Labels: &[]Label{}, @@ -89,7 +89,7 @@ func TestGetKey(t *testing.T) { } func TestConvertTargetLabelsToMapEmpty(t *testing.T) { - target := target{} + target := Target{} result := convertTargetLabelsToMap(target) if len(result) != 0 { t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) @@ -103,7 +103,7 @@ func TestConvertTargetLabelsToMap(t *testing.T) { Key: &key, Value: &value, } - target := target{ + target := Target{ Labels: &[]Label{label}, } expected := map[string]string{ @@ -122,7 +122,7 @@ func TestConvertTargetLabelsToMapEmptyKey(t *testing.T) { Key: &key, Value: &value, } - target := target{ + target := Target{ Labels: &[]Label{label}, } result := convertTargetLabelsToMap(target) @@ -144,7 +144,7 @@ func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { Key: &key2, Value: &value2, } - target := target{ + target := Target{ Labels: &[]Label{label, label2}, } expected := map[string]string{ @@ -158,7 +158,7 @@ func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { } func TestCreateDiscoveryEvent(t *testing.T) { - targets := []target{{ + targets := []Target{{ Address: "1.1.1.1", Name: "routername", Labels: &[]Label{}, @@ -181,7 +181,7 @@ func TestCreateDiscoveryEvent(t *testing.T) { } func TestCreateDiscoveryEventEmptyName(t *testing.T) { - targets := []target{{ + targets := []Target{{ Address: "1.1.1.1", Name: "", Labels: &[]Label{}, @@ -194,7 +194,7 @@ func TestCreateDiscoveryEventEmptyName(t *testing.T) { } func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { - targets := []target{{ + targets := []Target{{ Address: "", Name: "routername", Labels: &[]Label{}, @@ -207,7 +207,7 @@ func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { } func TestCreateDiscoveryEventWrongEvent(t *testing.T) { - targets := []target{{ + targets := []Target{{ Address: "1.1.1.1", Name: "", Labels: &[]Label{}, diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index e7745c9..7dd0157 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -14,6 +14,8 @@ paths: post: summary: "Create targets in the gNMIc Operator" operationId: "createTargets" + security: + - bearerAuth: [] requestBody: required: true content: @@ -27,13 +29,15 @@ paths: application/json: schema: $ref: '#/components/schemas/Targets' + '401': + description: Access token is missing or invalid + components: schemas: Targets: type: array items: $ref: '#/components/schemas/Target' - Label: type: object properties: @@ -41,7 +45,6 @@ components: type: string value: type: string - Target: type: object required: @@ -65,4 +68,8 @@ components: - created - updated - deleted + securitySchemes: + bearerAuth: + type: http + scheme: bearer \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 1acdf6a..e51997e 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,4 +1,5 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ + -H "Authorization: Bearer bWM7TRrWgwUCUsfwaUFPj1hlIRna2SMRl0XjvBry6ZvAc6oWSy97/BcZzcQWuDI1" \ -H "Content-Type: application/json" \ -d '[ { @@ -13,18 +14,17 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/ ]' -http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/default/target-source/http-discovery/createTargets -[ - { - "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 and data.primary_ip4.address else '' }}:{{ data.custom_fields.port }}", - "name": "{{ data.name }}", - "operation": "{{ event }}", - "profile": "{{ data.custom_fields.profile | default('') }}", - "labels": [ - { - "Key": "tags", - "Value": "{{ data.tags | map(attribute='name') | join(', ') }}" - } - ] - } -] \ No newline at end of file +curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ + -H "Authorization: Bearer bWM7TRrWgwUCUsfwaUFPj1hlWROOOONNNGGvBry6ZvAc6oWSy97/BcZzcQWuDI1" \ + -H "Content-Type: application/json" \ + -d '[ + { + "address": "1.1.1.1:123", + "name": "Router1", + "operation": "created", + "profile": "defaultProfile", + "labels": [ + { "key": "tags", "value": "tag1, tag2" } + ] + } + ]' \ No newline at end of file From 003f7ddafb5b9fac63419e836c9b7bc73629f278 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 17 May 2026 10:32:59 +0000 Subject: [PATCH 231/246] bearer token created once as kubernetes secret --- cmd/main.go | 12 ++- config/manager/manager.yaml | 1 + internal/apiserver/apiserver.go | 28 +------ internal/apiserver/auth.go | 132 ++++++++++++++++++++++++++++++++ internal/apiserver/temp.md | 2 +- 5 files changed, 140 insertions(+), 35 deletions(-) create mode 100644 internal/apiserver/auth.go diff --git a/cmd/main.go b/cmd/main.go index 9701cb3..ff262b0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -130,13 +130,7 @@ func main() { var api *apiserver.APIServer if apiAddr != "" { - apiBearerToken := os.Getenv("API_BEARER_TOKEN") - if apiBearerToken == "" { - setupLog.Error(errors.New("missing API_BEARER_TOKEN"), "unable to initialize API server") - os.Exit(1) - } - - api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize, apiBearerToken) + api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize, os.Getenv("API_BEARER_TOKEN")) if err != nil { setupLog.Error(err, "unable to initialize API server") os.Exit(1) @@ -250,6 +244,10 @@ func main() { if api != nil { err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + if err := api.InitializeAuthToken(ctx); err != nil { + return err + } + errCh := make(chan error) go func() { err := api.Server.ListenAndServe() diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b16cd6d..b377fe4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -87,6 +87,7 @@ spec: secretKeyRef: name: gnmic-api-auth key: bearer-token + optional: true securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 1af6129..02b78a3 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -8,7 +8,6 @@ package apiserver import ( "context" - "crypto/subtle" "fmt" "net/http" "strings" @@ -54,9 +53,6 @@ func New( gin.SetMode(gin.ReleaseMode) // To double-check router := gin.Default() logger := log.Log.WithValues("component", "api-server") - if bearerToken == "" { - return nil, fmt.Errorf("api bearer token cannot be empty") - } a := &APIServer{ Server: &http.Server{ @@ -68,7 +64,7 @@ func New( DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, logger: logger, - bearerToken: bearerToken, + bearerToken: strings.TrimSpace(bearerToken), } logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() @@ -142,25 +138,3 @@ func (a *APIServer) CreateTargets(c *gin.Context) { utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) c.JSON(http.StatusOK, payloadTargets) } - -func (a *APIServer) checkBearerToken(ctx *gin.Context) bool { - const bearerPrefix = "Bearer " - authHeader := strings.TrimSpace(ctx.GetHeader("Authorization")) - if !strings.HasPrefix(authHeader, bearerPrefix) { - ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing or invalid authorization header"}) - return false - } - - token := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) - if token == "" { - ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"}) - return false - } - - if subtle.ConstantTimeCompare([]byte(token), []byte(a.bearerToken)) != 1 { - ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"}) - return false - } - - return true -} diff --git a/internal/apiserver/auth.go b/internal/apiserver/auth.go new file mode 100644 index 0000000..e1db155 --- /dev/null +++ b/internal/apiserver/auth.go @@ -0,0 +1,132 @@ +package apiserver + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + apiAuthSecretName = "gnmic-api-auth" + apiAuthSecretKey = "bearer-token" +) + +func (a *APIServer) InitializeAuthToken(ctx context.Context) error { + if a.bearerToken != "" { + return nil + } + + bearerToken, err := ensureBearerToken(ctx, a.clusterReconciler, "") + if err != nil { + return err + } + a.bearerToken = bearerToken + return nil +} + +func ensureBearerToken(ctx context.Context, clusterReconciler *controller.ClusterReconciler, providedToken string) (string, error) { + if strings.TrimSpace(providedToken) != "" { + return strings.TrimSpace(providedToken), nil + } + + namespace := strings.TrimSpace(os.Getenv("POD_NAMESPACE")) + if namespace == "" { + namespace = "gnmic-system" + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + secretKey := types.NamespacedName{Name: apiAuthSecretName, Namespace: namespace} + secret := &corev1.Secret{} + if err := clusterReconciler.Get(ctx, secretKey, secret); err == nil { + token := strings.TrimSpace(string(secret.Data[apiAuthSecretKey])) + if token == "" { + return "", fmt.Errorf("secret %s/%s exists but %q is empty", namespace, apiAuthSecretName, apiAuthSecretKey) + } + return token, nil + } else if !apierrors.IsNotFound(err) { + return "", fmt.Errorf("failed to get secret %s/%s: %w", namespace, apiAuthSecretName, err) + } + + token, err := generateBearerToken() + if err != nil { + return "", err + } + + toCreate := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: apiAuthSecretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{ + apiAuthSecretKey: token, + }, + } + + if err := clusterReconciler.Create(ctx, toCreate); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("failed to create secret %s/%s: %w", namespace, apiAuthSecretName, err) + } + + if err := clusterReconciler.Get(ctx, secretKey, secret); err != nil { + return "", fmt.Errorf("failed to get existing secret %s/%s after create race: %w", namespace, apiAuthSecretName, err) + } + token = strings.TrimSpace(string(secret.Data[apiAuthSecretKey])) + if token == "" { + return "", fmt.Errorf("secret %s/%s exists but %q is empty", namespace, apiAuthSecretName, apiAuthSecretKey) + } + } + + return token, nil +} + +func generateBearerToken() (string, error) { + b := make([]byte, 48) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("failed to generate bearer token: %w", err) + } + + return base64.StdEncoding.EncodeToString(b), nil +} + +func (a *APIServer) checkBearerToken(ctx *gin.Context) bool { + if a.bearerToken == "" { + ctx.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "api authentication is not initialized"}) + return false + } + + const bearerPrefix = "Bearer " + authHeader := strings.TrimSpace(ctx.GetHeader("Authorization")) + if !strings.HasPrefix(authHeader, bearerPrefix) { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing or invalid authorization header"}) + return false + } + + token := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) + if token == "" { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"}) + return false + } + + if subtle.ConstantTimeCompare([]byte(token), []byte(a.bearerToken)) != 1 { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"}) + return false + } + + return true +} diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index e51997e..e675e3b 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ - -H "Authorization: Bearer bWM7TRrWgwUCUsfwaUFPj1hlIRna2SMRl0XjvBry6ZvAc6oWSy97/BcZzcQWuDI1" \ + -H "Authorization: Bearer SzLLLeVm7G68BzT+375zDA38g7SNs1dB9uRtyQsViS+EsqJs9kA51R2VKBWE3DI0" \ -H "Content-Type: application/json" \ -d '[ { From c56d65759ac5f2488601884809895f43a9a4ade9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 16:05:52 -0600 Subject: [PATCH 232/246] added comment to temporary message snapshots --- internal/controller/discovery/loaders/http/loader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 62d457a..5169e59 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -51,6 +51,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er return nil case <-ticker.C: + // Switch case + i only needed to test behavior for messages with different values. switch i { case 1: snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) From 72cfa78f8f445011816e15044d228fab5215387d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 16:06:13 -0600 Subject: [PATCH 233/246] renamed function to String() --- internal/controller/discovery/core/types.go | 2 +- internal/controller/discovery/mapper_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 7697bc3..5a1c8cf 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -47,7 +47,7 @@ type DiscoveryEvent struct { Event EventAction } -func (e EventAction) ToString() string { +func (e EventAction) String() string { switch e { case EventDelete: return "DELETE" diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go index 7cd9f77..dbcffe1 100644 --- a/internal/controller/discovery/mapper_test.go +++ b/internal/controller/discovery/mapper_test.go @@ -152,8 +152,8 @@ func TestGenerateEvents_AllDiscoveredTargetsBecomeApplyEvents(t *testing.T) { if event.Event != core.EventApply { t.Fatalf( "expected all events to be %s, got %s", - core.EventApply.ToString(), - event.Event.ToString(), + core.EventApply.String(), + event.Event.String(), ) } } @@ -175,8 +175,8 @@ func TestGenerateEvents_AllExistingTargetsBecomeDeleteEvents(t *testing.T) { if event.Event != core.EventDelete { t.Fatalf( "expected all events to be %s, got %s", - core.EventDelete.ToString(), - event.Event.ToString(), + core.EventDelete.String(), + event.Event.String(), ) } } From ed5b8cc53d81c17bce6a32756d1d12af9494f38a Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 19:35:55 -0600 Subject: [PATCH 234/246] return the values of reconcileDeletion directly --- internal/controller/targetsource_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 4f9b328..7f30fc8 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -93,7 +93,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request if r.DiscoveryRegistry.Exists(req.NamespacedName) { if targetSource.Generation != targetSource.Status.ObservedGeneration { - r.reconcileDeletion(ctx, req.NamespacedName, targetSource) + return r.reconcileDeletion(ctx, req.NamespacedName, targetSource) } else { logger.Info("Discovery runtime already running; reconciliation completed") return ctrl.Result{}, nil From c1a4e3ebdd9791ff5e23126ed57448a815afa5ea Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 19:40:22 -0600 Subject: [PATCH 235/246] changed function call for deferred events to processEvent --- internal/controller/discovery/message_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index aa051b9..1207536 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -320,7 +320,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot return nil default: } - if err := m.applyEvent(ctx, event, logger); err != nil { + if err := m.processEvent(ctx, event, logger); err != nil { return err } } From 21680f1233d3b72069a7665788f5cdd5648f8876 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 19:46:03 -0600 Subject: [PATCH 236/246] added error returns to applyEvent --- internal/controller/discovery/message_processor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go index 1207536..e202347 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -341,6 +341,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Error(err, "error deleting target", "targetName", event.Target.Name, ) + return err } else { logger.Info("deleted target object", "name", event.Target.Name, @@ -360,6 +361,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE logger.Error(err, "error applying target", "targetName", event.Target.Name, ) + return err } else { logger.Info("applied target object", "name", event.Target.Name, From cca49bb0de70b3bb578418ba23cb220e7aecd9a1 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 19:53:25 -0600 Subject: [PATCH 237/246] changed order of label copying --- internal/controller/discovery/mapper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go index ecbe23c..49f051c 100644 --- a/internal/controller/discovery/mapper.go +++ b/internal/controller/discovery/mapper.go @@ -28,9 +28,6 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou // Add default Target Profile from the TargetSource Spec TargetProfile t.Spec.Profile = ts.Spec.TargetProfile - // Copy TargetLabels from TargetSource Spec - maps.Copy(t.Labels, ts.Spec.TargetLabels) - // Handle labels from Source of Truth for k, v := range d.Labels { if strings.HasPrefix(k, ExternalLabelPrefix) { @@ -45,6 +42,9 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou } } + // Copy TargetLabels from TargetSource Spec + maps.Copy(t.Labels, ts.Spec.TargetLabels) + // Add TargetSource Label to the Target (precedence over all labels) t.Labels[LabelTargetSourceName] = ts.Name From f2829389fb9fd828ad8fedfaf7fb05a07223c085 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 17 May 2026 19:58:04 -0600 Subject: [PATCH 238/246] fixed failing label test --- internal/controller/discovery/mapper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go index dbcffe1..c7cfe07 100644 --- a/internal/controller/discovery/mapper_test.go +++ b/internal/controller/discovery/mapper_test.go @@ -387,7 +387,7 @@ func TestGenerateTargetResource_TargetSourceLabelsOverrideDiscoveredLabels(t *te target, _ := generateTargetResource(d, &ts) - if got := target.Labels["sharedLabel"]; got != "discoveredValue" { + if got := target.Labels["sharedLabel"]; got != "targetSourceValue" { t.Fatalf( "expected target source label to override discovered label, got %q", got, From a8a55b60693947187db1041e2bcb8fb038546caf Mon Sep 17 00:00:00 2001 From: mcdillson Date: Tue, 19 May 2026 20:56:19 +0000 Subject: [PATCH 239/246] fixed integration test resources for new DiscoveredTarget --- test/integration/http/resources/configmap.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/integration/http/resources/configmap.yaml b/test/integration/http/resources/configmap.yaml index f017566..685c670 100644 --- a/test/integration/http/resources/configmap.yaml +++ b/test/integration/http/resources/configmap.yaml @@ -6,7 +6,8 @@ data: targets.json: | [ { - "address": "clab-t1-spine1:57400", + "ip": "clab-t1-spine1", + "port": 57400, "name": "spine1", "labels": { "vendor": "nokia_srlinux", @@ -14,7 +15,8 @@ data: } }, { - "address": "clab-t1-leaf1:57400", + "ip": "clab-t1-leaf1", + "port": 57400, "name": "leaf1", "labels": { "vendor": "nokia_srlinux", @@ -22,11 +24,12 @@ data: } }, { - "address": "clab-t1-leaf2:57400", + "ip": "clab-t1-leaf2", + "port": 57400, "name": "leaf2", "labels": { "vendor": "nokia_srlinux", "role": "leaf" } } - ] \ No newline at end of file + ] From e802c7d0e244c1e1facd918e5e458dc76b0abb4c Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 20 May 2026 08:00:59 +0000 Subject: [PATCH 240/246] fix unit tests --- internal/apiserver/helpers_test.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go index af6e33d..52bf0c3 100644 --- a/internal/apiserver/helpers_test.go +++ b/internal/apiserver/helpers_test.go @@ -14,7 +14,7 @@ import ( func TestGetEventApply(t *testing.T) { target := Target{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "routername", Labels: &[]Label{}, Operation: "created", @@ -30,7 +30,7 @@ func TestGetEventApply(t *testing.T) { func TestGetEventDelete(t *testing.T) { target := Target{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "routername", Labels: &[]Label{}, Operation: "deleted", @@ -46,7 +46,7 @@ func TestGetEventDelete(t *testing.T) { func TestGetEventEmptyOperation(t *testing.T) { target := Target{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "routername", Labels: &[]Label{}, Operation: "", @@ -59,7 +59,7 @@ func TestGetEventEmptyOperation(t *testing.T) { func TestGetEventUpdate(t *testing.T) { target := Target{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "routername", Labels: &[]Label{}, Operation: "updated", @@ -159,7 +159,7 @@ func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { func TestCreateDiscoveryEvent(t *testing.T) { targets := []Target{{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "routername", Labels: &[]Label{}, Operation: "updated"}} @@ -168,7 +168,7 @@ func TestCreateDiscoveryEvent(t *testing.T) { { Target: core.DiscoveredTarget{ Name: "routername", - Address: "1.1.1.1", + Address: "1.1.1.1:22", Labels: map[string]string{}, }, Event: core.EventApply, @@ -182,7 +182,7 @@ func TestCreateDiscoveryEvent(t *testing.T) { func TestCreateDiscoveryEventEmptyName(t *testing.T) { targets := []Target{{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "", Labels: &[]Label{}, Operation: "updated"}} @@ -208,7 +208,7 @@ func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { func TestCreateDiscoveryEventWrongEvent(t *testing.T) { targets := []Target{{ - Address: "1.1.1.1", + Address: "1.1.1.1:22", Name: "", Labels: &[]Label{}, Operation: "wrongOperation"}} @@ -293,3 +293,11 @@ func TestVerifyAddressNoPort(t *testing.T) { t.Errorf("addDefaultPortIfEmpty(address) = %s; want %s", convertedAddress, expected) } } + +func TestVerifyWrongAddressFormat(t *testing.T) { + address := "10.10.10.10" + result, err := validateAddress(address) + if err == nil { + t.Errorf("TestVerifyWrongAddressFormat expected error due to wrong address format(missing port), instead got: %s", result) + } +} \ No newline at end of file From 0929af419a3505f95029601574d4a5f6dd9cc39c Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 20 May 2026 16:04:41 +0000 Subject: [PATCH 241/246] auth refactor --- cmd/main.go | 2 +- internal/apiserver/apiserver.go | 6 +- internal/apiserver/apiserver_test.go | 2 + internal/apiserver/auth.go | 102 +++++++++++++-------------- internal/apiserver/auth_test.go | 10 +++ internal/apiserver/helpers.go | 6 +- internal/apiserver/openapi.yaml | 4 +- internal/apiserver/temp.md | 18 +---- 8 files changed, 71 insertions(+), 79 deletions(-) create mode 100644 internal/apiserver/apiserver_test.go create mode 100644 internal/apiserver/auth_test.go diff --git a/cmd/main.go b/cmd/main.go index ff262b0..23f944a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -244,7 +244,7 @@ func main() { if api != nil { err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - if err := api.InitializeAuthToken(ctx); err != nil { + if err := api.InitializeBearerToken(ctx); err != nil { return err } diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 02b78a3..e064c6f 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -10,7 +10,6 @@ import ( "context" "fmt" "net/http" - "strings" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" @@ -32,7 +31,7 @@ type APIServer struct { ] chunzSize int logger logr.Logger - bearerToken string + bearerToken bool } type urlStruct struct { @@ -64,7 +63,6 @@ func New( DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, logger: logger, - bearerToken: strings.TrimSpace(bearerToken), } logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() @@ -109,7 +107,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { ) logger.Info("Received POST request for CreateTargets") - if !a.checkBearerToken(c) { + if !a.verifyBearerToken(c, a.clusterReconciler) { logger.Info("Unauthorized request for CreateTargets") return } diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go new file mode 100644 index 0000000..c129d1a --- /dev/null +++ b/internal/apiserver/apiserver_test.go @@ -0,0 +1,2 @@ +package apiserver + diff --git a/internal/apiserver/auth.go b/internal/apiserver/auth.go index e1db155..8e4898a 100644 --- a/internal/apiserver/auth.go +++ b/internal/apiserver/auth.go @@ -17,6 +17,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) const ( @@ -24,50 +25,31 @@ const ( apiAuthSecretKey = "bearer-token" ) -func (a *APIServer) InitializeAuthToken(ctx context.Context) error { - if a.bearerToken != "" { +// InitializeBearerToken creates a new bearer token in form of a Kubernetes secret, only if it doesn't exist yet. +func (a *APIServer) InitializeBearerToken(ctx context.Context) error { + if bearerTokenExists(a.clusterReconciler) { return nil } - - bearerToken, err := ensureBearerToken(ctx, a.clusterReconciler, "") + err := createBearerToken(ctx, a.clusterReconciler) if err != nil { return err } - a.bearerToken = bearerToken return nil } -func ensureBearerToken(ctx context.Context, clusterReconciler *controller.ClusterReconciler, providedToken string) (string, error) { - if strings.TrimSpace(providedToken) != "" { - return strings.TrimSpace(providedToken), nil - } - +// createBearerToken creates a new Opaque kubernetes secret +func createBearerToken(ctx context.Context, clusterReconciler *controller.ClusterReconciler) error { + logger := log.FromContext(ctx).WithValues("component", "apiserver") namespace := strings.TrimSpace(os.Getenv("POD_NAMESPACE")) - if namespace == "" { - namespace = "gnmic-system" - } ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - - secretKey := types.NamespacedName{Name: apiAuthSecretName, Namespace: namespace} - secret := &corev1.Secret{} - if err := clusterReconciler.Get(ctx, secretKey, secret); err == nil { - token := strings.TrimSpace(string(secret.Data[apiAuthSecretKey])) - if token == "" { - return "", fmt.Errorf("secret %s/%s exists but %q is empty", namespace, apiAuthSecretName, apiAuthSecretKey) - } - return token, nil - } else if !apierrors.IsNotFound(err) { - return "", fmt.Errorf("failed to get secret %s/%s: %w", namespace, apiAuthSecretName, err) - } - - token, err := generateBearerToken() + token, err := getStringForBearerToken() if err != nil { - return "", err + return err } - toCreate := &corev1.Secret{ + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: apiAuthSecretName, Namespace: namespace, @@ -78,38 +60,26 @@ func ensureBearerToken(ctx context.Context, clusterReconciler *controller.Cluste }, } - if err := clusterReconciler.Create(ctx, toCreate); err != nil { + if err := clusterReconciler.Create(ctx, secret); err != nil { if !apierrors.IsAlreadyExists(err) { - return "", fmt.Errorf("failed to create secret %s/%s: %w", namespace, apiAuthSecretName, err) - } - - if err := clusterReconciler.Get(ctx, secretKey, secret); err != nil { - return "", fmt.Errorf("failed to get existing secret %s/%s after create race: %w", namespace, apiAuthSecretName, err) - } - token = strings.TrimSpace(string(secret.Data[apiAuthSecretKey])) - if token == "" { - return "", fmt.Errorf("secret %s/%s exists but %q is empty", namespace, apiAuthSecretName, apiAuthSecretKey) + return fmt.Errorf("failed to create secret %s/%s: %w", namespace, apiAuthSecretName, err) } } - - return token, nil + logger.Info("Created %s / %s as kubernetes secret in namespace %s", apiAuthSecretName, apiAuthSecretKey, namespace) + return nil } -func generateBearerToken() (string, error) { +// getStringForBearerToken returns a base64 encoded string used for the bearer token. +func getStringForBearerToken() (string, error) { b := make([]byte, 48) if _, err := rand.Read(b); err != nil { return "", fmt.Errorf("failed to generate bearer token: %w", err) } - return base64.StdEncoding.EncodeToString(b), nil } -func (a *APIServer) checkBearerToken(ctx *gin.Context) bool { - if a.bearerToken == "" { - ctx.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "api authentication is not initialized"}) - return false - } - +// verifyBearerToken verifies bearer token from authorization header with value stored in kubernetes secret. +func (a *APIServer) verifyBearerToken(ctx *gin.Context, clusterReconciler *controller.ClusterReconciler) bool { const bearerPrefix = "Bearer " authHeader := strings.TrimSpace(ctx.GetHeader("Authorization")) if !strings.HasPrefix(authHeader, bearerPrefix) { @@ -117,16 +87,42 @@ func (a *APIServer) checkBearerToken(ctx *gin.Context) bool { return false } - token := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) - if token == "" { - ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"}) + tokenSecret, err := getBearerToken(clusterReconciler) + if err != nil { + ctx.AbortWithStatusJSON(http.StatusUnauthorized, err) return false } - if subtle.ConstantTimeCompare([]byte(token), []byte(a.bearerToken)) != 1 { + tokenHeader := strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) + if subtle.ConstantTimeCompare([]byte(tokenHeader), tokenSecret) != 1 { ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"}) return false } + return true +} +// bearerTokenExists returns true if the bearerToken exists and false if it doesn't. +func bearerTokenExists(clusterReconciler *controller.ClusterReconciler) bool { + _, err := getBearerToken(clusterReconciler) + if err != nil { + return false + } return true } + +// getBearerToken returns bearer token stored as kubernetes secret. +func getBearerToken(clusterReconciler *controller.ClusterReconciler) ([]byte, error) { + namespace := strings.TrimSpace(os.Getenv("POD_NAMESPACE")) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + var secret corev1.Secret + if err := clusterReconciler.Get(ctx, types.NamespacedName{Name: apiAuthSecretName, Namespace: namespace}, &secret); err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %w", namespace, apiAuthSecretName, err) + } + token, ok := secret.Data[apiAuthSecretKey] + if !ok { + return nil, fmt.Errorf("secret %s/%s does not contain key %q", namespace, apiAuthSecretName, apiAuthSecretKey) + } + return token, nil +} diff --git a/internal/apiserver/auth_test.go b/internal/apiserver/auth_test.go new file mode 100644 index 0000000..1c7ecf6 --- /dev/null +++ b/internal/apiserver/auth_test.go @@ -0,0 +1,10 @@ +package apiserver + +import "testing" + +func TestGenereateBearerToken(t *testing.T) { + _, err := getStringForBearerToken() + if err != nil { + t.Errorf("generateBearerToken returns err: %s", err) + } +} diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 6b116f4..2aab134 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -27,6 +27,7 @@ func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error if err != nil { return nil, err } + // validate, problem while applying verifiedAddress, err := validateAddress(target.Address) if err != nil { return nil, err @@ -35,7 +36,7 @@ func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, - Address: verifiedAddress, + Address: verifiedAddress, // FQDN, Hostname oder IP Labels: convertTargetLabelsToMap(target), }, Event: event, @@ -45,7 +46,7 @@ func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error return targets, nil } -// validateAddress checks if the address is of format IPv4 or IPv6 and adds default port 57400 if empty. +// validateAddress func validateAddress(address string) (string, error) { address, port, err := net.SplitHostPort(address) if err != nil { @@ -63,6 +64,7 @@ func getKey(u urlStruct) types.NamespacedName { Namespace: u.Namespace, Name: u.Name, } + // or kubectl get secret -n gnmic-system gnmic-api-auth -o jsonpath="{.data.bearer-token}" | base64 --decode return key } diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 7dd0157..ef52114 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -54,8 +54,8 @@ components: properties: name: type: string - address: - type: string # in the format "IP:port" + ip: + type: string # in the format "IP:port". To split up into different profile: type: string labels: diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index e675e3b..5d1500a 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ - -H "Authorization: Bearer SzLLLeVm7G68BzT+375zDA38g7SNs1dB9uRtyQsViS+EsqJs9kA51R2VKBWE3DI0" \ + -H "Authorization: Bearer 61unglgq///281Jo9tu5o+r3uVdohxrJWPXFalHlWGSet1W7NAfRVrDIP6tw+0ru" \ -H "Content-Type: application/json" \ -d '[ { @@ -12,19 +12,3 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/ ] } ]' - - -curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ - -H "Authorization: Bearer bWM7TRrWgwUCUsfwaUFPj1hlWROOOONNNGGvBry6ZvAc6oWSy97/BcZzcQWuDI1" \ - -H "Content-Type: application/json" \ - -d '[ - { - "address": "1.1.1.1:123", - "name": "Router1", - "operation": "created", - "profile": "defaultProfile", - "labels": [ - { "key": "tags", "value": "tag1, tag2" } - ] - } - ]' \ No newline at end of file From 73a2773dc9ff2861442f65343a3c3eef6ea61bb9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 20 May 2026 16:45:59 +0000 Subject: [PATCH 242/246] change gin default router --- internal/apiserver/apiserver.go | 10 +++++----- internal/apiserver/apiserver_test.go | 2 -- internal/apiserver/auth.go | 1 + internal/apiserver/temp.md | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 internal/apiserver/apiserver_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index e064c6f..b9efcf1 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -49,10 +49,9 @@ func New( discoveryChunksize int, bearerToken string, ) (*APIServer, error) { - gin.SetMode(gin.ReleaseMode) // To double-check - router := gin.Default() + router := gin.New() + router.Use(gin.Recovery()) logger := log.Log.WithValues("component", "api-server") - a := &APIServer{ Server: &http.Server{ Addr: addr, @@ -64,8 +63,8 @@ func New( chunzSize: discoveryChunksize, logger: logger, } - logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() + logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) return a, nil } @@ -126,7 +125,8 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + url.Namespace + " / " + url.Name + " does not exist"}) return } - // make sure channel is not closed if targetsource in registry is deleted + // WROMA: both of these things are not relevant here, but instead in utils.send. TODO, check with Daniel if and how this can be implemented + // make sure channel is not closed if targetsource in registry is deleted -> // timeout for sending to the channel targets, err := createDiscoveryEvent(payloadTargets) if err != nil { diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go deleted file mode 100644 index c129d1a..0000000 --- a/internal/apiserver/apiserver_test.go +++ /dev/null @@ -1,2 +0,0 @@ -package apiserver - diff --git a/internal/apiserver/auth.go b/internal/apiserver/auth.go index 8e4898a..1ef598e 100644 --- a/internal/apiserver/auth.go +++ b/internal/apiserver/auth.go @@ -124,5 +124,6 @@ func getBearerToken(clusterReconciler *controller.ClusterReconciler) ([]byte, er if !ok { return nil, fmt.Errorf("secret %s/%s does not contain key %q", namespace, apiAuthSecretName, apiAuthSecretKey) } + // kubectl get secret -n gnmic-system gnmic-api-auth -o jsonpath="{.data.bearer-token}" | base64 --decode return token, nil } diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 5d1500a..7d584f5 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ - -H "Authorization: Bearer 61unglgq///281Jo9tu5o+r3uVdohxrJWPXFalHlWGSet1W7NAfRVrDIP6tw+0ru" \ + -H "Authorization: Bearer fEPGF5qwVfM7vvEw2vYuaPojcda/a78aOtqmW4oEFYZUJF67yXluSjDoTKmey5zU" \ -H "Content-Type: application/json" \ -d '[ { From f158b532f0ec4b065507e64351aa0a88003e7b71 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 21 May 2026 13:55:30 +0000 Subject: [PATCH 243/246] openapi: correct routeloading, support udpated target fields --- go.mod | 14 +++--- go.sum | 67 ++++++++++++++++++++++++++ internal/apiserver/apiserver.go | 16 +++--- internal/apiserver/auth.go | 7 ++- internal/apiserver/gen.go | 50 ++++++++++--------- internal/apiserver/helpers.go | 15 +++--- internal/apiserver/helpers_test.go | 55 ++++++++++++++------- internal/apiserver/openapi.yaml | 16 +++--- internal/apiserver/temp.md | 9 ++-- lab/dev/resources/targetsource/ts.yaml | 13 +++++ 10 files changed, 184 insertions(+), 78 deletions(-) create mode 100644 lab/dev/resources/targetsource/ts.yaml diff --git a/go.mod b/go.mod index 3b8b6a2..5100842 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,9 @@ module github.com/gnmic/operator go 1.25.5 require ( - github.com/bytedance/gopkg v0.1.3 github.com/cert-manager/cert-manager v1.19.3 github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 - github.com/onsi/ginkgo/v2 v2.28.1 - github.com/onsi/gomega v1.39.0 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.28.3 github.com/onsi/gomega v1.40.0 @@ -23,6 +20,7 @@ require ( ) require ( + github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect @@ -55,7 +53,7 @@ require ( github.com/woodsbury/decimal128 v1.3.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect golang.org/x/arch v0.22.0 // indirect - golang.org/x/crypto v0.48.0 // indirect + golang.org/x/crypto v0.50.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -63,7 +61,7 @@ require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/PaesslerAG/gval v1.0.0 // indirect - github.com/PaesslerAG/jsonpath v0.1.1 // indirect + github.com/PaesslerAG/jsonpath v0.1.1 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -107,15 +105,15 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/term v0.42.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.44.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.79.3 // indirect diff --git a/go.sum b/go.sum index 30fedd3..9ebcdc7 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,7 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -183,8 +184,28 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.3 h1:4JvMdwtFU0imd8fHx25OJXoDMRexnf8v5NHKYSTTji4= github.com/onsi/ginkgo/v2 v2.28.3/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= @@ -279,24 +300,70 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b9efcf1..0d8bd25 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -49,8 +49,11 @@ func New( discoveryChunksize int, bearerToken string, ) (*APIServer, error) { - router := gin.New() - router.Use(gin.Recovery()) + // router := gin.New() + // router.Use(gin.Recovery()) + // gin.SetMode(gin.ReleaseMode) + router := gin.Default() + logger := log.Log.WithValues("component", "api-server") a := &APIServer{ Server: &http.Server{ @@ -63,7 +66,7 @@ func New( chunzSize: discoveryChunksize, logger: logger, } - a.routes() + RegisterHandlers(router, a) logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) return a, nil } @@ -72,11 +75,6 @@ func (a *APIServer) Router() *gin.Engine { return a.router } -func (a *APIServer) routes() { - a.router.GET("/clusters/:namespace/:name/plan", a.GetClusterPlan) - a.router.POST("/api/v1/:namespace/target-source/:name/createTargets", a.CreateTargets) -} - // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) @@ -97,7 +95,7 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { } // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. -func (a *APIServer) CreateTargets(c *gin.Context) { +func (a *APIServer) ApplyTargets(c *gin.Context) { url := parseURI(c) logger := log.FromContext(c.Request.Context()).WithValues( "component", "apiserver", diff --git a/internal/apiserver/auth.go b/internal/apiserver/auth.go index 1ef598e..ebdb4c4 100644 --- a/internal/apiserver/auth.go +++ b/internal/apiserver/auth.go @@ -65,7 +65,12 @@ func createBearerToken(ctx context.Context, clusterReconciler *controller.Cluste return fmt.Errorf("failed to create secret %s/%s: %w", namespace, apiAuthSecretName, err) } } - logger.Info("Created %s / %s as kubernetes secret in namespace %s", apiAuthSecretName, apiAuthSecretKey, namespace) + logger.Info( + "Created kubernetes auth secret", + "secret", fmt.Sprintf("%s/%s", namespace, apiAuthSecretName), + "key", apiAuthSecretKey, + "namespace", namespace, + ) return nil } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index e132343..231e293 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -49,11 +49,12 @@ type Label struct { // Target defines model for Target. type Target struct { - Address string `json:"address"` - Labels *[]Label `json:"labels,omitempty"` - Name string `json:"name"` - Operation TargetOperation `json:"operation"` - Profile *string `json:"profile,omitempty"` + Ip string `json:"ip"` + Labels *[]Label `json:"labels,omitempty"` + Name string `json:"name"` + Operation TargetOperation `json:"operation"` + Port *int `json:"port,omitempty"` + TargetProfile *string `json:"targetProfile,omitempty"` } // TargetOperation defines model for Target.Operation. @@ -62,16 +63,16 @@ type TargetOperation string // Targets defines model for Targets. type Targets = []Target -// CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. -type CreateTargetsJSONRequestBody = Targets +// ApplyTargetsJSONRequestBody defines body for ApplyTargets for application/json ContentType. +type ApplyTargetsJSONRequestBody = Targets // ServerInterface represents all server handlers. type ServerInterface interface { // Create targets in the gNMIc Operator - // (POST /createTargets) - CreateTargets(c *gin.Context) + // (POST /api/v1/:namespace/target-source/:name/applyTargets) + ApplyTargets(c *gin.Context) // Get cluster plan - // (GET /plan) + // (GET /clusters/:namespace/:name/plan) GetClusterPlan(c *gin.Context) } @@ -84,8 +85,8 @@ type ServerInterfaceWrapper struct { type MiddlewareFunc func(c *gin.Context) -// CreateTargets operation middleware -func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { +// ApplyTargets operation middleware +func (siw *ServerInterfaceWrapper) ApplyTargets(c *gin.Context) { c.Set(BearerAuthScopes, []string{}) @@ -96,7 +97,7 @@ func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { } } - siw.Handler.CreateTargets(c) + siw.Handler.ApplyTargets(c) } // GetClusterPlan operation middleware @@ -139,22 +140,23 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options ErrorHandler: errorHandler, } - router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) - router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) + router.POST(options.BaseURL+"/api/v1/:namespace/target-source/:name/applyTargets", wrapper.ApplyTargets) + router.GET(options.BaseURL+"/clusters/:namespace/:name/plan", wrapper.GetClusterPlan) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/7RTTW8aMRD9K9a0xxVL0p72RqMqQupH1OSGOBjvAE68tjseI6Fo/3tle7Owgki59MRg", - "z7x57/ntKyjXeWfRcoDmFYLaYydz+UNu0KTCk/NIrDEfv+Ax/fDRIzQQmLTdQV/BQZqIV2766u3EbZ5R", - "cep9krRDvsSWbUsYwlV8k+jkK83Y5eIz4RYa+FSfNNSDgLqwPy2XRPKY/lvZ4dUFiYhk7Wy6RRs7aFag", - "CCVjCxVE3w5ViwZTta4uQTy5rTbv+ED4N2rCNgFnGtWo+Hz9+l3HPi5/cPhCf19BQBVJ8/ExtRbfNygJ", - "aRF5P2YgzZRjGDH2zB76hKHt1mWNmpNY2P36uVTid5bgSPz5/vgkFg9LqOCAFLKnMJ/NZzeD0VZ6DQ18", - "md3O5lCBl7zPROri95la70IOymjPsoUG7iZtxVkM/M21OZzKWUab56T3Rqs8WT+H8rjFpI9ZGIrg09Mx", - "RcwHwTsbin+385v/s7bFoEj7ksq3EIghlCJEpTCEbTQmR/troTEdWuQewe4FrdBBdDoEbXfCkdD2II1u", - "J6mAZjXNw2rdrysIseskHUfrBQ9ctBW8RzENQEasvZFZ9/CpT1/wHvnOxMBID6ntwtD5pZKzfkHIkSwO", - "3Edy98hClTaR1/d93/8LAAD//+4Fc3XkBAAA", + "H4sIAAAAAAAC/7SUTW/bPAzHv4rA5zl6cdrt5FtWDEWBvRRrb0UOqswkam1JI6kARuHvPkhyUwdJgV52", + "CiPx5c8fKb+A8X3wDp0wNC/AZoe9zuZ3/YhdMgL5gCQW8/EzDulHhoDQAAtZt4Wxgr3uIp65GavXE//4", + "hEaS772mLcppbhvOpu6SkuIg2Gfjf8INNPBf/Sa/nrTXRfhbXU2kh/Tf6R7PFkgatFjv0i262EPzAIZQ", + "C7ZQQQztZLXYYbLW1WmS4Elm2a0T3CJlHbndW/Ib272DiPBPtIRtKpxlVgnGXNn6XY4fJzNxP0EzVsBo", + "IlkZ7pJrmcYjakJaRdkdNiPFlGM45NiJBBhTDus2PrdnJfUJ258/boz6lVvwpH5/u7tXq9sbqGCPxBk3", + "LBfLxcU0A6eDhQY+Ly4XS6ggaNllIbUOtt5f1E0iw0EbrAvST+wjGSwXtQ6hG2ZIguc8kAPDmxYaWM29", + "Cnhk+erbvNbGO0GXw1I6a3Jg/cRlNwrIj2HmAuVtskIR8wEH77gwvlxe/JuyLbIhG8pSvy6KysmxVRyN", + "QeZN7Lr8Mr4UGcdBq+yjxD+jU5ZVb5mt2ypPyrq97mx7tDnQPBzvzMN6XFfAse81DdDAVX5QSiYt1inZ", + "oTpekpyxNl1kQeL5xMuMQ6czkenzcTzaa5SrEnmb3E5QL097nPkrQonkcOrqIPsaRU2CVC4/juP4NwAA", + "//9oPXG9OAUAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 2aab134..6caa2bb 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -20,24 +20,25 @@ func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error if target.Name == "" { return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name.", i) } - if target.Address == "" { - return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address.", i) + if target.Ip == "" { + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Ip.", i) } event, err := getEvent(target, i) if err != nil { return nil, err } - // validate, problem while applying - verifiedAddress, err := validateAddress(target.Address) + if err != nil { return nil, err } targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ - Name: target.Name, - Address: verifiedAddress, // FQDN, Hostname oder IP - Labels: convertTargetLabelsToMap(target), + Name: target.Name, + IP: target.Ip, + Port: int32(*target.Port), + Labels: convertTargetLabelsToMap(target), + TargetProfile: *target.TargetProfile, }, Event: event, }) diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go index 52bf0c3..bc952a2 100644 --- a/internal/apiserver/helpers_test.go +++ b/internal/apiserver/helpers_test.go @@ -13,8 +13,10 @@ import ( ) func TestGetEventApply(t *testing.T) { + port := 22 target := Target{ - Address: "1.1.1.1:22", + Ip: "1.1.1.1", + Port: &port, Name: "routername", Labels: &[]Label{}, Operation: "created", @@ -29,8 +31,10 @@ func TestGetEventApply(t *testing.T) { } func TestGetEventDelete(t *testing.T) { + port := 22 target := Target{ - Address: "1.1.1.1:22", + Ip: "1.1.1.1", + Port: &port, Name: "routername", Labels: &[]Label{}, Operation: "deleted", @@ -45,8 +49,10 @@ func TestGetEventDelete(t *testing.T) { } func TestGetEventEmptyOperation(t *testing.T) { + port := 22 target := Target{ - Address: "1.1.1.1:22", + Ip: "1.1.1.1", + Port: &port, Name: "routername", Labels: &[]Label{}, Operation: "", @@ -58,8 +64,10 @@ func TestGetEventEmptyOperation(t *testing.T) { } func TestGetEventUpdate(t *testing.T) { + port := 22 target := Target{ - Address: "1.1.1.1:22", + Ip: "1.1.1.1", + Port: &port, Name: "routername", Labels: &[]Label{}, Operation: "updated", @@ -158,18 +166,24 @@ func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { } func TestCreateDiscoveryEvent(t *testing.T) { + port := 22 + targetprofile := "" targets := []Target{{ - Address: "1.1.1.1:22", - Name: "routername", - Labels: &[]Label{}, - Operation: "updated"}} + Name: "router1", + Ip: "1.1.1.1", + Port: &port, + Labels: &[]Label{}, + TargetProfile: &targetprofile, + Operation: "updated"}} expected := []core.DiscoveryEvent{ { Target: core.DiscoveredTarget{ - Name: "routername", - Address: "1.1.1.1:22", - Labels: map[string]string{}, + Name: "router1", + IP: "1.1.1.1", + Port: 22, + Labels: map[string]string{}, + TargetProfile: "", }, Event: core.EventApply, }, @@ -181,9 +195,10 @@ func TestCreateDiscoveryEvent(t *testing.T) { } func TestCreateDiscoveryEventEmptyName(t *testing.T) { + port := 22 targets := []Target{{ - Address: "1.1.1.1:22", - Name: "", + Ip: "1.1.1.1", + Port: &port, Labels: &[]Label{}, Operation: "updated"}} @@ -193,9 +208,11 @@ func TestCreateDiscoveryEventEmptyName(t *testing.T) { } } -func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { +func TestCreateDiscoveryEventEmptyIP(t *testing.T) { + port := 22 targets := []Target{{ - Address: "", + Ip: "", + Port: &port, Name: "routername", Labels: &[]Label{}, Operation: "updated"}} @@ -207,11 +224,13 @@ func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { } func TestCreateDiscoveryEventWrongEvent(t *testing.T) { + port := 22 targets := []Target{{ - Address: "1.1.1.1:22", + Ip: "1.1.1.1", + Port: &port, Name: "", Labels: &[]Label{}, - Operation: "wrongOperation"}} + Operation: "upWROOONGdated"}} result, err := createDiscoveryEvent(targets) if err == nil { @@ -300,4 +319,4 @@ func TestVerifyWrongAddressFormat(t *testing.T) { if err == nil { t.Errorf("TestVerifyWrongAddressFormat expected error due to wrong address format(missing port), instead got: %s", result) } -} \ No newline at end of file +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index ef52114..a342465 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -3,17 +3,17 @@ info: title: "gNMIc Operator REST API" version: "0.0.1" paths: - /plan: + /clusters/:namespace/:name/plan: get: summary: "Get cluster plan" operationId: "getClusterPlan" responses: '200': description: "ClusterPlan returned" - /createTargets: + /api/v1/:namespace/target-source/:name/applyTargets: post: summary: "Create targets in the gNMIc Operator" - operationId: "createTargets" + operationId: "applyTargets" security: - bearerAuth: [] requestBody: @@ -24,7 +24,7 @@ paths: $ref: '#/components/schemas/Targets' responses: '201': - description: "Targets created successfully" + description: "Targets applied successfully" content: application/json: schema: @@ -49,14 +49,16 @@ components: type: object required: - name - - address + - ip - operation properties: name: type: string ip: - type: string # in the format "IP:port". To split up into different - profile: + type: string + port: + type: integer + targetProfile: type: string labels: type: array diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 7d584f5..6d8996b 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,12 +1,13 @@ -curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ - -H "Authorization: Bearer fEPGF5qwVfM7vvEw2vYuaPojcda/a78aOtqmW4oEFYZUJF67yXluSjDoTKmey5zU" \ +curl -X POST "http://localhost:8082/api/v1/default/target-source/targetsource-1/applyTargets" \ + -H "Authorization: Bearer I+MieBB72PAD5Cu8y4iOc75q+xYiE8WhXjFA8K5Xm/4DtjA6GJufQisZuM7JIWQS" \ -H "Content-Type: application/json" \ -d '[ { - "address": "1.1.1.1:123", + "ip": "1.1.1.1", + "port": 22, "name": "Router1", "operation": "created", - "profile": "defaultProfile", + "targetProfile": "defaultProfile", "labels": [ { "key": "tags", "value": "tag1, tag2" } ] diff --git a/lab/dev/resources/targetsource/ts.yaml b/lab/dev/resources/targetsource/ts.yaml new file mode 100644 index 0000000..0bc4c79 --- /dev/null +++ b/lab/dev/resources/targetsource/ts.yaml @@ -0,0 +1,13 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: targetsource-1 +spec: + provider: + http: + url: http://targetsource-1:8080/targets + acceptPush: true + targetLabels: + datacenter: dc-a + environment: production + targetProfile: default \ No newline at end of file From b9fd2dd57659a55b652191315eb22d566d339af0 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 21 May 2026 14:48:10 +0000 Subject: [PATCH 244/246] contract rendered as md --- .../rest-api/.openapi-generator-ignore | 23 ++++++++ .../rest-api/.openapi-generator/FILES | 4 ++ .../rest-api/.openapi-generator/VERSION | 1 + .../user-guide/rest-api/Apis/DefaultApi.md | 57 +++++++++++++++++++ .../docs/user-guide/rest-api/Models/Label.md | 10 ++++ .../docs/user-guide/rest-api/Models/Target.md | 14 +++++ .../docs/user-guide/rest-api/README.md | 28 +++++++++ internal/apiserver/apiserver.go | 4 +- internal/apiserver/openapi.yaml | 15 ++++- 9 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 docs/content/docs/user-guide/rest-api/.openapi-generator-ignore create mode 100644 docs/content/docs/user-guide/rest-api/.openapi-generator/FILES create mode 100644 docs/content/docs/user-guide/rest-api/.openapi-generator/VERSION create mode 100644 docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md create mode 100644 docs/content/docs/user-guide/rest-api/Models/Label.md create mode 100644 docs/content/docs/user-guide/rest-api/Models/Target.md create mode 100644 docs/content/docs/user-guide/rest-api/README.md diff --git a/docs/content/docs/user-guide/rest-api/.openapi-generator-ignore b/docs/content/docs/user-guide/rest-api/.openapi-generator-ignore new file mode 100644 index 0000000..7484ee5 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/.openapi-generator-ignore @@ -0,0 +1,23 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/docs/content/docs/user-guide/rest-api/.openapi-generator/FILES b/docs/content/docs/user-guide/rest-api/.openapi-generator/FILES new file mode 100644 index 0000000..6398cda --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/.openapi-generator/FILES @@ -0,0 +1,4 @@ +Apis/DefaultApi.md +Models/Label.md +Models/Target.md +README.md diff --git a/docs/content/docs/user-guide/rest-api/.openapi-generator/VERSION b/docs/content/docs/user-guide/rest-api/.openapi-generator/VERSION new file mode 100644 index 0000000..ca7bf6e --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/.openapi-generator/VERSION @@ -0,0 +1 @@ +7.23.0-SNAPSHOT diff --git a/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md b/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md new file mode 100644 index 0000000..6eaf3f4 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md @@ -0,0 +1,57 @@ +# DefaultApi + +All URIs are relative to *http://localhost* + +| Method | HTTP request | Description | +|------------- | ------------- | -------------| +| [**applyTargets**](DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received in body are applied in gNMIc Operator. | +| [**getClusterPlan**](DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | + + + +# **applyTargets** +> List applyTargets(Target) + +Targets received in body are applied in gNMIc Operator. + +### Parameters + +|Name | Type | Description | Notes | +|------------- | ------------- | ------------- | -------------| +| **Target** | [**List**](../Models/Target.md)| | | + +### Return type + +[**List**](../Models/Target.md) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + + +# **getClusterPlan** +> getClusterPlan() + +Get cluster plan. + +### Parameters +This endpoint does not need any parameter. + +### Return type + +null (empty response body) + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: Not defined + diff --git a/docs/content/docs/user-guide/rest-api/Models/Label.md b/docs/content/docs/user-guide/rest-api/Models/Label.md new file mode 100644 index 0000000..7c5ad79 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/Models/Label.md @@ -0,0 +1,10 @@ +# Label +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **key** | **String** | key value, e.g. vendor | [optional] [default to null] | +| **value** | **String** | value, e.g. srlinux | [optional] [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/content/docs/user-guide/rest-api/Models/Target.md b/docs/content/docs/user-guide/rest-api/Models/Target.md new file mode 100644 index 0000000..7edb156 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/Models/Target.md @@ -0,0 +1,14 @@ +# Target +## Properties + +| Name | Type | Description | Notes | +|------------ | ------------- | ------------- | -------------| +| **name** | **String** | Routername | [default to null] | +| **ip** | **String** | IPv4 or IPv6 | [default to null] | +| **port** | **Integer** | gNMIc port | [optional] [default to null] | +| **targetProfile** | **String** | TargetProfile applied to specific router | [optional] [default to null] | +| **labels** | [**List**](Label.md) | Input of labels through key:value pair | [optional] [default to null] | +| **operation** | **String** | Either created, updated or deleted. created and updated internally is the same operation (apply) | [default to null] | + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + diff --git a/docs/content/docs/user-guide/rest-api/README.md b/docs/content/docs/user-guide/rest-api/README.md new file mode 100644 index 0000000..5721369 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/README.md @@ -0,0 +1,28 @@ +# Documentation for gNMIc Operator REST API + + +## Documentation for API Endpoints + +All URIs are relative to *http://localhost* + +| Class | Method | HTTP request | Description | +|------------ | ------------- | ------------- | -------------| +| *DefaultApi* | [**applyTargets**](Apis/DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received in body are applied in gNMIc Operator. | +*DefaultApi* | [**getClusterPlan**](Apis/DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | + + + +## Documentation for Models + + - [Label](./Models/Label.md) + - [Target](./Models/Target.md) + + + +## Documentation for Authorization + + +### bearerAuth + +- **Type**: HTTP Bearer Token authentication + diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 0d8bd25..9797a79 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -3,9 +3,11 @@ package apiserver //go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml // To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) // Then use: go generate ./internal/apiserver - +// // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +// docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli generate -i /local/internal/apiserver/openapi.yaml -g markdown -o /local/docs/content/docs/user-guide/rest-api + import ( "context" "fmt" diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index a342465..2612707 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -1,18 +1,18 @@ -openapi: 3.2.0 +openapi: 3.0.3 info: title: "gNMIc Operator REST API" version: "0.0.1" paths: /clusters/:namespace/:name/plan: get: - summary: "Get cluster plan" + summary: "Get cluster plan." operationId: "getClusterPlan" responses: '200': description: "ClusterPlan returned" /api/v1/:namespace/target-source/:name/applyTargets: post: - summary: "Create targets in the gNMIc Operator" + summary: "Targets received are applied in gNMIc Operator." operationId: "applyTargets" security: - bearerAuth: [] @@ -39,12 +39,15 @@ components: items: $ref: '#/components/schemas/Target' Label: + description: Label must be passed as key:value pair. Multiple values per key possible type: object properties: key: + description: key value, e.g. vendor type: string value: type: string + description: value, e.g. srlinux Target: type: object required: @@ -54,14 +57,19 @@ components: properties: name: type: string + description: Routername ip: type: string + description: IPv4 or IPv6 port: type: integer + description: gNMIc port targetProfile: type: string + description: TargetProfile applied to specific router labels: type: array + description: Input of labels through key:value pair items: $ref: '#/components/schemas/Label' operation: @@ -70,6 +78,7 @@ components: - created - updated - deleted + description: Either created, updated or deleted. created and updated internally is the same operation (apply) securitySchemes: bearerAuth: type: http From b2092c108001e01dd8ea56491ad4363474f71f65 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 21 May 2026 14:53:20 +0000 Subject: [PATCH 245/246] add _index.md --- docs/content/docs/user-guide/rest-api/_index.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/content/docs/user-guide/rest-api/_index.md diff --git a/docs/content/docs/user-guide/rest-api/_index.md b/docs/content/docs/user-guide/rest-api/_index.md new file mode 100644 index 0000000..9c37fb9 --- /dev/null +++ b/docs/content/docs/user-guide/rest-api/_index.md @@ -0,0 +1,8 @@ +--- +title: "REST APi" +linkTitle: "rest-api" +weight: 20 +menu: + main: + weight: 20 +--- \ No newline at end of file From 14883c972e047e115276716069d8cb525fb0df4d Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 21 May 2026 15:24:28 +0000 Subject: [PATCH 246/246] move header to README.md --- docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md | 4 ++-- docs/content/docs/user-guide/rest-api/README.md | 2 +- docs/content/docs/user-guide/rest-api/_index.md | 8 -------- 3 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 docs/content/docs/user-guide/rest-api/_index.md diff --git a/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md b/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md index 6eaf3f4..a69288c 100644 --- a/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md +++ b/docs/content/docs/user-guide/rest-api/Apis/DefaultApi.md @@ -4,7 +4,7 @@ All URIs are relative to *http://localhost* | Method | HTTP request | Description | |------------- | ------------- | -------------| -| [**applyTargets**](DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received in body are applied in gNMIc Operator. | +| [**applyTargets**](DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received are applied in gNMIc Operator. | | [**getClusterPlan**](DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | @@ -12,7 +12,7 @@ All URIs are relative to *http://localhost* # **applyTargets** > List applyTargets(Target) -Targets received in body are applied in gNMIc Operator. +Targets received are applied in gNMIc Operator. ### Parameters diff --git a/docs/content/docs/user-guide/rest-api/README.md b/docs/content/docs/user-guide/rest-api/README.md index 5721369..1867b0a 100644 --- a/docs/content/docs/user-guide/rest-api/README.md +++ b/docs/content/docs/user-guide/rest-api/README.md @@ -7,7 +7,7 @@ All URIs are relative to *http://localhost* | Class | Method | HTTP request | Description | |------------ | ------------- | ------------- | -------------| -| *DefaultApi* | [**applyTargets**](Apis/DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received in body are applied in gNMIc Operator. | +| *DefaultApi* | [**applyTargets**](Apis/DefaultApi.md#applyTargets) | **POST** /api/v1/:namespace/target-source/:name/applyTargets | Targets received are applied in gNMIc Operator. | *DefaultApi* | [**getClusterPlan**](Apis/DefaultApi.md#getClusterPlan) | **GET** /clusters/:namespace/:name/plan | Get cluster plan. | diff --git a/docs/content/docs/user-guide/rest-api/_index.md b/docs/content/docs/user-guide/rest-api/_index.md deleted file mode 100644 index 9c37fb9..0000000 --- a/docs/content/docs/user-guide/rest-api/_index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "REST APi" -linkTitle: "rest-api" -weight: 20 -menu: - main: - weight: 20 ---- \ No newline at end of file