From 181ea95a26c2f88be3b8b01f0b4e6de0c176d39d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 16 Apr 2026 12:29:50 +0000
Subject: [PATCH 001/165] extend TargetSource CRD by http token

---
 api/v1alpha1/targetsource_types.go                     | 3 ++-
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 3cf029b..057bbb2 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -39,7 +39,8 @@ type ProviderSpec struct {
 }
 
 type HTTPConfig struct {
-	URL string `json:"url,omitempty"`
+	URL   string `json:"url,omitempty"`
+	Token string `json:"token,omitempty"`
 }
 
 type ConsulConfig struct {
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 0129a88..7aa6084 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -56,6 +56,8 @@ spec:
                     type: object
                   http:
                     properties:
+                      token:
+                        type: string
                       url:
                         type: string
                     type: object

From 2fddddf3e33ba4112550c59b59583729e548bc0d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 16 Apr 2026 12:30:08 +0000
Subject: [PATCH 002/165] add pull logic as poc

---
 .../discovery/loaders/http_pull/loader.go     | 74 +++++++++++++------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go
index 22868b2..7162119 100644
--- a/internal/controller/discovery/loaders/http_pull/loader.go
+++ b/internal/controller/discovery/loaders/http_pull/loader.go
@@ -2,6 +2,9 @@ package http_pull
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
 	"time"
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -31,6 +34,10 @@ func (l *Loader) Start(
 
 	logger.Info("HTTP pull loader started")
 
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
 	// Only for debugging: emit a static snapshot every 30 seconds
 	ticker := time.NewTicker(30 * time.Second)
 	defer ticker.Stop()
@@ -42,32 +49,26 @@ func (l *Loader) Start(
 			return nil
 
 		case <-ticker.C:
-			// Example snapshot (placeholder)
-			targets := []core.DiscoveryMessage{
-				{
-					Target: core.DiscoveredTarget{
-						Name:    "ceos1",
-						Address: "clab-3-nodes-ceos1:6030",
-						Labels:  map[string]string{"TargetSource": targetsourceName},
-					},
-					Event: core.CREATE,
-				},
-				{
-					Target: core.DiscoveredTarget{
-						Name:    "leaf1",
-						Address: "clab-3-nodes-leaf1:57400",
-						Labels:  map[string]string{"TargetSource": targetsourceName},
-					},
-					Event: core.CREATE,
-				},
+			targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, spec.Provider.HTTP.URL, spec.Provider.HTTP.Token)
+			if err != nil {
+				logger.Error(err, "failed to fetch targets from HTTP endpoint")
+				continue
+			}
+
+			var messages []core.DiscoveryMessage
+			for _, target := range targets {
+				messages = append(messages, core.DiscoveryMessage{
+					Target: target,
+					Event:  core.CREATE,
+				})
 			}
 
 			// Non-blocking context-aware send
 			select {
-			case out <- targets:
-				logger.V(1).Info(
+			case out <- messages:
+				logger.Info(
 					"emitted target snapshot",
-					"count", len(targets),
+					"count", len(messages),
 				)
 			case <-ctx.Done():
 				logger.Info("context cancelled while emitting targets")
@@ -76,3 +77,34 @@ func (l *Loader) Start(
 		}
 	}
 }
+
+func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http.Client, url string, token string) ([]core.DiscoveredTarget, error) {
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodGet,
+		url,
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", "Token +"+token)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	var targets []core.DiscoveredTarget
+	if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil {
+		return nil, err
+	}
+
+	return targets, nil
+}

From 64a83cd28ab91846126ae612f09f0292cd2fd62d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 16 Apr 2026 13:47:48 +0000
Subject: [PATCH 003/165] fix request header typo

---
 internal/controller/discovery/loaders/http_pull/loader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go
index 7162119..7bd0bd1 100644
--- a/internal/controller/discovery/loaders/http_pull/loader.go
+++ b/internal/controller/discovery/loaders/http_pull/loader.go
@@ -89,7 +89,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http.
 		return nil, err
 	}
 	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Authorization", "Token +"+token)
+	req.Header.Set("Authorization", "Token "+token)
 
 	resp, err := client.Do(req)
 	if err != nil {

From 98823e83dc124853258357e34c4e1571dafe66a5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 16 Apr 2026 14:25:18 +0000
Subject: [PATCH 004/165] refactor pull implementation

---
 .../discovery/loaders/http_pull/loader.go     | 107 +++++++++++-------
 1 file changed, 66 insertions(+), 41 deletions(-)

diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go
index 7bd0bd1..fb081f8 100644
--- a/internal/controller/discovery/loaders/http_pull/loader.go
+++ b/internal/controller/discovery/loaders/http_pull/loader.go
@@ -3,6 +3,7 @@ package http_pull
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
@@ -13,9 +14,14 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 )
 
+const (
+	defaultPollInterval = 30 * time.Second
+)
+
+// Loader implements the HTTP pull discovery mechanism
 type Loader struct{}
 
-// New instantiates the http_pull loader
+// New returns a new http_pull loader instance
 func New() core.Loader {
 	return &Loader{}
 }
@@ -30,18 +36,59 @@ func (l *Loader) Start(
 	spec gnmicv1alpha1.TargetSourceSpec,
 	out chan<- []core.DiscoveryMessage,
 ) error {
-	logger := log.FromContext(ctx).WithValues("loader", l.Name())
+	logger := log.FromContext(ctx).WithValues(
+		"loader", l.Name(),
+		"targetSource", targetsourceName,
+	)
 
-	logger.Info("HTTP pull loader started")
+	// Input Validation of spec
+	if spec.Provider == nil || spec.Provider.HTTP == nil {
+		return errors.New("http_pull loader requires spec.provider.http to be set")
+	}
 
 	client := &http.Client{
 		Timeout: 30 * time.Second,
 	}
 
-	// Only for debugging: emit a static snapshot every 30 seconds
-	ticker := time.NewTicker(30 * time.Second)
+	interval := defaultPollInterval
+	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
+	logger.Info("HTTP pull loader started", "interval", interval.String())
+
+	// helper function to fetch targets and emit discovery messages
+	fetchAndEmit := func() {
+		targets, err := l.fetchTargetsFromHTTPEndpoint(
+			ctx,
+			client,
+			spec.Provider.HTTP.URL,
+			spec.Provider.HTTP.Token,
+		)
+		if err != nil {
+			logger.Error(err, "failed to fetch targets from HTTP endpoint")
+			return
+		}
+
+		messages := make([]core.DiscoveryMessage, 0, len(targets))
+		for _, target := range targets {
+			messages = append(messages, core.DiscoveryMessage{
+				Target: target,
+				Event:  core.CREATE,
+			})
+		}
+
+		select {
+		case out <- messages:
+			logger.Info("emitted target snapshot", "count", len(messages))
+		case <-ctx.Done():
+			logger.Info("context cancelled while emitting targets")
+		}
+	}
+
+	// Immediate fetch on startup
+	fetchAndEmit()
+
+	// Periodic fetch
 	for {
 		select {
 		case <-ctx.Done():
@@ -49,61 +96,39 @@ func (l *Loader) Start(
 			return nil
 
 		case <-ticker.C:
-			targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, spec.Provider.HTTP.URL, spec.Provider.HTTP.Token)
-			if err != nil {
-				logger.Error(err, "failed to fetch targets from HTTP endpoint")
-				continue
-			}
-
-			var messages []core.DiscoveryMessage
-			for _, target := range targets {
-				messages = append(messages, core.DiscoveryMessage{
-					Target: target,
-					Event:  core.CREATE,
-				})
-			}
-
-			// Non-blocking context-aware send
-			select {
-			case out <- messages:
-				logger.Info(
-					"emitted target snapshot",
-					"count", len(messages),
-				)
-			case <-ctx.Done():
-				logger.Info("context cancelled while emitting targets")
-				return nil
-			}
+			fetchAndEmit()
 		}
 	}
 }
 
-func (l *Loader) fetchTargetsFromHTTPEndpoint(ctx context.Context, client *http.Client, url string, token string) ([]core.DiscoveredTarget, error) {
-	req, err := http.NewRequestWithContext(
-		ctx,
-		http.MethodGet,
-		url,
-		nil,
-	)
+func (l *Loader) fetchTargetsFromHTTPEndpoint(
+	ctx context.Context,
+	client *http.Client,
+	url string,
+	token string,
+) ([]core.DiscoveredTarget, error) {
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
 	}
+
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("Authorization", "Token "+token)
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("HTTP request failed: %w", err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
 	}
 
 	var targets []core.DiscoveredTarget
 	if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
 	}
 
 	return targets, nil

From e76c6f35c7d9bf06ae79a7677e09e78cc5bedebf Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 16:10:07 +0000
Subject: [PATCH 005/165] restructure discovery structs

---
 .../discovery/core/loader_interface.go        |  2 +-
 .../discovery/core/message_interface.go       |  8 ++++++++
 internal/controller/discovery/core/types.go   | 19 +++++++++++++------
 3 files changed, 22 insertions(+), 7 deletions(-)
 create mode 100644 internal/controller/discovery/core/message_interface.go

diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go
index 2b87a0a..f8e343b 100644
--- a/internal/controller/discovery/core/loader_interface.go
+++ b/internal/controller/discovery/core/loader_interface.go
@@ -12,7 +12,7 @@ type Loader interface {
 	// Name returns the unique loader identifier e.g. "http_pull"
 	Name() string
 
-	// Start begins discovery and pushes target snapshots into the out channel
+	// Start begins discovery and pushes target snapshots or events into the out channel
 	// The loader must stop cleanly when ctx is cancelled
 	Start(
 		ctx context.Context,
diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go
new file mode 100644
index 0000000..0836bc6
--- /dev/null
+++ b/internal/controller/discovery/core/message_interface.go
@@ -0,0 +1,8 @@
+package core
+
+type DiscoveryMessage interface {
+	isDiscoveryMessage()
+}
+
+func (DiscoveryEvent) isDiscoveryMessage()    {}
+func (DiscoverySnapshot) isDiscoveryMessage() {}
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 406a22b..f56eaa2 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -9,14 +9,21 @@ type DiscoveredTarget struct {
 }
 
 const (
-	DELETE DiscoveryEvent = 0
-	CREATE DiscoveryEvent = 1
-	UPDATE DiscoveryEvent = 2
+	DELETE EventAction = 0
+	CREATE EventAction = 1
+	UPDATE EventAction = 2
 )
 
-type DiscoveryEvent int
+type EventAction int
 
-type DiscoveryMessage struct {
+type DiscoveryEvent struct {
 	Target DiscoveredTarget
-	Event  DiscoveryEvent
+	Event  EventAction
+}
+
+type DiscoverySnapshot struct {
+	Target      []DiscoveredTarget
+	Event       EventAction
+	SnapshotID  string
+	IsLastChunk bool
 }

From 3c18fb54fbb78db867ebba48ef3ff7e0b58e5e0a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 16:11:42 +0000
Subject: [PATCH 006/165] offload sending logic from loader implementation

---
 internal/controller/discovery/core/sender.go  | 69 +++++++++++++++++++
 .../discovery/loaders/http_pull/loader.go     | 45 ++++++------
 2 files changed, 90 insertions(+), 24 deletions(-)
 create mode 100644 internal/controller/discovery/core/sender.go

diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go
new file mode 100644
index 0000000..84de206
--- /dev/null
+++ b/internal/controller/discovery/core/sender.go
@@ -0,0 +1,69 @@
+package core
+
+import (
+	"context"
+)
+
+// sendMessages sends discovery messages over a channel in a context-aware manner
+func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case out <- messages:
+	}
+	return nil
+}
+
+// createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots
+func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot {
+	if chunkSize <= 0 {
+		chunkSize = 1
+	}
+
+	var snapshots []DiscoverySnapshot
+	totalTargets := len(targets)
+
+	for i := 0; i < totalTargets; i += chunkSize {
+		end := i + chunkSize
+		if end > totalTargets {
+			end = totalTargets
+		}
+
+		chunk := targets[i:end]
+		snapshots = append(snapshots, DiscoverySnapshot{
+			Target:      chunk,
+			SnapshotID:  snapshotID,
+			IsLastChunk: (end == totalTargets),
+		})
+	}
+
+	return snapshots
+}
+
+// SendSnapshot sends discovered targets as a snapshot over a channel in chunks
+func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error {
+	snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize)
+
+	for _, snapshot := range snapshots {
+		// Convert DiscoverySnapshot to DiscoveryMessage interface
+		messages := make([]DiscoveryMessage, 1)
+		messages[0] = snapshot
+
+		if err := sendMessages(ctx, out, messages); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// SendEvents sends discovery messages over channel in a context-aware manner
+func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error {
+	// Convert DiscoveryEvent slice to DiscoveryMessage slice
+	messages := make([]DiscoveryMessage, len(events))
+	for i, msg := range events {
+		messages[i] = msg
+	}
+
+	return sendMessages(ctx, out, messages)
+}
diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go
index 22868b2..94660d0 100644
--- a/internal/controller/discovery/loaders/http_pull/loader.go
+++ b/internal/controller/discovery/loaders/http_pull/loader.go
@@ -2,12 +2,18 @@ package http_pull
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/google/uuid"
+)
+
+const (
+	chunkSize = 100
 )
 
 type Loader struct{}
@@ -27,7 +33,11 @@ func (l *Loader) Start(
 	spec gnmicv1alpha1.TargetSourceSpec,
 	out chan<- []core.DiscoveryMessage,
 ) error {
-	logger := log.FromContext(ctx).WithValues("loader", l.Name())
+	logger := log.FromContext(ctx).WithValues(
+		"component", "loader",
+		"name", l.Name(),
+		"targetsource", targetsourceName,
+	)
 
 	logger.Info("HTTP pull loader started")
 
@@ -43,35 +53,22 @@ func (l *Loader) Start(
 
 		case <-ticker.C:
 			// Example snapshot (placeholder)
-			targets := []core.DiscoveryMessage{
+			snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString())
+			targets := []core.DiscoveredTarget{
 				{
-					Target: core.DiscoveredTarget{
-						Name:    "ceos1",
-						Address: "clab-3-nodes-ceos1:6030",
-						Labels:  map[string]string{"TargetSource": targetsourceName},
-					},
-					Event: core.CREATE,
+					Name:    "ceos1",
+					Address: "clab-3-nodes-ceos1:6030",
+					Labels:  map[string]string{"TargetSource": targetsourceName},
 				},
 				{
-					Target: core.DiscoveredTarget{
-						Name:    "leaf1",
-						Address: "clab-3-nodes-leaf1:57400",
-						Labels:  map[string]string{"TargetSource": targetsourceName},
-					},
-					Event: core.CREATE,
+					Name:    "leaf1",
+					Address: "clab-3-nodes-leaf1:57400",
+					Labels:  map[string]string{"TargetSource": targetsourceName},
 				},
 			}
 
-			// Non-blocking context-aware send
-			select {
-			case out <- targets:
-				logger.V(1).Info(
-					"emitted target snapshot",
-					"count", len(targets),
-				)
-			case <-ctx.Done():
-				logger.Info("context cancelled while emitting targets")
-				return nil
+			if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil {
+				return err
 			}
 		}
 	}

From 86ab0f35818be90b429177c013a78b7c3fed083f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 16:16:08 +0000
Subject: [PATCH 007/165] implement type assertion based on received message

---
 .../controller/discovery/target_manager.go    | 75 +++++++++++++------
 1 file changed, 52 insertions(+), 23 deletions(-)

diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go
index 245942d..f44e33c 100644
--- a/internal/controller/discovery/target_manager.go
+++ b/internal/controller/discovery/target_manager.go
@@ -9,23 +9,26 @@ import (
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/go-logr/logr"
 )
 
-// TargetManager consumes discovered targets and applies them to Kubernetes.
+// TargetManager consumes discovered targets and applies them to Kubernetes
 type TargetManager struct {
 	client       client.Client
 	scheme       *runtime.Scheme
 	targetSource *gnmicv1alpha1.TargetSource
 	in           <-chan []core.DiscoveryMessage
+	collected    map[string][]core.DiscoveredTarget
 }
 
-// NewTargetManager wires a TargetManager instance.
+// NewTargetManager wires a TargetManager instance
 func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager {
 	return &TargetManager{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
 		in:           in,
+		collected:    make(map[string][]core.DiscoveredTarget),
 	}
 }
 
@@ -43,28 +46,54 @@ func (m *TargetManager) Run(ctx context.Context) error {
 			logger.Info("target manager stopped")
 			return nil
 
-		case targets := <-m.in:
-			logger.Info(
-				"received discovered targets",
-				"count", len(targets),
-			)
+		case messages := <-m.in:
+			for _, message := range messages {
+				// Type assert to determine if this is a snapshot or event
+				switch msg := message.(type) {
+				case core.DiscoverySnapshot:
+					// Collect snapshot chunks
+					logger.Info(
+						"received snapshot chunk",
+						"snapshotID", msg.SnapshotID,
+						"targetCount", len(msg.Target),
+					)
+					m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...)
+					if msg.IsLastChunk {
+						m.processSnapshot(msg.SnapshotID, logger)
+					}
 
-			// List existing Target CRs owned by this TargetSource
-			// var existing gnmicv1alpha1.TargetList
-			// if err := m.client.List(
-			// 	ctx,
-			// 	&existing,
-			// 	client.MatchingLabels{
-			// 		"gnmic.dev/targetsource": m.targetsource,
-			// 	},
-			// ); err != nil {
-			// 	return err
-			// }
-
-			// TODO: Target Lifecycle Management
-			// 1. Compare and determine which Targets to create/update/delete
-			// 2. Create/update/delete Target CRs accordingly
-			// 3. Update TargetSource status with sync results
+				case core.DiscoveryEvent:
+					// Process individual event-driven update
+					logger.Info(
+						"received discovery event",
+						"target", msg.Target.Name,
+					)
+					switch msg.Event {
+					case core.CREATE:
+						logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
+					case core.UPDATE:
+						logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
+					case core.DELETE:
+						logger.Info("Would delete target", "name", msg.Target.Name)
+					}
+				}
+			}
 		}
 	}
 }
+
+// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
+func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) {
+	targets := m.collected[snapshotID]
+	delete(m.collected, snapshotID)
+
+	logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets))
+
+	if m.targetSource.Spec.Provider.HTTP != nil {
+		logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name)
+	}
+
+	for _, target := range targets {
+		logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels)
+	}
+}

From 8b36d7dd34e1200a50cc1c9c1176e9cbfbf97371 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 17:28:51 +0000
Subject: [PATCH 008/165] add http_push skeleton

---
 .../discovery/loaders/http_push/loader.go     | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go
index 95dc1e9..2e4ae0e 100644
--- a/internal/controller/discovery/loaders/http_push/loader.go
+++ b/internal/controller/discovery/loaders/http_push/loader.go
@@ -1,4 +1,53 @@
 package http_push
 
+import (
+	"context"
+	"errors"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
 // this file implements the logic receive target updates via HTTP push
 // REST API defined internal/apiserver
+
+// Loader implements the HTTP pull discovery mechanism
+type Loader struct{}
+
+// New returns a new http_pull loader instance
+func New() core.Loader {
+	return &Loader{}
+}
+
+func (l *Loader) Name() string {
+	return "http_push"
+}
+
+func (l *Loader) Start(
+	ctx context.Context,
+	targetsourceName string,
+	spec gnmicv1alpha1.TargetSourceSpec,
+	out chan<- []core.DiscoveryMessage,
+) error {
+	logger := log.FromContext(ctx).WithValues(
+		"component", "loader",
+		"name", l.Name(),
+		"targetsource", targetsourceName,
+	)
+	logger.Info("HTTP push loader started")
+
+	// Input Validation of spec
+	if spec.Provider == nil || spec.Provider.HTTP == nil {
+		return errors.New("http_push loader requires spec.provider.http to be set")
+	}
+
+	// Receive target updates via HTTP push
+	var targetEvents []core.DiscoveryEvent
+
+	if err := core.SendEvents(ctx, out, targetEvents); err != nil {
+		logger.Error(err, "failed to send events")
+		return nil
+	}
+	return nil
+}

From efbf727aed95de42e0a582333e90262689a2a3e5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 17:30:22 +0000
Subject: [PATCH 009/165] add http_push skeleton

---
 .../discovery/loaders/http_push/loader.go     | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go
index 95dc1e9..2e4ae0e 100644
--- a/internal/controller/discovery/loaders/http_push/loader.go
+++ b/internal/controller/discovery/loaders/http_push/loader.go
@@ -1,4 +1,53 @@
 package http_push
 
+import (
+	"context"
+	"errors"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
 // this file implements the logic receive target updates via HTTP push
 // REST API defined internal/apiserver
+
+// Loader implements the HTTP pull discovery mechanism
+type Loader struct{}
+
+// New returns a new http_pull loader instance
+func New() core.Loader {
+	return &Loader{}
+}
+
+func (l *Loader) Name() string {
+	return "http_push"
+}
+
+func (l *Loader) Start(
+	ctx context.Context,
+	targetsourceName string,
+	spec gnmicv1alpha1.TargetSourceSpec,
+	out chan<- []core.DiscoveryMessage,
+) error {
+	logger := log.FromContext(ctx).WithValues(
+		"component", "loader",
+		"name", l.Name(),
+		"targetsource", targetsourceName,
+	)
+	logger.Info("HTTP push loader started")
+
+	// Input Validation of spec
+	if spec.Provider == nil || spec.Provider.HTTP == nil {
+		return errors.New("http_push loader requires spec.provider.http to be set")
+	}
+
+	// Receive target updates via HTTP push
+	var targetEvents []core.DiscoveryEvent
+
+	if err := core.SendEvents(ctx, out, targetEvents); err != nil {
+		logger.Error(err, "failed to send events")
+		return nil
+	}
+	return nil
+}

From 60a5eb3a34a741077ec465b20266ecc58eecc59b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 17:43:15 +0000
Subject: [PATCH 010/165] refactor targetsource_controller.go

---
 .../controller/targetsource_controller.go     | 130 +++++++++++-------
 lab/dev/resources/targetsources/ctest1.yaml   |   3 +-
 2 files changed, 83 insertions(+), 50 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 8cd6f68..9fb587f 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -55,92 +55,124 @@ type TargetSourceReconciler struct {
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := log.FromContext(ctx)
+	logger := log.FromContext(ctx).WithValues(
+		"Name", req.NamespacedName,
+	)
 
+	targetSource, err := r.getTargetSource(ctx, req.NamespacedName)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Handle deletion with finalizer
+	if !targetSource.DeletionTimestamp.IsZero() {
+		return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource)
+	}
+
+	// Ensure finalizer is set
+	if err := r.ensureFinalizer(ctx, targetSource); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Check if pipeline is already running
+	if r.isPipelineRunning(req.NamespacedName) {
+		return ctrl.Result{}, nil
+	}
+
+	// Start discovery pipeline
+	if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	logger.Info("TargetSource pipeline started")
+	return ctrl.Result{}, nil
+}
+
+// getTargetSource retrieves a TargetSource by name, handling cleanup if not found
+func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) {
 	var targetSource gnmicv1alpha1.TargetSource
-	if err := r.Get(ctx, req.NamespacedName, &targetSource); err != nil {
+	if err := r.Get(ctx, key, &targetSource); err != nil {
 		// If the TargetSource no longer exists, ensure runtime cleanup
 		if client.IgnoreNotFound(err) == nil {
-			r.stopDiscovery(req.NamespacedName)
+			r.stopDiscovery(key)
 		}
-		return ctrl.Result{}, client.IgnoreNotFound(err)
+		return nil, client.IgnoreNotFound(err)
 	}
+	return &targetSource, nil
+}
 
-	logger.Info("reconciling TargetSource", "name", targetSource.Name)
-
-	// Handle deletion with finalizer
-	if !targetSource.DeletionTimestamp.IsZero() {
-		logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name)
+// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer
+func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name)
 
-		r.stopDiscovery(req.NamespacedName)
+	r.stopDiscovery(key)
 
-		// Remove finalizer if exists
-		if controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) {
-			controllerutil.RemoveFinalizer(&targetSource, targetSourceFinalizer)
-			if err := r.Update(ctx, &targetSource); err != nil {
-				return ctrl.Result{}, err
-			}
+	// Remove finalizer if exists
+	if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) {
+		controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer)
+		if err := r.Update(ctx, targetSource); err != nil {
+			return ctrl.Result{}, err
 		}
+	}
 
-		return ctrl.Result{}, nil
+	return ctrl.Result{}, nil
+}
+
+// ensureFinalizer adds the finalizer if not present and updates the TargetSource
+func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error {
+	if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) {
+		return nil
 	}
 
-	// Ensure finalizer is set
-	if !controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) {
-		controllerutil.AddFinalizer(&targetSource, targetSourceFinalizer)
-		if err := r.Update(ctx, &targetSource); err != nil {
-			return ctrl.Result{}, err
-		}
-		// Requeue to continue with a clean state
-		return ctrl.Result{}, nil
+	controllerutil.AddFinalizer(targetSource, targetSourceFinalizer)
+	if err := r.Update(ctx, targetSource); err != nil {
+		return err
 	}
 
-	// TODO:
-	// 1. Check if a pipeline is already running for this TargetSource
-	// 2. If not, create and start a new pipeline:
-	//    a. Create a Loader based on TargetSource spec
-	//    b. Start the Loader in a new goroutine, passing a channel for discovered targets
-	//    c. Start a TargetManager in another goroutine to consume discovered targets and manage Target CRs
-	// 3. If yes, check if the spec has changed and restart the pipeline if needed
+	return nil
+}
 
+// isPipelineRunning checks if a discovery pipeline is already running for the given key
+func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool {
 	r.mu.Lock()
-	_, exists := r.running[req.NamespacedName]
-	r.mu.Unlock()
+	defer r.mu.Unlock()
 
-	// If a targetsource loader exists, return immediately without starting
-	// any new loader or target manager
-	if exists {
-		return ctrl.Result{}, nil
-	}
+	_, exists := r.running[key]
+	return exists
+}
 
-	loader, err := discovery.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec)
+// startDiscoveryPipeline creates and starts the loader and target manager
+func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error {
+	loader, err := discovery.NewLoader(
+		targetSource.ObjectMeta.Name,
+		targetSource.ObjectMeta.Namespace,
+		targetSource.Spec,
+	)
 	if err != nil {
-		return ctrl.Result{}, err
+		return err
 	}
 
 	runtimeCtx, cancel := context.WithCancel(context.Background())
-
 	targetChannel := make(chan []core.DiscoveryMessage, 10)
 
-	// start loader
+	// Start loader
 	go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel)
 
-	// start target manager
+	// Start target manager
 	manager := discovery.NewTargetManager(
 		r.Client,
 		r.Scheme,
-		&targetSource,
+		targetSource,
 		targetChannel,
 	)
 	go manager.Run(runtimeCtx)
 
 	r.mu.Lock()
-	r.running[req.NamespacedName] = runningSource{cancel: cancel}
+	r.running[key] = runningSource{cancel: cancel}
 	r.mu.Unlock()
 
-	logger.Info("TargetSource pipeline started", "name", targetSource.Name)
-
-	return ctrl.Result{}, nil
+	return nil
 }
 
 // stopDiscovery stops and removes a running discovery pipeline
diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml
index e0aea43..bdb1bf8 100644
--- a/lab/dev/resources/targetsources/ctest1.yaml
+++ b/lab/dev/resources/targetsources/ctest1.yaml
@@ -5,7 +5,8 @@ metadata:
 spec:
   provider:
     http:
-      url: http://inventory-service:8080/targets
+      url: http://srbsci-121:8081/api/dcim/devices/?export=test
+      token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1
   labels:
     source: inventory
   type: http

From 1bc5d2be5e429076d9bb95578cf56eb2a42fda14 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 17:49:59 +0000
Subject: [PATCH 011/165] remove targetsource ressource to not impact main

---
 lab/dev/resources/targetsources/ctest1.yaml | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100644 lab/dev/resources/targetsources/ctest1.yaml

diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml
deleted file mode 100644
index bdb1bf8..0000000
--- a/lab/dev/resources/targetsources/ctest1.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: operator.gnmic.dev/v1alpha1
-kind: TargetSource
-metadata:
-  name: http-discovery
-spec:
-  provider:
-    http:
-      url: http://srbsci-121:8081/api/dcim/devices/?export=test
-      token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1
-  labels:
-    source: inventory
-  type: http
-  profile: eos
\ No newline at end of file

From 14e7765ae44c19dad961dc367a7a2da4ff818190 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 18:09:17 +0000
Subject: [PATCH 012/165] add batching to DiscoveryEvent's

---
 internal/controller/discovery/core/sender.go  | 29 +++++++++++++++----
 .../discovery/loaders/http_push/loader.go     |  3 +-
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go
index 84de206..cc8e3c1 100644
--- a/internal/controller/discovery/core/sender.go
+++ b/internal/controller/discovery/core/sender.go
@@ -58,12 +58,29 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []
 }
 
 // SendEvents sends discovery messages over channel in a context-aware manner
-func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error {
-	// Convert DiscoveryEvent slice to DiscoveryMessage slice
-	messages := make([]DiscoveryMessage, len(events))
-	for i, msg := range events {
-		messages[i] = msg
+func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error {
+	if chunkSize <= 0 {
+		chunkSize = 1
 	}
 
-	return sendMessages(ctx, out, messages)
+	totalEvents := len(events)
+	for i := 0; i < totalEvents; i += chunkSize {
+		end := i + chunkSize
+		if end > totalEvents {
+			end = totalEvents
+		}
+
+		chunk := events[i:end]
+		// Convert DiscoveryEvent chunk to DiscoveryMessage slice
+		messages := make([]DiscoveryMessage, len(chunk))
+		for j, event := range chunk {
+			messages[j] = event
+		}
+
+		if err := sendMessages(ctx, out, messages); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go
index 2e4ae0e..572df1d 100644
--- a/internal/controller/discovery/loaders/http_push/loader.go
+++ b/internal/controller/discovery/loaders/http_push/loader.go
@@ -44,8 +44,9 @@ func (l *Loader) Start(
 
 	// Receive target updates via HTTP push
 	var targetEvents []core.DiscoveryEvent
+	const chunkSize = 100
 
-	if err := core.SendEvents(ctx, out, targetEvents); err != nil {
+	if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil {
 		logger.Error(err, "failed to send events")
 		return nil
 	}

From b4337ead8b4eb7f3bb3b764f2141707f69698483 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 18:26:33 +0000
Subject: [PATCH 013/165] refactored sender.go

---
 internal/controller/discovery/core/sender.go  | 65 ++++++++++---------
 internal/controller/discovery/core/types.go   |  2 +-
 .../controller/discovery/target_manager.go    |  4 +-
 3 files changed, 39 insertions(+), 32 deletions(-)

diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go
index cc8e3c1..3e6b4aa 100644
--- a/internal/controller/discovery/core/sender.go
+++ b/internal/controller/discovery/core/sender.go
@@ -14,6 +14,24 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [
 	return nil
 }
 
+// forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize
+func forEachChunk(total, chunkSize int, fn func(start, end int) error) error {
+	if chunkSize <= 0 {
+		chunkSize = 1
+	}
+
+	for i := 0; i < total; i += chunkSize {
+		end := i + chunkSize
+		if end > total {
+			end = total
+		}
+		if err := fn(i, end); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots
 func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot {
 	if chunkSize <= 0 {
@@ -23,19 +41,15 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu
 	var snapshots []DiscoverySnapshot
 	totalTargets := len(targets)
 
-	for i := 0; i < totalTargets; i += chunkSize {
-		end := i + chunkSize
-		if end > totalTargets {
-			end = totalTargets
-		}
-
+	_ = forEachChunk(totalTargets, chunkSize, func(i, end int) error {
 		chunk := targets[i:end]
 		snapshots = append(snapshots, DiscoverySnapshot{
-			Target:      chunk,
+			Targets:     chunk,
 			SnapshotID:  snapshotID,
 			IsLastChunk: (end == totalTargets),
 		})
-	}
+		return nil
+	})
 
 	return snapshots
 }
@@ -45,7 +59,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []
 	snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize)
 
 	for _, snapshot := range snapshots {
-		// Convert DiscoverySnapshot to DiscoveryMessage interface
+		// Convert DiscoverySnapshot to DiscoveryMessage
 		messages := make([]DiscoveryMessage, 1)
 		messages[0] = snapshot
 
@@ -57,30 +71,23 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []
 	return nil
 }
 
+func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage {
+	message := make([]DiscoveryMessage, len(events))
+	for i, event := range events {
+		message[i] = event
+	}
+	return message
+}
+
 // SendEvents sends discovery messages over channel in a context-aware manner
 func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error {
 	if chunkSize <= 0 {
 		chunkSize = 1
 	}
+	messages := eventsToMessages(events)
+	total := len(messages)
 
-	totalEvents := len(events)
-	for i := 0; i < totalEvents; i += chunkSize {
-		end := i + chunkSize
-		if end > totalEvents {
-			end = totalEvents
-		}
-
-		chunk := events[i:end]
-		// Convert DiscoveryEvent chunk to DiscoveryMessage slice
-		messages := make([]DiscoveryMessage, len(chunk))
-		for j, event := range chunk {
-			messages[j] = event
-		}
-
-		if err := sendMessages(ctx, out, messages); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return forEachChunk(total, chunkSize, func(i, end int) error {
+		return sendMessages(ctx, out, messages[i:end])
+	})
 }
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index f56eaa2..cac249d 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -22,7 +22,7 @@ type DiscoveryEvent struct {
 }
 
 type DiscoverySnapshot struct {
-	Target      []DiscoveredTarget
+	Targets     []DiscoveredTarget
 	Event       EventAction
 	SnapshotID  string
 	IsLastChunk bool
diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go
index f44e33c..153723c 100644
--- a/internal/controller/discovery/target_manager.go
+++ b/internal/controller/discovery/target_manager.go
@@ -55,9 +55,9 @@ func (m *TargetManager) Run(ctx context.Context) error {
 					logger.Info(
 						"received snapshot chunk",
 						"snapshotID", msg.SnapshotID,
-						"targetCount", len(msg.Target),
+						"targetCount", len(msg.Targets),
 					)
-					m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...)
+					m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...)
 					if msg.IsLastChunk {
 						m.processSnapshot(msg.SnapshotID, logger)
 					}

From 30f3ecb6f291c55d7cdd2b73e9257189acacd106 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 17 Apr 2026 19:20:56 +0000
Subject: [PATCH 014/165] load buffer and chunk size from env variable

---
 cmd/main.go                                      | 10 ++++++++--
 internal/controller/discovery/core/sender.go     | 11 -----------
 internal/controller/discovery/core/types.go      |  4 ++++
 internal/controller/discovery/loader.go          |  4 ++--
 .../discovery/loaders/http_pull/loader.go        | 16 +++++++---------
 .../discovery/loaders/http_push/loader.go        | 13 +++++++------
 internal/controller/targetsource_controller.go   | 10 +++++++++-
 7 files changed, 37 insertions(+), 31 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index 4c37a0d..eacdee5 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -64,6 +64,8 @@ func main() {
 	var probeAddr string
 	var devMode bool
 	var apiAddr string
+	var discoveryChunkSize int
+	var discoveryBufferSize int
 	flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.")
 	flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.")
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
@@ -71,6 +73,8 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.IntVar(&discoveryChunkSize, "discovery-chunk-size", 100, "Maximum number of targets/events sent in a single discovery message.")
+	flag.IntVar(&discoveryBufferSize, "discovery-buffer-size", 10, "Amount of discovery messages that can be queued in the channel buffer.")
 	opts := zap.Options{
 		Development: devMode,
 	}
@@ -117,8 +121,10 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.TargetSourceReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:     mgr.GetClient(),
+		Scheme:     mgr.GetScheme(),
+		BufferSize: discoveryBufferSize,
+		ChunkSize:  discoveryChunkSize,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "TargetSource")
 		os.Exit(1)
diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go
index 3e6b4aa..843f30e 100644
--- a/internal/controller/discovery/core/sender.go
+++ b/internal/controller/discovery/core/sender.go
@@ -16,10 +16,6 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [
 
 // forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize
 func forEachChunk(total, chunkSize int, fn func(start, end int) error) error {
-	if chunkSize <= 0 {
-		chunkSize = 1
-	}
-
 	for i := 0; i < total; i += chunkSize {
 		end := i + chunkSize
 		if end > total {
@@ -34,10 +30,6 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error {
 
 // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots
 func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot {
-	if chunkSize <= 0 {
-		chunkSize = 1
-	}
-
 	var snapshots []DiscoverySnapshot
 	totalTargets := len(targets)
 
@@ -81,9 +73,6 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage {
 
 // SendEvents sends discovery messages over channel in a context-aware manner
 func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error {
-	if chunkSize <= 0 {
-		chunkSize = 1
-	}
 	messages := eventsToMessages(events)
 	total := len(messages)
 
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index cac249d..69a407e 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -1,5 +1,9 @@
 package core
 
+type LoaderConfig struct {
+	ChunkSize int
+}
+
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go
index ad1e83f..e0834c0 100644
--- a/internal/controller/discovery/loader.go
+++ b/internal/controller/discovery/loader.go
@@ -9,12 +9,12 @@ import (
 )
 
 // NewLoader creates a loader by name
-func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
+func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) {
 	loaderName := namespace + "/" + name
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		return http_pull.New(), nil
+		return http_pull.New(cfg), nil
 	case spec.Provider.Consul != nil:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
 	default:
diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go
index 94660d0..8213c8a 100644
--- a/internal/controller/discovery/loaders/http_pull/loader.go
+++ b/internal/controller/discovery/loaders/http_pull/loader.go
@@ -12,15 +12,13 @@ import (
 	"github.com/google/uuid"
 )
 
-const (
-	chunkSize = 100
-)
-
-type Loader struct{}
+type Loader struct {
+	cfg core.LoaderConfig
+}
 
-// New instantiates the http_pull loader
-func New() core.Loader {
-	return &Loader{}
+// New instantiates the http_pull loader with the provided config
+func New(cfg core.LoaderConfig) core.Loader {
+	return &Loader{cfg: cfg}
 }
 
 func (l *Loader) Name() string {
@@ -67,7 +65,7 @@ func (l *Loader) Start(
 				},
 			}
 
-			if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil {
+			if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
 				return err
 			}
 		}
diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go
index 572df1d..025176f 100644
--- a/internal/controller/discovery/loaders/http_push/loader.go
+++ b/internal/controller/discovery/loaders/http_push/loader.go
@@ -13,11 +13,13 @@ import (
 // REST API defined internal/apiserver
 
 // Loader implements the HTTP pull discovery mechanism
-type Loader struct{}
+type Loader struct{
+	cfg core.LoaderConfig
+}
 
-// New returns a new http_pull loader instance
-func New() core.Loader {
-	return &Loader{}
+// New returns a new http_push loader instance configured with cfg
+func New(cfg core.LoaderConfig) core.Loader {
+	return &Loader{cfg: cfg}
 }
 
 func (l *Loader) Name() string {
@@ -44,9 +46,8 @@ func (l *Loader) Start(
 
 	// Receive target updates via HTTP push
 	var targetEvents []core.DiscoveryEvent
-	const chunkSize = 100
 
-	if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil {
+	if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil {
 		logger.Error(err, "failed to send events")
 		return nil
 	}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 9fb587f..fce6742 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -45,6 +45,9 @@ type TargetSourceReconciler struct {
 
 	mu      sync.Mutex
 	running map[client.ObjectKey]runningSource
+
+	BufferSize int
+	ChunkSize  int
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -144,17 +147,22 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool {
 
 // startDiscoveryPipeline creates and starts the loader and target manager
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error {
+	cfg := core.LoaderConfig{
+		ChunkSize: r.ChunkSize,
+	}
+
 	loader, err := discovery.NewLoader(
 		targetSource.ObjectMeta.Name,
 		targetSource.ObjectMeta.Namespace,
 		targetSource.Spec,
+		cfg,
 	)
 	if err != nil {
 		return err
 	}
 
 	runtimeCtx, cancel := context.WithCancel(context.Background())
-	targetChannel := make(chan []core.DiscoveryMessage, 10)
+	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 
 	// Start loader
 	go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel)

From 586001e963125cde484ddead4e16ef11c4939c7b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 22 Apr 2026 12:58:44 +0000
Subject: [PATCH 015/165] rename file to helpers

---
 internal/controller/discovery/core/{sender.go => helpers.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename internal/controller/discovery/core/{sender.go => helpers.go} (100%)

diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/helpers.go
similarity index 100%
rename from internal/controller/discovery/core/sender.go
rename to internal/controller/discovery/core/helpers.go

From 7430815bb78b417702c6df5b8e85377e63193a4b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 22 Apr 2026 13:03:03 +0000
Subject: [PATCH 016/165] rebuild and reformat

---
 api/v1alpha1/zz_generated.deepcopy.go                | 2 +-
 internal/controller/discovery/loaders/push/loader.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 3848412..61e81fd 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -21,7 +21,7 @@ limitations under the License.
 package v1alpha1
 
 import (
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go
index 5b00081..ec70830 100644
--- a/internal/controller/discovery/loaders/push/loader.go
+++ b/internal/controller/discovery/loaders/push/loader.go
@@ -13,7 +13,7 @@ import (
 // REST API defined internal/apiserver
 
 // Loader implements the HTTP pull discovery mechanism
-type Loader struct{
+type Loader struct {
 	cfg core.LoaderConfig
 }
 

From 255a1f3facb9f3c6b4e4ae17b4ad1afae0bcd0bd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 23 Apr 2026 07:15:38 +0000
Subject: [PATCH 017/165] consolidate pull and push to http

---
 internal/controller/discovery/loader.go       |  4 +-
 .../controller/discovery/loaders/all/all.go   |  3 +-
 .../loaders/{pull => http}/loader.go          | 10 ++--
 .../discovery/loaders/http/loader_test.go     |  1 +
 .../discovery/loaders/pull/loader_test.go     |  1 -
 .../discovery/loaders/push/loader.go          | 55 -------------------
 .../discovery/loaders/push/loader_test.go     |  1 -
 7 files changed, 9 insertions(+), 66 deletions(-)
 rename internal/controller/discovery/loaders/{pull => http}/loader.go (89%)
 create mode 100644 internal/controller/discovery/loaders/http/loader_test.go
 delete mode 100644 internal/controller/discovery/loaders/pull/loader_test.go
 delete mode 100644 internal/controller/discovery/loaders/push/loader.go
 delete mode 100644 internal/controller/discovery/loaders/push/loader_test.go

diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go
index 64dc8d3..42ce8da 100644
--- a/internal/controller/discovery/loader.go
+++ b/internal/controller/discovery/loader.go
@@ -5,7 +5,7 @@ import (
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
-	pull "github.com/gnmic/operator/internal/controller/discovery/loaders/pull"
+	http "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
 
 // NewLoader creates a loader by name
@@ -14,7 +14,7 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		return pull.New(cfg), nil
+		return http.New(cfg), nil
 	case spec.Provider.Consul != nil:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
 	default:
diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go
index d05604b..3590cda 100644
--- a/internal/controller/discovery/loaders/all/all.go
+++ b/internal/controller/discovery/loaders/all/all.go
@@ -1,6 +1,5 @@
 package all
 
 import (
-	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/pull"
-	// _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/push"
+	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
diff --git a/internal/controller/discovery/loaders/pull/loader.go b/internal/controller/discovery/loaders/http/loader.go
similarity index 89%
rename from internal/controller/discovery/loaders/pull/loader.go
rename to internal/controller/discovery/loaders/http/loader.go
index 729233d..f014a2f 100644
--- a/internal/controller/discovery/loaders/pull/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -1,4 +1,4 @@
-package pull
+package http
 
 import (
 	"context"
@@ -16,13 +16,13 @@ type Loader struct {
 	cfg core.LoaderConfig
 }
 
-// New instantiates the pull loader with the provided config
+// New instantiates the http loader with the provided config
 func New(cfg core.LoaderConfig) core.Loader {
 	return &Loader{cfg: cfg}
 }
 
 func (l *Loader) Name() string {
-	return "pull"
+	return "http"
 }
 
 func (l *Loader) Start(
@@ -37,7 +37,7 @@ func (l *Loader) Start(
 		"targetsource", targetsourceName,
 	)
 
-	logger.Info("HTTP pull loader started")
+	logger.Info("HTTP loader started")
 
 	// Only for debugging: emit a static snapshot every 30 seconds
 	ticker := time.NewTicker(30 * time.Second)
@@ -46,7 +46,7 @@ func (l *Loader) Start(
 	for {
 		select {
 		case <-ctx.Done():
-			logger.Info("HTTP pull loader stopped")
+			logger.Info("HTTP loader stopped")
 			return nil
 
 		case <-ticker.C:
diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/loader_test.go
new file mode 100644
index 0000000..d02cfda
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/loader_test.go
@@ -0,0 +1 @@
+package http
diff --git a/internal/controller/discovery/loaders/pull/loader_test.go b/internal/controller/discovery/loaders/pull/loader_test.go
deleted file mode 100644
index 0493bec..0000000
--- a/internal/controller/discovery/loaders/pull/loader_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package pull
diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go
deleted file mode 100644
index ec70830..0000000
--- a/internal/controller/discovery/loaders/push/loader.go
+++ /dev/null
@@ -1,55 +0,0 @@
-package push
-
-import (
-	"context"
-	"errors"
-
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-	"github.com/gnmic/operator/internal/controller/discovery/core"
-	"sigs.k8s.io/controller-runtime/pkg/log"
-)
-
-// this file implements the logic receive target updates via HTTP push
-// REST API defined internal/apiserver
-
-// Loader implements the HTTP pull discovery mechanism
-type Loader struct {
-	cfg core.LoaderConfig
-}
-
-// New returns a new http_push loader instance configured with cfg
-func New(cfg core.LoaderConfig) core.Loader {
-	return &Loader{cfg: cfg}
-}
-
-func (l *Loader) Name() string {
-	return "http_push"
-}
-
-func (l *Loader) Start(
-	ctx context.Context,
-	targetsourceName string,
-	spec gnmicv1alpha1.TargetSourceSpec,
-	out chan<- []core.DiscoveryMessage,
-) error {
-	logger := log.FromContext(ctx).WithValues(
-		"component", "loader",
-		"name", l.Name(),
-		"targetsource", targetsourceName,
-	)
-	logger.Info("HTTP push loader started")
-
-	// Input Validation of spec
-	if spec.Provider == nil || spec.Provider.HTTP == nil {
-		return errors.New("http_push loader requires spec.provider.http to be set")
-	}
-
-	// Receive target updates via HTTP push
-	var targetEvents []core.DiscoveryEvent
-
-	if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil {
-		logger.Error(err, "failed to send events")
-		return nil
-	}
-	return nil
-}
diff --git a/internal/controller/discovery/loaders/push/loader_test.go b/internal/controller/discovery/loaders/push/loader_test.go
deleted file mode 100644
index 63fdf61..0000000
--- a/internal/controller/discovery/loaders/push/loader_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package push

From bd2b45f63366eaaba0170c37e1783e018049eaca Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 23 Apr 2026 12:34:50 +0000
Subject: [PATCH 018/165] rename target manager to target applier

---
 .../controller/discovery/{target_manager.go => target_applier.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename internal/controller/discovery/{target_manager.go => target_applier.go} (100%)

diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_applier.go
similarity index 100%
rename from internal/controller/discovery/target_manager.go
rename to internal/controller/discovery/target_applier.go

From 5a561a768f1a2d17e1ed09a40b82884bc512527f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 23 Apr 2026 13:07:25 +0000
Subject: [PATCH 019/165] implement a generic registry

---
 .../controller/discovery/registry/registry.go | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 internal/controller/discovery/registry/registry.go

diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go
new file mode 100644
index 0000000..7da0757
--- /dev/null
+++ b/internal/controller/discovery/registry/registry.go
@@ -0,0 +1,61 @@
+package registry
+
+import (
+	"fmt"
+	"sync"
+)
+
+/* USAGE
+
+// create registry once in main.go
+discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]()
+
+// inside targetsource controller, when starting discovery pipeline:
+key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName)
+if err := discoveryReg.Register(key, out); err != nil {
+  logger.Error(err, "could not register loader")
+  return err
+}
+defer discoveryReg.Unregister(key)
+
+// CHECK REGISTRY
+ch, ok := discoveryReg.Get(ns + "/" + ts)
+if !ok {
+  http.Error(w, "no loader for targetsource", http.StatusNotFound)
+  return
+}
+// then deliver payload to ch
+*/
+
+// Registry is a thread-safe map: key -> channel of T.
+type Registry[T any] struct {
+	mu sync.RWMutex
+	m  map[string]chan<- T
+}
+
+func NewRegistry[T any]() *Registry[T] {
+	return &Registry[T]{m: make(map[string]chan<- T)}
+}
+
+func (r *Registry[T]) Register(key string, ch chan<- T) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if _, exists := r.m[key]; exists {
+		return fmt.Errorf("already registered: %s", key)
+	}
+	r.m[key] = ch
+	return nil
+}
+
+func (r *Registry[T]) Unregister(key string) {
+	r.mu.Lock()
+	delete(r.m, key)
+	r.mu.Unlock()
+}
+
+func (r *Registry[T]) Get(key string) (chan<- T, bool) {
+	r.mu.RLock()
+	ch, ok := r.m[key]
+	r.mu.RUnlock()
+	return ch, ok
+}

From f5481b8f9c7627d9c499c9156afdb3c7c2346146 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 23 Apr 2026 13:08:01 +0000
Subject: [PATCH 020/165] add a discoveryTegistry to share targetchannel
 between apiserver and target manager

---
 cmd/main.go                                   | 14 +++++++++----
 internal/apiserver/apiserver.go               |  4 ++++
 .../controller/targetsource_controller.go     | 20 ++++++++++++++++---
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index eacdee5..5cf8169 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -40,6 +40,8 @@ import (
 	operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/apiserver"
 	"github.com/gnmic/operator/internal/controller"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1"
 	//+kubebuilder:scaffold:imports
 )
@@ -83,6 +85,8 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]()
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
@@ -121,10 +125,11 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.TargetSourceReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		BufferSize: discoveryBufferSize,
-		ChunkSize:  discoveryChunkSize,
+		Client:            mgr.GetClient(),
+		Scheme:            mgr.GetScheme(),
+		BufferSize:        discoveryBufferSize,
+		ChunkSize:         discoveryChunkSize,
+		DiscoveryRegistry: discoveryRegistry,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "TargetSource")
 		os.Exit(1)
@@ -226,6 +231,7 @@ func main() {
 
 	if apiAddr != "" {
 		apiServer := apiserver.New(apiAddr, clusterReconciler)
+		apiServer.DiscoveryRegistry = discoveryRegistry
 		err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
 			errCh := make(chan error)
 			go func() {
diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go
index f31abaa..b84eb9a 100644
--- a/internal/apiserver/apiserver.go
+++ b/internal/apiserver/apiserver.go
@@ -5,11 +5,15 @@ import (
 	"net/http"
 
 	"github.com/gnmic/operator/internal/controller"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/gnmic/operator/internal/controller/discovery/registry"
 )
 
 type APIServer struct {
 	Server            *http.Server
 	clusterReconciler *controller.ClusterReconciler
+
+	DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage]
 }
 
 func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer {
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index fce6742..c714acc 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -30,6 +30,7 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/all"
+	"github.com/gnmic/operator/internal/controller/discovery/registry"
 )
 
 const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
@@ -48,6 +49,8 @@ type TargetSourceReconciler struct {
 
 	BufferSize int
 	ChunkSize  int
+
+	DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -164,6 +167,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta
 	runtimeCtx, cancel := context.WithCancel(context.Background())
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 
+	registryKey := key.Namespace + "/" + key.Name
+	if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil {
+		cancel()
+		return err
+	}
+
 	// Start loader
 	go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel)
 
@@ -187,12 +196,17 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta
 // for the given TargetSource key
 func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) {
 	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	if running, ok := r.running[key]; ok {
+	running, ok := r.running[key]
+	if ok {
 		running.cancel()
 		delete(r.running, key)
 	}
+	r.mu.Unlock()
+
+	if ok {
+		registryKey := key.Namespace + "/" + key.Name
+		r.DiscoveryRegistry.Unregister(registryKey)
+	}
 }
 
 // SetupWithManager sets up the controller with the Manager.

From 22683f4e4b0ee7853f45c8fce20c7d1646317162 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 23 Apr 2026 16:37:08 +0000
Subject: [PATCH 021/165] remove unused event action from DiscoverySnapshot

---
 internal/controller/discovery/core/types.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 69a407e..61209fd 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -27,7 +27,6 @@ type DiscoveryEvent struct {
 
 type DiscoverySnapshot struct {
 	Targets     []DiscoveredTarget
-	Event       EventAction
 	SnapshotID  string
 	IsLastChunk bool
 }

From 922bbc6a6be0900f27e9aed9c09d6bce1c19caf6 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 07:32:40 +0000
Subject: [PATCH 022/165] rename target manager to target applier

---
 .../controller/discovery/target_applier.go     | 18 +++++++++---------
 internal/controller/targetsource_controller.go |  4 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go
index 153723c..3babebf 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_applier.go
@@ -12,8 +12,8 @@ import (
 	"github.com/go-logr/logr"
 )
 
-// TargetManager consumes discovered targets and applies them to Kubernetes
-type TargetManager struct {
+// TargetApplier consumes discovered targets and applies them to Kubernetes
+type TargetApplier struct {
 	client       client.Client
 	scheme       *runtime.Scheme
 	targetSource *gnmicv1alpha1.TargetSource
@@ -21,9 +21,9 @@ type TargetManager struct {
 	collected    map[string][]core.DiscoveredTarget
 }
 
-// NewTargetManager wires a TargetManager instance
-func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager {
-	return &TargetManager{
+// NewTargetApplier wires a TargetApplier instance
+func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier {
+	return &TargetApplier{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
@@ -34,16 +34,16 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (m *TargetManager) Run(ctx context.Context) error {
+func (m *TargetApplier) Run(ctx context.Context) error {
 	logger := log.FromContext(ctx).
 		WithValues("targetSource", m.targetSource)
 
-	logger.Info("target manager started")
+	logger.Info("target applier started")
 
 	for {
 		select {
 		case <-ctx.Done():
-			logger.Info("target manager stopped")
+			logger.Info("target applier stopped")
 			return nil
 
 		case messages := <-m.in:
@@ -83,7 +83,7 @@ func (m *TargetManager) Run(ctx context.Context) error {
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) {
+func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) {
 	targets := m.collected[snapshotID]
 	delete(m.collected, snapshotID)
 
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index c714acc..78d64d0 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -176,8 +176,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta
 	// Start loader
 	go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel)
 
-	// Start target manager
-	manager := discovery.NewTargetManager(
+	// Start target applier
+	manager := discovery.NewTargetApplier(
 		r.Client,
 		r.Scheme,
 		targetSource,

From 733927fa680c2896c83ee2863f7d2c2b24575448 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 07:51:37 +0000
Subject: [PATCH 023/165] implement key for registry as a comparable

---
 cmd/main.go                                   |  3 +-
 internal/apiserver/apiserver.go               |  3 +-
 .../controller/discovery/registry/registry.go | 39 +++++--------------
 .../controller/targetsource_controller.go     | 13 +++----
 4 files changed, 19 insertions(+), 39 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index 5cf8169..e4bad31 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -28,6 +28,7 @@ import (
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -85,7 +86,7 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]()
+	discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go
index b84eb9a..17e5c82 100644
--- a/internal/apiserver/apiserver.go
+++ b/internal/apiserver/apiserver.go
@@ -7,13 +7,14 @@ import (
 	"github.com/gnmic/operator/internal/controller"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 type APIServer struct {
 	Server            *http.Server
 	clusterReconciler *controller.ClusterReconciler
 
-	DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage]
+	DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage]
 }
 
 func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer {
diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go
index 7da0757..1892d28e 100644
--- a/internal/controller/discovery/registry/registry.go
+++ b/internal/controller/discovery/registry/registry.go
@@ -5,39 +5,18 @@ import (
 	"sync"
 )
 
-/* USAGE
-
-// create registry once in main.go
-discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]()
-
-// inside targetsource controller, when starting discovery pipeline:
-key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName)
-if err := discoveryReg.Register(key, out); err != nil {
-  logger.Error(err, "could not register loader")
-  return err
-}
-defer discoveryReg.Unregister(key)
-
-// CHECK REGISTRY
-ch, ok := discoveryReg.Get(ns + "/" + ts)
-if !ok {
-  http.Error(w, "no loader for targetsource", http.StatusNotFound)
-  return
-}
-// then deliver payload to ch
-*/
-
-// Registry is a thread-safe map: key -> channel of T.
-type Registry[T any] struct {
+// Registry is a thread-safe key -> channel registry
+// K must be comparable so it can be used as a map key
+type Registry[K comparable, V any] struct {
 	mu sync.RWMutex
-	m  map[string]chan<- T
+	m  map[K]chan<- V
 }
 
-func NewRegistry[T any]() *Registry[T] {
-	return &Registry[T]{m: make(map[string]chan<- T)}
+func NewRegistry[K comparable, V any]() *Registry[K, V] {
+	return &Registry[K, V]{m: make(map[K]chan<- V)}
 }
 
-func (r *Registry[T]) Register(key string, ch chan<- T) error {
+func (r *Registry[K, V]) Register(key K, ch chan<- V) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if _, exists := r.m[key]; exists {
@@ -47,13 +26,13 @@ func (r *Registry[T]) Register(key string, ch chan<- T) error {
 	return nil
 }
 
-func (r *Registry[T]) Unregister(key string) {
+func (r *Registry[K, V]) Unregister(key K) {
 	r.mu.Lock()
 	delete(r.m, key)
 	r.mu.Unlock()
 }
 
-func (r *Registry[T]) Get(key string) (chan<- T, bool) {
+func (r *Registry[K, V]) Get(key K) (chan<- V, bool) {
 	r.mu.RLock()
 	ch, ok := r.m[key]
 	r.mu.RUnlock()
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 78d64d0..d97b3a6 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -50,7 +51,7 @@ type TargetSourceReconciler struct {
 	BufferSize int
 	ChunkSize  int
 
-	DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage]
+	DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -149,7 +150,7 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool {
 }
 
 // startDiscoveryPipeline creates and starts the loader and target manager
-func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error {
+func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error {
 	cfg := core.LoaderConfig{
 		ChunkSize: r.ChunkSize,
 	}
@@ -167,8 +168,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta
 	runtimeCtx, cancel := context.WithCancel(context.Background())
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 
-	registryKey := key.Namespace + "/" + key.Name
-	if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil {
+	if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil {
 		cancel()
 		return err
 	}
@@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta
 
 // stopDiscovery stops and removes a running discovery pipeline
 // for the given TargetSource key
-func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) {
+func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 	r.mu.Lock()
 	running, ok := r.running[key]
 	if ok {
@@ -204,8 +204,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) {
 	r.mu.Unlock()
 
 	if ok {
-		registryKey := key.Namespace + "/" + key.Name
-		r.DiscoveryRegistry.Unregister(registryKey)
+		r.DiscoveryRegistry.Unregister(key)
 	}
 }
 

From 9d305601d18ae0f9f4d9f0168ec799b15e8b4a2a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 08:10:41 +0000
Subject: [PATCH 024/165] fix error message and add a word of caution for key
 comparables

---
 internal/controller/discovery/registry/registry.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go
index 1892d28e..093bd2c 100644
--- a/internal/controller/discovery/registry/registry.go
+++ b/internal/controller/discovery/registry/registry.go
@@ -7,6 +7,7 @@ import (
 
 // Registry is a thread-safe key -> channel registry
 // K must be comparable so it can be used as a map key
+// DO NOT USE a pointer type as K
 type Registry[K comparable, V any] struct {
 	mu sync.RWMutex
 	m  map[K]chan<- V
@@ -20,7 +21,7 @@ func (r *Registry[K, V]) Register(key K, ch chan<- V) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if _, exists := r.m[key]; exists {
-		return fmt.Errorf("already registered: %s", key)
+		return fmt.Errorf("already registered: %v", key)
 	}
 	r.m[key] = ch
 	return nil

From dafa82bb1fd1fbbb5369d14ff82594be38b19ddb Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 08:11:17 +0000
Subject: [PATCH 025/165] consistently use namespaced name as refference to the
 targetsource

---
 .../discovery/core/loader_interface.go          |  3 ++-
 internal/controller/discovery/loader.go         |  8 ++++----
 .../controller/discovery/loaders/http/loader.go | 11 ++++++-----
 internal/controller/targetsource_controller.go  | 17 ++++++++---------
 4 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go
index 17cd5f4..8964be8 100644
--- a/internal/controller/discovery/core/loader_interface.go
+++ b/internal/controller/discovery/core/loader_interface.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // Loader defines a pluggable TargetSource loader interface
@@ -16,7 +17,7 @@ type Loader interface {
 	// The loader must stop cleanly when ctx is cancelled
 	Start(
 		ctx context.Context,
-		targetsourceName string,
+		targetsourceName types.NamespacedName,
 		spec gnmicv1alpha1.TargetSourceSpec,
 		out chan<- []DiscoveryMessage,
 	) error
diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go
index 42ce8da..0d8ddd3 100644
--- a/internal/controller/discovery/loader.go
+++ b/internal/controller/discovery/loader.go
@@ -6,19 +6,19 @@ import (
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	http "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // NewLoader creates a loader by name
-func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) {
-	loaderName := namespace + "/" + name
+func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) {
 
 	switch {
 	case spec.Provider.HTTP != nil:
 		return http.New(cfg), nil
 	case spec.Provider.Consul != nil:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name)
 	default:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name)
 	}
 
 }
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index f014a2f..09bb7d6 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
@@ -27,14 +28,14 @@ func (l *Loader) Name() string {
 
 func (l *Loader) Start(
 	ctx context.Context,
-	targetsourceName string,
+	targetsourceNN types.NamespacedName,
 	spec gnmicv1alpha1.TargetSourceSpec,
 	out chan<- []core.DiscoveryMessage,
 ) error {
 	logger := log.FromContext(ctx).WithValues(
 		"component", "loader",
 		"name", l.Name(),
-		"targetsource", targetsourceName,
+		"targetsource", targetsourceNN,
 	)
 
 	logger.Info("HTTP loader started")
@@ -51,17 +52,17 @@ func (l *Loader) Start(
 
 		case <-ticker.C:
 			// Example snapshot (placeholder)
-			snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString())
+			snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString())
 			targets := []core.DiscoveredTarget{
 				{
 					Name:    "ceos1",
 					Address: "clab-3-nodes-ceos1:6030",
-					Labels:  map[string]string{"TargetSource": targetsourceName},
+					Labels:  map[string]string{"TargetSource": targetsourceNN.String()},
 				},
 				{
 					Name:    "leaf1",
 					Address: "clab-3-nodes-leaf1:57400",
-					Labels:  map[string]string{"TargetSource": targetsourceName},
+					Labels:  map[string]string{"TargetSource": targetsourceNN.String()},
 				},
 			}
 
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index d97b3a6..62b057d 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -46,7 +46,7 @@ type TargetSourceReconciler struct {
 	Scheme *runtime.Scheme
 
 	mu      sync.Mutex
-	running map[client.ObjectKey]runningSource
+	running map[types.NamespacedName]runningSource
 
 	BufferSize int
 	ChunkSize  int
@@ -96,7 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 }
 
 // getTargetSource retrieves a TargetSource by name, handling cleanup if not found
-func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) {
+func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) {
 	var targetSource gnmicv1alpha1.TargetSource
 	if err := r.Get(ctx, key, &targetSource); err != nil {
 		// If the TargetSource no longer exists, ensure runtime cleanup
@@ -109,9 +109,9 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client
 }
 
 // handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer
-func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
+func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
-	logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name)
+	logger.Info("TargetSource is being deleted, stopping pipeline", "name", key)
 
 	r.stopDiscovery(key)
 
@@ -141,7 +141,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 }
 
 // isPipelineRunning checks if a discovery pipeline is already running for the given key
-func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool {
+func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
@@ -156,8 +156,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	loader, err := discovery.NewLoader(
-		targetSource.ObjectMeta.Name,
-		targetSource.ObjectMeta.Namespace,
+		key,
 		targetSource.Spec,
 		cfg,
 	)
@@ -174,7 +173,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Start loader
-	go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel)
+	go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel)
 
 	// Start target applier
 	manager := discovery.NewTargetApplier(
@@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.running = make(map[client.ObjectKey]runningSource)
+	r.running = make(map[types.NamespacedName]runningSource)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&gnmicv1alpha1.TargetSource{}).

From 2973c03a665beeb3b53ef7ff71d55921c21053e1 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 09:05:21 +0000
Subject: [PATCH 026/165] improve context cancling and error handling

---
 .../controller/discovery/target_applier.go    | 102 ++++++++++++------
 1 file changed, 68 insertions(+), 34 deletions(-)

diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go
index 3babebf..7fed5c9 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_applier.go
@@ -37,49 +37,83 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 func (m *TargetApplier) Run(ctx context.Context) error {
 	logger := log.FromContext(ctx).
 		WithValues("targetSource", m.targetSource)
-
 	logger.Info("target applier started")
 
-	for {
+	queue := make([]core.DiscoveryMessage, 0, 265)
+
+	for ctx.Err() == nil {
 		select {
+		case batch, ok := <-m.in:
+			if !ok {
+				// Channel closed, pipeline is shutting down
+				logger.Info("input channel closed, stopping target applier")
+				return nil
+			}
+			queue = append(queue, batch...)
+
 		case <-ctx.Done():
-			logger.Info("target applier stopped")
+			logger.Info("context canceled, stopping target applier")
 			return nil
+		}
 
-		case messages := <-m.in:
-			for _, message := range messages {
-				// Type assert to determine if this is a snapshot or event
-				switch msg := message.(type) {
-				case core.DiscoverySnapshot:
-					// Collect snapshot chunks
-					logger.Info(
-						"received snapshot chunk",
-						"snapshotID", msg.SnapshotID,
-						"targetCount", len(msg.Targets),
-					)
-					m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...)
-					if msg.IsLastChunk {
-						m.processSnapshot(msg.SnapshotID, logger)
-					}
-
-				case core.DiscoveryEvent:
-					// Process individual event-driven update
-					logger.Info(
-						"received discovery event",
-						"target", msg.Target.Name,
-					)
-					switch msg.Event {
-					case core.CREATE:
-						logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
-					case core.UPDATE:
-						logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
-					case core.DELETE:
-						logger.Info("Would delete target", "name", msg.Target.Name)
-					}
-				}
+		for len(queue) > 0 {
+			if ctx.Err() != nil {
+				break
 			}
+
+			msg := queue[0]
+			queue = queue[1:]
+
+			if err := m.handleMessage(ctx, msg, logger); err != nil {
+				// Returning error lets the supervisor (controller)
+				// tear down and restart the pipeline via reconciliation
+				// Q: when to return an error vs just log and continue?
+				return err
+			}
+
 		}
 	}
+
+	logger.Info("target applier stopped")
+	return nil
+}
+
+func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	// Type assert to determine if this is a snapshot or event
+	switch msg := message.(type) {
+	case core.DiscoverySnapshot:
+		// Collect snapshot chunks
+		logger.Info(
+			"received snapshot chunk",
+			"snapshotID", msg.SnapshotID,
+			"targetCount", len(msg.Targets),
+		)
+		m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...)
+		if msg.IsLastChunk {
+			m.processSnapshot(msg.SnapshotID, logger)
+		}
+
+	case core.DiscoveryEvent:
+		// Process individual event-driven update
+		logger.Info(
+			"received discovery event",
+			"target", msg.Target.Name,
+		)
+		switch msg.Event {
+		case core.CREATE:
+			logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
+		case core.UPDATE:
+			logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
+		case core.DELETE:
+			logger.Info("Would delete target", "name", msg.Target.Name)
+		}
+	}
+
+	return nil
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly

From c95bdaf389038386a0b0b98759c98d4c10cb3f31 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 10:01:31 +0000
Subject: [PATCH 027/165] add supervised goroutines

---
 .../controller/targetsource_controller.go     | 54 +++++++++++++++++--
 1 file changed, 51 insertions(+), 3 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 62b057d..9fad373 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -18,6 +18,7 @@ package controller
 
 import (
 	"context"
+	"fmt"
 	"sync"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -172,17 +173,45 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		return err
 	}
 
+	// goroutines use done channel to report termination (nil or error) back to supervisor
+	// Buffer size = supervised goroutines = 2 (loader + applier)
+	done := make(chan error, 2)
+
 	// Start loader
-	go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel)
+	go runWithRecovery(
+		runtimeCtx,
+		"loader",
+		func(ctx context.Context) error {
+			return loader.Start(ctx, key, targetSource.Spec, targetChannel)
+		},
+		done,
+	)
 
 	// Start target applier
-	manager := discovery.NewTargetApplier(
+	applier := discovery.NewTargetApplier(
 		r.Client,
 		r.Scheme,
 		targetSource,
 		targetChannel,
 	)
-	go manager.Run(runtimeCtx)
+	go runWithRecovery(
+		runtimeCtx,
+		"target-applier",
+		applier.Run,
+		done,
+	)
+
+	// Supervision goroutine to handle pipeline termination
+	go func() {
+		err := <-done
+		logger := log.FromContext(context.Background()).WithValues("targetSource", key)
+		if err != nil {
+			logger.Error(err, "Discovery pipeline terminated with error")
+		}
+
+		// Ensure cleanup on termination
+		r.stopDiscovery(key)
+	}()
 
 	r.mu.Lock()
 	r.running[key] = runningSource{cancel: cancel}
@@ -207,6 +236,25 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 	}
 }
 
+// runWithRecovery executes a worker function under panic protection
+// and reports termination (nil or error) through done.
+func runWithRecovery(
+	ctx context.Context,
+	name string,
+	run func(context.Context) error,
+	done chan<- error,
+) {
+	defer func() {
+		if r := recover(); r != nil {
+			done <- fmt.Errorf("panic in %s: %v", name, r)
+		}
+	}()
+
+	// Normal exit path
+	err := run(ctx)
+	done <- err
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.running = make(map[types.NamespacedName]runningSource)

From 0aa883d98c940ebf374c6b9492522e63a601ac6d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 12:52:08 +0000
Subject: [PATCH 028/165] refactor target applier

---
 internal/controller/discovery/target_applier.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go
index 7fed5c9..c60f2b8 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_applier.go
@@ -36,10 +36,14 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 // and reconciles Target CRs accordingly
 func (m *TargetApplier) Run(ctx context.Context) error {
 	logger := log.FromContext(ctx).
-		WithValues("targetSource", m.targetSource)
+		WithValues(
+			"name", m.targetSource.Name,
+			"namespace", m.targetSource.Namespace,
+		)
+
 	logger.Info("target applier started")
 
-	queue := make([]core.DiscoveryMessage, 0, 265)
+	queue := []core.DiscoveryMessage{}
 
 	for ctx.Err() == nil {
 		select {
@@ -58,7 +62,7 @@ func (m *TargetApplier) Run(ctx context.Context) error {
 
 		for len(queue) > 0 {
 			if ctx.Err() != nil {
-				break
+				return ctx.Err()
 			}
 
 			msg := queue[0]

From 27b2b1f711a4f60edd2609d8e2822adbfaf07991 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 14:15:34 +0000
Subject: [PATCH 029/165] add supervisor for the discovery pipelines

---
 internal/controller/discovery/supervisor.go   | 123 ++++++++++++++++++
 .../controller/targetsource_controller.go     | 117 +++++++----------
 2 files changed, 171 insertions(+), 69 deletions(-)
 create mode 100644 internal/controller/discovery/supervisor.go

diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
new file mode 100644
index 0000000..ff19604
--- /dev/null
+++ b/internal/controller/discovery/supervisor.go
@@ -0,0 +1,123 @@
+package discovery
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+type ComponentExit struct {
+	Name string
+	Err  error
+}
+
+type RestartPolicy struct {
+	MaxRestarts int
+	Backoff     time.Duration
+}
+
+type Supervisor struct {
+	ctx      context.Context
+	cancel   context.CancelFunc
+	policy   RestartPolicy
+	failures int
+	exits    chan ComponentExit
+	wg       sync.WaitGroup
+	stopped  bool
+	stopMu   sync.Mutex
+}
+
+func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor {
+	ctx, cancel := context.WithCancel(parentCtx)
+	return &Supervisor{
+		ctx:      ctx,
+		cancel:   cancel,
+		policy:   policy,
+		exits:    make(chan ComponentExit, 4),
+		failures: 0,
+	}
+}
+
+func (s *Supervisor) Context() context.Context {
+	return s.ctx
+}
+
+func (s *Supervisor) Stop() {
+	s.stopMu.Lock()
+	defer s.stopMu.Unlock()
+
+	if s.stopped {
+		return
+	}
+
+	s.stopped = true
+	s.cancel()
+}
+
+func (s *Supervisor) Run(
+	start func(ctx context.Context, exits chan<- ComponentExit),
+) error {
+	logger := log.FromContext(s.ctx).WithName("discovery-supervisor")
+
+	for {
+		if s.failures > 0 {
+			logger.Info("Restarting pipeline",
+				"attempt", s.failures,
+				"maxAttempts", s.policy.MaxRestarts,
+			)
+
+			runtimeCtx, cancel := context.WithCancel(s.ctx)
+			s.wg = sync.WaitGroup{}
+			start(runtimeCtx, s.exits)
+			exit := <-s.exits // first failure wins
+
+			logger.Error(exit.Err,
+				"Pipeline component crashed",
+				"component", exit.Name,
+			)
+
+			cancel()
+			s.wg.Wait()
+
+			s.failures++
+			if s.failures >= s.policy.MaxRestarts {
+				logger.Error(exit.Err,
+					"Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart",
+					"restarts", s.failures,
+				)
+				s.Stop()
+				return exit.Err
+			}
+
+			select {
+			case <-time.After(s.policy.Backoff):
+				// continue to restart
+			case <-s.ctx.Done():
+				// Supervisor context canceled during backoff
+				return s.ctx.Err()
+			}
+		}
+	}
+}
+
+func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) {
+	s.wg.Add(1)
+
+	go func() {
+		defer s.wg.Done()
+
+		err := fn(s.ctx)
+		if err == nil {
+			err = context.Canceled // treat normal exit as cancellation
+		}
+
+		select {
+		case s.exits <- ComponentExit{Name: name, Err: err}:
+			// exit reported successfully
+		case <-s.ctx.Done():
+			// Supervisor context canceled before reporting exit
+		}
+	}()
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 9fad373..5d83db9 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -18,8 +18,8 @@ package controller
 
 import (
 	"context"
-	"fmt"
 	"sync"
+	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -33,9 +33,14 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/all"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
+	"github.com/go-logr/logr"
 )
 
-const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
+const (
+	targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
+	pipelineMaxRestarts   = 5
+	pipelineBackoff       = 3 * time.Second
+)
 
 type runningSource struct {
 	cancel context.CancelFunc
@@ -63,8 +68,8 @@ type TargetSourceReconciler struct {
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := log.FromContext(ctx).WithValues(
-		"Name", req.NamespacedName,
+	logger := log.FromContext(ctx).WithName("targetsource controller").WithValues(
+		"targetsource", req.NamespacedName,
 	)
 
 	targetSource, err := r.getTargetSource(ctx, req.NamespacedName)
@@ -88,7 +93,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	// Start discovery pipeline
-	if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil {
+	if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -151,70 +156,63 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo
 }
 
 // startDiscoveryPipeline creates and starts the loader and target manager
-func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error {
-	cfg := core.LoaderConfig{
-		ChunkSize: r.ChunkSize,
-	}
-
-	loader, err := discovery.NewLoader(
-		key,
-		targetSource.Spec,
-		cfg,
+func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
+	supervisor := discovery.NewSupervisor(
+		context.Background(),
+		discovery.RestartPolicy{
+			MaxRestarts: pipelineMaxRestarts,
+			Backoff:     pipelineBackoff,
+		},
 	)
-	if err != nil {
-		return err
-	}
 
-	runtimeCtx, cancel := context.WithCancel(context.Background())
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
-
 	if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil {
-		cancel()
 		return err
 	}
 
-	// goroutines use done channel to report termination (nil or error) back to supervisor
-	// Buffer size = supervised goroutines = 2 (loader + applier)
-	done := make(chan error, 2)
+	start := func(ctx context.Context, exits chan<- discovery.ComponentExit) {
+		// Create loader instance
+		loader, err := discovery.NewLoader(
+			key,
+			targetSource.Spec,
+			core.LoaderConfig{
+				ChunkSize: r.ChunkSize,
+			},
+		)
+		if err != nil {
+			return
+		}
+
+		// Create target applier instance
+		applier := discovery.NewTargetApplier(
+			r.Client,
+			r.Scheme,
+			targetSource,
+			targetChannel,
+		)
 
-	// Start loader
-	go runWithRecovery(
-		runtimeCtx,
-		"loader",
-		func(ctx context.Context) error {
+		// Start loader
+		supervisor.Go("loader", func(ctx context.Context) error {
 			return loader.Start(ctx, key, targetSource.Spec, targetChannel)
-		},
-		done,
-	)
+		})
+		// Start target applier
+		supervisor.Go("target-applier", applier.Run)
 
-	// Start target applier
-	applier := discovery.NewTargetApplier(
-		r.Client,
-		r.Scheme,
-		targetSource,
-		targetChannel,
-	)
-	go runWithRecovery(
-		runtimeCtx,
-		"target-applier",
-		applier.Run,
-		done,
-	)
+	}
 
-	// Supervision goroutine to handle pipeline termination
 	go func() {
-		err := <-done
-		logger := log.FromContext(context.Background()).WithValues("targetSource", key)
+		err := supervisor.Run(start)
 		if err != nil {
-			logger.Error(err, "Discovery pipeline terminated with error")
+			logger.Error(err, "Discovery pipeline stopped permanently")
 		}
 
-		// Ensure cleanup on termination
+		close(targetChannel)
+		r.DiscoveryRegistry.Unregister(key)
 		r.stopDiscovery(key)
 	}()
 
 	r.mu.Lock()
-	r.running[key] = runningSource{cancel: cancel}
+	r.running[key] = runningSource{cancel: supervisor.Stop}
 	r.mu.Unlock()
 
 	return nil
@@ -236,25 +234,6 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 	}
 }
 
-// runWithRecovery executes a worker function under panic protection
-// and reports termination (nil or error) through done.
-func runWithRecovery(
-	ctx context.Context,
-	name string,
-	run func(context.Context) error,
-	done chan<- error,
-) {
-	defer func() {
-		if r := recover(); r != nil {
-			done <- fmt.Errorf("panic in %s: %v", name, r)
-		}
-	}()
-
-	// Normal exit path
-	err := run(ctx)
-	done <- err
-}
-
 // SetupWithManager sets up the controller with the Manager.
 func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.running = make(map[types.NamespacedName]runningSource)

From 22fe2d894e2109c817a11b3153f298ba0fb8eb06 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 14:55:21 +0000
Subject: [PATCH 030/165] improve readability

---
 internal/controller/targetsource_controller.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 5d83db9..db60520 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -38,8 +38,9 @@ import (
 
 const (
 	targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
-	pipelineMaxRestarts   = 5
-	pipelineBackoff       = 3 * time.Second
+
+	pipelineMaxRestarts = 5
+	pipelineBackoff     = 3 * time.Second
 )
 
 type runningSource struct {

From 58538c76c0583e031b56031f72e639450c918910 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 15:02:48 +0000
Subject: [PATCH 031/165] remove side-effects from getter getTargetSource

---
 internal/controller/targetsource_controller.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index db60520..33342b4 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -75,6 +75,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 
 	targetSource, err := r.getTargetSource(ctx, req.NamespacedName)
 	if err != nil {
+		// If the TargetSource no longer exists, ensure runtime cleanup
+		if client.IgnoreNotFound(err) == nil {
+			logger.Info("TargetSource not found, ensuring cleanup")
+			r.stopDiscovery(req.NamespacedName)
+			return ctrl.Result{}, nil
+		}
 		return ctrl.Result{}, err
 	}
 
@@ -106,11 +112,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) {
 	var targetSource gnmicv1alpha1.TargetSource
 	if err := r.Get(ctx, key, &targetSource); err != nil {
-		// If the TargetSource no longer exists, ensure runtime cleanup
-		if client.IgnoreNotFound(err) == nil {
-			r.stopDiscovery(key)
-		}
-		return nil, client.IgnoreNotFound(err)
+		return nil, err
 	}
 	return &targetSource, nil
 }

From 4f0457ec86f4ed5df64a4216aadc8e3fc3551391 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 24 Apr 2026 17:38:41 +0000
Subject: [PATCH 032/165] redesign supervisor

---
 internal/controller/discovery/supervisor.go   | 145 ++++++++----------
 .../controller/targetsource_controller.go     |  85 +++++-----
 2 files changed, 106 insertions(+), 124 deletions(-)

diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
index ff19604..c716965 100644
--- a/internal/controller/discovery/supervisor.go
+++ b/internal/controller/discovery/supervisor.go
@@ -8,116 +8,93 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-type ComponentExit struct {
-	Name string
-	Err  error
-}
-
 type RestartPolicy struct {
 	MaxRestarts int
 	Backoff     time.Duration
 }
 
+type Component struct {
+	Name   string
+	Run    func(ctx context.Context) error
+	Policy RestartPolicy
+}
+
 type Supervisor struct {
-	ctx      context.Context
-	cancel   context.CancelFunc
-	policy   RestartPolicy
-	failures int
-	exits    chan ComponentExit
-	wg       sync.WaitGroup
-	stopped  bool
-	stopMu   sync.Mutex
+	ctx    context.Context
+	cancel context.CancelFunc
+
+	stopped bool
+	mu      sync.Mutex
+
+	components []Component
 }
 
-func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor {
-	ctx, cancel := context.WithCancel(parentCtx)
+func NewSupervisor(parent context.Context) *Supervisor {
+	ctx, cancel := context.WithCancel(parent)
 	return &Supervisor{
-		ctx:      ctx,
-		cancel:   cancel,
-		policy:   policy,
-		exits:    make(chan ComponentExit, 4),
-		failures: 0,
+		ctx:    ctx,
+		cancel: cancel,
 	}
 }
 
-func (s *Supervisor) Context() context.Context {
-	return s.ctx
+func (s *Supervisor) AddComponent(c Component) {
+	s.components = append(s.components, c)
 }
 
-func (s *Supervisor) Stop() {
-	s.stopMu.Lock()
-	defer s.stopMu.Unlock()
+func (s *Supervisor) runComponent(c Component) {
+	logger := log.FromContext(s.ctx).WithValues(
+		"component", c.Name,
+	)
 
-	if s.stopped {
-		return
-	}
-
-	s.stopped = true
-	s.cancel()
-}
-
-func (s *Supervisor) Run(
-	start func(ctx context.Context, exits chan<- ComponentExit),
-) error {
-	logger := log.FromContext(s.ctx).WithName("discovery-supervisor")
+	failures := 0
 
 	for {
-		if s.failures > 0 {
-			logger.Info("Restarting pipeline",
-				"attempt", s.failures,
-				"maxAttempts", s.policy.MaxRestarts,
-			)
+		err := c.Run(s.ctx)
+		if s.ctx.Err() != nil {
+			return
+		}
 
-			runtimeCtx, cancel := context.WithCancel(s.ctx)
-			s.wg = sync.WaitGroup{}
-			start(runtimeCtx, s.exits)
-			exit := <-s.exits // first failure wins
+		failures++
+		logger.Error(err,
+			"Component failed",
+			"attempt", failures,
+		)
 
-			logger.Error(exit.Err,
-				"Pipeline component crashed",
-				"component", exit.Name,
+		if failures >= c.Policy.MaxRestarts {
+			logger.Error(err,
+				"Component exceeded restart limit; stopping discovery pipeline",
+				"restarts", failures,
 			)
+			s.Stop()
+			return
+		}
 
-			cancel()
-			s.wg.Wait()
-
-			s.failures++
-			if s.failures >= s.policy.MaxRestarts {
-				logger.Error(exit.Err,
-					"Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart",
-					"restarts", s.failures,
-				)
-				s.Stop()
-				return exit.Err
-			}
-
-			select {
-			case <-time.After(s.policy.Backoff):
-				// continue to restart
-			case <-s.ctx.Done():
-				// Supervisor context canceled during backoff
-				return s.ctx.Err()
-			}
+		select {
+		case <-time.After(c.Policy.Backoff):
+		case <-s.ctx.Done():
+			return
 		}
 	}
 }
 
-func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) {
-	s.wg.Add(1)
+func (s *Supervisor) Run() {
+	for _, c := range s.components {
+		component := c
+		go s.runComponent(component)
+	}
+}
 
-	go func() {
-		defer s.wg.Done()
+func (s *Supervisor) Stop() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 
-		err := fn(s.ctx)
-		if err == nil {
-			err = context.Canceled // treat normal exit as cancellation
-		}
+	if s.stopped {
+		return
+	}
+	s.stopped = true
+	s.cancel()
+}
 
-		select {
-		case s.exits <- ComponentExit{Name: name, Err: err}:
-			// exit reported successfully
-		case <-s.ctx.Done():
-			// Supervisor context canceled before reporting exit
-		}
-	}()
+func (s *Supervisor) Done() <-chan struct{} {
+	return s.ctx.Done()
 }
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 33342b4..68f47eb 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -160,54 +160,57 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo
 
 // startDiscoveryPipeline creates and starts the loader and target manager
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
-	supervisor := discovery.NewSupervisor(
-		context.Background(),
-		discovery.RestartPolicy{
-			MaxRestarts: pipelineMaxRestarts,
-			Backoff:     pipelineBackoff,
-		},
-	)
+	supervisor := discovery.NewSupervisor(context.Background())
 
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 	if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil {
 		return err
 	}
 
-	start := func(ctx context.Context, exits chan<- discovery.ComponentExit) {
-		// Create loader instance
-		loader, err := discovery.NewLoader(
-			key,
-			targetSource.Spec,
-			core.LoaderConfig{
-				ChunkSize: r.ChunkSize,
-			},
-		)
-		if err != nil {
-			return
-		}
+	// Create loader instance
+	loader, err := discovery.NewLoader(
+		key,
+		targetSource.Spec,
+		core.LoaderConfig{ChunkSize: r.ChunkSize},
+	)
+	if err != nil {
+		return err
+	}
 
-		// Create target applier instance
-		applier := discovery.NewTargetApplier(
-			r.Client,
-			r.Scheme,
-			targetSource,
-			targetChannel,
-		)
+	// Create target applier instance
+	applier := discovery.NewTargetApplier(
+		r.Client,
+		r.Scheme,
+		targetSource,
+		targetChannel,
+	)
 
-		// Start loader
-		supervisor.Go("loader", func(ctx context.Context) error {
+	supervisor.AddComponent(discovery.Component{
+		Name: "loader",
+		Run: func(ctx context.Context) error {
 			return loader.Start(ctx, key, targetSource.Spec, targetChannel)
-		})
-		// Start target applier
-		supervisor.Go("target-applier", applier.Run)
+		},
+		Policy: discovery.RestartPolicy{
+			MaxRestarts: pipelineMaxRestarts,
+			Backoff:     pipelineBackoff,
+		},
+	})
 
-	}
+	supervisor.AddComponent(discovery.Component{
+		Name: "target-applier",
+		Run:  applier.Run,
+		Policy: discovery.RestartPolicy{
+			MaxRestarts: pipelineMaxRestarts,
+			Backoff:     pipelineBackoff,
+		},
+	})
+
+	supervisor.Run()
 
 	go func() {
-		err := supervisor.Run(start)
-		if err != nil {
-			logger.Error(err, "Discovery pipeline stopped permanently")
-		}
+		<-supervisor.Done()
+
+		logger.Info("Pipeline stopped; performing final cleanup")
 
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
@@ -215,25 +218,27 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}()
 
 	r.mu.Lock()
-	r.running[key] = runningSource{cancel: supervisor.Stop}
+	r.running[key] = runningSource{
+		cancel: func() {
+			supervisor.Stop()
+		},
+	}
 	r.mu.Unlock()
 
 	return nil
 }
 
 // stopDiscovery stops and removes a running discovery pipeline
-// for the given TargetSource key
 func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 	r.mu.Lock()
 	running, ok := r.running[key]
 	if ok {
-		running.cancel()
 		delete(r.running, key)
 	}
 	r.mu.Unlock()
 
 	if ok {
-		r.DiscoveryRegistry.Unregister(key)
+		running.cancel()
 	}
 }
 

From 60491be6b980c081f46955c59a8dc995db26c2e0 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Sat, 25 Apr 2026 09:06:20 +0000
Subject: [PATCH 033/165] add dependency handling of discovery pipeline
 components

---
 api/v1alpha1/targetsource_types.go            |   7 ++
 api/v1alpha1/zz_generated.deepcopy.go         |  21 ++++
 .../operator.gnmic.dev_targetsources.yaml     |   5 +
 internal/controller/discovery/supervisor.go   | 112 +++++++++---------
 .../controller/targetsource_controller.go     |  67 +++++++----
 5 files changed, 133 insertions(+), 79 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index feea000..a936e66 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -24,6 +24,8 @@ import (
 // +kubebuilder:validation:Required
 type TargetSourceSpec struct {
 	Provider *ProviderSpec `json:"provider"`
+	// +kubebuilder:validation:Optional
+	Webhook WebhookSpec `json:"webhook,omitempty"`
 	//
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
@@ -37,6 +39,11 @@ type ProviderSpec struct {
 	Consul *ConsulConfig `json:"consul,omitempty"`
 }
 
+type WebhookSpec struct {
+	// +kubebuilder:validation:Optional
+	Enabled *bool `json:"enabled,omitempty"`
+}
+
 type HTTPConfig struct {
 	// +kubebuilder:validation:MinLength=1
 	URL string `json:"url"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 61e81fd..608d47e 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -1292,6 +1292,7 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) {
 		*out = new(ProviderSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	in.Webhook.DeepCopyInto(&out.Webhook)
 	if in.TargetLabels != nil {
 		in, out := &in.TargetLabels, &out.TargetLabels
 		*out = make(map[string]string, len(*in))
@@ -1477,3 +1478,23 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
+func (in *WebhookSpec) DeepCopy() *WebhookSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookSpec)
+	in.DeepCopyInto(out)
+	return out
+}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index f373822..b385d8e 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -67,6 +67,11 @@ spec:
               targetProfile:
                 minLength: 1
                 type: string
+              webhook:
+                properties:
+                  enabled:
+                    type: boolean
+                type: object
             required:
             - provider
             - targetProfile
diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
index c716965..128305a 100644
--- a/internal/controller/discovery/supervisor.go
+++ b/internal/controller/discovery/supervisor.go
@@ -14,19 +14,20 @@ type RestartPolicy struct {
 }
 
 type Component struct {
-	Name   string
-	Run    func(ctx context.Context) error
-	Policy RestartPolicy
+	Name             string
+	Run              func(ctx context.Context) error
+	Policy           RestartPolicy
+	DegradeOnFailure bool
 }
 
 type Supervisor struct {
 	ctx    context.Context
 	cancel context.CancelFunc
 
-	stopped bool
-	mu      sync.Mutex
+	wg sync.WaitGroup
 
-	components []Component
+	mu      sync.Mutex
+	stopped bool
 }
 
 func NewSupervisor(parent context.Context) *Supervisor {
@@ -37,53 +38,6 @@ func NewSupervisor(parent context.Context) *Supervisor {
 	}
 }
 
-func (s *Supervisor) AddComponent(c Component) {
-	s.components = append(s.components, c)
-}
-
-func (s *Supervisor) runComponent(c Component) {
-	logger := log.FromContext(s.ctx).WithValues(
-		"component", c.Name,
-	)
-
-	failures := 0
-
-	for {
-		err := c.Run(s.ctx)
-		if s.ctx.Err() != nil {
-			return
-		}
-
-		failures++
-		logger.Error(err,
-			"Component failed",
-			"attempt", failures,
-		)
-
-		if failures >= c.Policy.MaxRestarts {
-			logger.Error(err,
-				"Component exceeded restart limit; stopping discovery pipeline",
-				"restarts", failures,
-			)
-			s.Stop()
-			return
-		}
-
-		select {
-		case <-time.After(c.Policy.Backoff):
-		case <-s.ctx.Done():
-			return
-		}
-	}
-}
-
-func (s *Supervisor) Run() {
-	for _, c := range s.components {
-		component := c
-		go s.runComponent(component)
-	}
-}
-
 func (s *Supervisor) Stop() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -98,3 +52,55 @@ func (s *Supervisor) Stop() {
 func (s *Supervisor) Done() <-chan struct{} {
 	return s.ctx.Done()
 }
+
+func (s *Supervisor) Wait() {
+	s.wg.Wait()
+}
+
+func (s *Supervisor) RunComponent(component Component) {
+	s.wg.Add(1)
+
+	go func() {
+		defer s.wg.Done()
+
+		logger := log.FromContext(s.ctx).WithValues("component", component.Name)
+		failures := 0
+
+		for {
+			logger.Info("starting component")
+			err := component.Run(s.ctx)
+
+			if s.ctx.Err() != nil {
+				logger.Info("component stopped due to pipeline shutdown")
+				return
+			}
+
+			failures++
+			logger.Error(err,
+				"component failed to run",
+				"attempt", failures,
+				"max", component.Policy.MaxRestarts,
+			)
+
+			if failures >= component.Policy.MaxRestarts {
+				if component.DegradeOnFailure {
+					logger.Error(err,
+						"component permanently failed; shutting down pipeline",
+					)
+					s.Stop()
+				} else {
+					logger.Info(
+						"optional component permanently failed; continuing without it",
+					)
+				}
+				return
+			}
+
+			select {
+			case <-time.After(component.Policy.Backoff):
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}()
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 68f47eb..a687e80 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -167,16 +167,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		return err
 	}
 
-	// Create loader instance
-	loader, err := discovery.NewLoader(
-		key,
-		targetSource.Spec,
-		core.LoaderConfig{ChunkSize: r.ChunkSize},
-	)
-	if err != nil {
-		return err
-	}
-
 	// Create target applier instance
 	applier := discovery.NewTargetApplier(
 		r.Client,
@@ -184,34 +174,59 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		targetSource,
 		targetChannel,
 	)
-
-	supervisor.AddComponent(discovery.Component{
-		Name: "loader",
-		Run: func(ctx context.Context) error {
-			return loader.Start(ctx, key, targetSource.Spec, targetChannel)
-		},
-		Policy: discovery.RestartPolicy{
-			MaxRestarts: pipelineMaxRestarts,
-			Backoff:     pipelineBackoff,
-		},
-	})
-
-	supervisor.AddComponent(discovery.Component{
+	// Start target applier
+	applierReady := make(chan struct{})
+	supervisor.RunComponent(discovery.Component{
 		Name: "target-applier",
-		Run:  applier.Run,
 		Policy: discovery.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
+		DegradeOnFailure: true,
+		Run: func(ctx context.Context) error {
+			close(applierReady)
+			return applier.Run(ctx)
+		},
 	})
+	// Wait for applier to be ready before starting loader
+	select {
+	case <-applierReady:
+	case <-supervisor.Done():
+		return nil
+	}
 
-	supervisor.Run()
+	// Create loader instance
+	loaderConfigured := targetSource.Spec.Provider != nil
+	webhookConfigured := targetSource.Spec.Webhook.Enabled != nil
+	if loaderConfigured {
+		loader, err := discovery.NewLoader(
+			key,
+			targetSource.Spec,
+			core.LoaderConfig{ChunkSize: r.ChunkSize},
+		)
+		if err != nil {
+			supervisor.Stop()
+			return err
+		}
+
+		supervisor.RunComponent(discovery.Component{
+			Name: "loader",
+			Policy: discovery.RestartPolicy{
+				MaxRestarts: pipelineMaxRestarts,
+				Backoff:     pipelineBackoff,
+			},
+			DegradeOnFailure: !webhookConfigured,
+			Run: func(ctx context.Context) error {
+				return loader.Start(ctx, key, targetSource.Spec, targetChannel)
+			},
+		})
+	}
 
 	go func() {
 		<-supervisor.Done()
+		supervisor.Wait() // Wait for components to exit
 
 		logger.Info("Pipeline stopped; performing final cleanup")
-
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
 		r.stopDiscovery(key)

From b8a6d272d479a97f05b5adeb6f9081520a236f8e Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Sat, 25 Apr 2026 09:31:29 +0000
Subject: [PATCH 034/165] refactor code

---
 internal/controller/discovery/supervisor.go   | 54 +++++-----
 .../controller/targetsource_controller.go     | 99 ++++++++++---------
 2 files changed, 87 insertions(+), 66 deletions(-)

diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
index 128305a..710381e 100644
--- a/internal/controller/discovery/supervisor.go
+++ b/internal/controller/discovery/supervisor.go
@@ -8,18 +8,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
-type RestartPolicy struct {
-	MaxRestarts int
-	Backoff     time.Duration
-}
-
-type Component struct {
-	Name             string
-	Run              func(ctx context.Context) error
-	Policy           RestartPolicy
-	DegradeOnFailure bool
-}
-
+// Supervisor coordinates the runtime lifecycle of pipeline components
+//
+// Guarantees:
+// - Each component is restarted independently
+// - Permanent failure escalates according to policy
+// - Stop() cancels all components
+// - Wait() blocks until all goroutines exit
 type Supervisor struct {
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -30,14 +25,30 @@ type Supervisor struct {
 	stopped bool
 }
 
-func NewSupervisor(parent context.Context) *Supervisor {
-	ctx, cancel := context.WithCancel(parent)
+// RestartPolicy defines the restart behavior for a component
+type RestartPolicy struct {
+	MaxRestarts int
+	Backoff     time.Duration
+}
+
+type ComponentSpec struct {
+	Name   string
+	Run    func(ctx context.Context) error
+	Policy RestartPolicy
+	// EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline
+	EscalatesOnFailure bool
+}
+
+// NewSupervisor creates a new Supervisor with a cancellable context
+func NewSupervisor(parentCtx context.Context) *Supervisor {
+	ctx, cancel := context.WithCancel(parentCtx)
 	return &Supervisor{
 		ctx:    ctx,
 		cancel: cancel,
 	}
 }
 
+// Stop signals all supervised components to stop by canceling the context
 func (s *Supervisor) Stop() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -49,15 +60,14 @@ func (s *Supervisor) Stop() {
 	s.cancel()
 }
 
-func (s *Supervisor) Done() <-chan struct{} {
-	return s.ctx.Done()
-}
+// Done returns a channel that is closed when the pipeline is stopped
+func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() }
 
-func (s *Supervisor) Wait() {
-	s.wg.Wait()
-}
+// Wait blocks until all supervised components have exited
+func (s *Supervisor) Wait() { s.wg.Wait() }
 
-func (s *Supervisor) RunComponent(component Component) {
+// StartSupervisedComponent starts and supervises a component
+func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) {
 	s.wg.Add(1)
 
 	go func() {
@@ -83,7 +93,7 @@ func (s *Supervisor) RunComponent(component Component) {
 			)
 
 			if failures >= component.Policy.MaxRestarts {
-				if component.DegradeOnFailure {
+				if component.EscalatesOnFailure {
 					logger.Error(err,
 						"component permanently failed; shutting down pipeline",
 					)
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index a687e80..f04eced 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -43,17 +43,26 @@ const (
 	pipelineBackoff     = 3 * time.Second
 )
 
-type runningSource struct {
+// pipelineHandle represents a controller-owned handle to a running pipeline
+// The controller never manipulates internals; it only invokes cancel()
+type pipelineHandle struct {
 	cancel context.CancelFunc
 }
 
 // TargetSourceReconciler reconciles a TargetSource object
+//
+// Responsibilities:
+// - Ensure at most one pipeline per TargetSource
+// - Start pipelines on reconcile
+// - Stop pipelines on deletion or NotFound
+// - Delegate runtime failure handling to the Supervisor
 type TargetSourceReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 
-	mu      sync.Mutex
-	running map[types.NamespacedName]runningSource
+	mu sync.Mutex
+	// runningPipelines tracks currently active pipelines by NamespacedName
+	runningPipelines map[types.NamespacedName]pipelineHandle
 
 	BufferSize int
 	ChunkSize  int
@@ -69,47 +78,43 @@ type TargetSourceReconciler struct {
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := log.FromContext(ctx).WithName("targetsource controller").WithValues(
-		"targetsource", req.NamespacedName,
-	)
+	logger := log.FromContext(ctx).
+		WithName("targetsource controller").
+		WithValues("targetsource", req.NamespacedName)
 
-	targetSource, err := r.getTargetSource(ctx, req.NamespacedName)
+	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
 	if err != nil {
 		// If the TargetSource no longer exists, ensure runtime cleanup
 		if client.IgnoreNotFound(err) == nil {
-			logger.Info("TargetSource not found, ensuring cleanup")
-			r.stopDiscovery(req.NamespacedName)
+			logger.Info("TargetSource not found; stopping discovery pipeline")
+			r.stopDiscoveryPipeline(req.NamespacedName)
 			return ctrl.Result{}, nil
 		}
 		return ctrl.Result{}, err
 	}
 
-	// Handle deletion with finalizer
 	if !targetSource.DeletionTimestamp.IsZero() {
-		return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource)
+		return r.reconcileDeletion(ctx, req.NamespacedName, targetSource)
 	}
 
-	// Ensure finalizer is set
 	if err := r.ensureFinalizer(ctx, targetSource); err != nil {
 		return ctrl.Result{}, err
 	}
 
-	// Check if pipeline is already running
-	if r.isPipelineRunning(req.NamespacedName) {
+	if r.hasPipelineRunning(req.NamespacedName) {
 		return ctrl.Result{}, nil
 	}
 
-	// Start discovery pipeline
 	if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil {
 		return ctrl.Result{}, err
 	}
 
-	logger.Info("TargetSource pipeline started")
+	logger.Info("Discover pipeline started")
 	return ctrl.Result{}, nil
 }
 
-// getTargetSource retrieves a TargetSource by name, handling cleanup if not found
-func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) {
+// fetchTargetSource retrieves a TargetSource by name, handling cleanup if not found
+func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) {
 	var targetSource gnmicv1alpha1.TargetSource
 	if err := r.Get(ctx, key, &targetSource); err != nil {
 		return nil, err
@@ -117,12 +122,20 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.
 	return &targetSource, nil
 }
 
-// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer
-func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
+// hasPipelineRunning checks if a discovery pipeline is already running for the given key
+func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	_, exists := r.runningPipelines[key]
+	return exists
+}
+
+// reconcileDeletion stops the discovery pipeline and removes the finalizer
+func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 	logger.Info("TargetSource is being deleted, stopping pipeline", "name", key)
 
-	r.stopDiscovery(key)
+	r.stopDiscoveryPipeline(key)
 
 	// Remove finalizer if exists
 	if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) {
@@ -149,16 +162,13 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 	return nil
 }
 
-// isPipelineRunning checks if a discovery pipeline is already running for the given key
-func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	_, exists := r.running[key]
-	return exists
-}
-
-// startDiscoveryPipeline creates and starts the loader and target manager
+// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource
+//
+// Pipeline semantics:
+// 1. target-applier is mandatory and must start first
+// 2. loader is optional and conditional on spec
+// 3. Permanent failure of required components shuts down the pipeline
+// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
 	supervisor := discovery.NewSupervisor(context.Background())
 
@@ -176,15 +186,15 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	)
 	// Start target applier
 	applierReady := make(chan struct{})
-	supervisor.RunComponent(discovery.Component{
+	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
 		Name: "target-applier",
 		Policy: discovery.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
-		DegradeOnFailure: true,
+		EscalatesOnFailure: true,
 		Run: func(ctx context.Context) error {
-			close(applierReady)
+			close(applierReady) // Signals that applier started successfully
 			return applier.Run(ctx)
 		},
 	})
@@ -209,31 +219,32 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 			return err
 		}
 
-		supervisor.RunComponent(discovery.Component{
+		supervisor.StartSupervisedComponent(discovery.ComponentSpec{
 			Name: "loader",
 			Policy: discovery.RestartPolicy{
 				MaxRestarts: pipelineMaxRestarts,
 				Backoff:     pipelineBackoff,
 			},
-			DegradeOnFailure: !webhookConfigured,
+			EscalatesOnFailure: !webhookConfigured,
 			Run: func(ctx context.Context) error {
 				return loader.Start(ctx, key, targetSource.Spec, targetChannel)
 			},
 		})
 	}
 
+	// Monitor supervisor in a separate goroutine to handle shutdown and cleanup
 	go func() {
 		<-supervisor.Done()
 		supervisor.Wait() // Wait for components to exit
 
-		logger.Info("Pipeline stopped; performing final cleanup")
+		logger.Info("Pipeline stopped; cleaning up")
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
-		r.stopDiscovery(key)
+		r.stopDiscoveryPipeline(key)
 	}()
 
 	r.mu.Lock()
-	r.running[key] = runningSource{
+	r.runningPipelines[key] = pipelineHandle{
 		cancel: func() {
 			supervisor.Stop()
 		},
@@ -243,12 +254,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	return nil
 }
 
-// stopDiscovery stops and removes a running discovery pipeline
-func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
+// stopDiscoveryPipeline stops and removes a running discovery pipeline
+func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) {
 	r.mu.Lock()
-	running, ok := r.running[key]
+	running, ok := r.runningPipelines[key]
 	if ok {
-		delete(r.running, key)
+		delete(r.runningPipelines, key)
 	}
 	r.mu.Unlock()
 
@@ -259,7 +270,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) {
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.running = make(map[types.NamespacedName]runningSource)
+	r.runningPipelines = make(map[types.NamespacedName]pipelineHandle)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&gnmicv1alpha1.TargetSource{}).

From eedfedf930d6f78ef9ca430115bb121ee9db129c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Sat, 25 Apr 2026 12:55:52 +0000
Subject: [PATCH 035/165] improve context handling of and target applier
 semantics

---
 internal/controller/discovery/core/helpers.go |  14 +-
 internal/controller/discovery/core/types.go   |   5 +-
 .../controller/discovery/target_applier.go    | 209 ++++++++++++++----
 3 files changed, 184 insertions(+), 44 deletions(-)

diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/helpers.go
index 843f30e..f24b50c 100644
--- a/internal/controller/discovery/core/helpers.go
+++ b/internal/controller/discovery/core/helpers.go
@@ -2,6 +2,7 @@ package core
 
 import (
 	"context"
+	"fmt"
 )
 
 // sendMessages sends discovery messages over a channel in a context-aware manner
@@ -32,13 +33,15 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error {
 func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot {
 	var snapshots []DiscoverySnapshot
 	totalTargets := len(targets)
+	totalChunks := (totalTargets + chunkSize - 1) / chunkSize
 
 	_ = forEachChunk(totalTargets, chunkSize, func(i, end int) error {
 		chunk := targets[i:end]
 		snapshots = append(snapshots, DiscoverySnapshot{
 			Targets:     chunk,
 			SnapshotID:  snapshotID,
-			IsLastChunk: (end == totalTargets),
+			ChunkIndex:  i / chunkSize,
+			TotalChunks: totalChunks,
 		})
 		return nil
 	})
@@ -48,8 +51,11 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu
 
 // SendSnapshot sends discovered targets as a snapshot over a channel in chunks
 func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error {
-	snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize)
+	if len(targets) == 0 {
+		return fmt.Errorf("no targets in Snapshot")
+	}
 
+	snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize)
 	for _, snapshot := range snapshots {
 		// Convert DiscoverySnapshot to DiscoveryMessage
 		messages := make([]DiscoveryMessage, 1)
@@ -73,6 +79,10 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage {
 
 // SendEvents sends discovery messages over channel in a context-aware manner
 func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error {
+	if len(events) == 0 {
+		return fmt.Errorf("no events to process")
+	}
+
 	messages := eventsToMessages(events)
 	total := len(messages)
 
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 61209fd..3f6957a 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -26,7 +26,8 @@ type DiscoveryEvent struct {
 }
 
 type DiscoverySnapshot struct {
-	Targets     []DiscoveredTarget
 	SnapshotID  string
-	IsLastChunk bool
+	ChunkIndex  int
+	TotalChunks int
+	Targets     []DiscoveredTarget
 }
diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go
index c60f2b8..ee127c5 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_applier.go
@@ -2,6 +2,7 @@ package discovery
 
 import (
 	"context"
+	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -12,13 +13,23 @@ import (
 	"github.com/go-logr/logr"
 )
 
+type snapshotBuffer struct {
+	snapshotID  string
+	totalChunks int
+	received    map[int][]core.DiscoveredTarget
+	complete    bool
+}
+
 // TargetApplier consumes discovered targets and applies them to Kubernetes
 type TargetApplier struct {
-	client       client.Client
-	scheme       *runtime.Scheme
-	targetSource *gnmicv1alpha1.TargetSource
-	in           <-chan []core.DiscoveryMessage
-	collected    map[string][]core.DiscoveredTarget
+	client         client.Client
+	scheme         *runtime.Scheme
+	targetSource   *gnmicv1alpha1.TargetSource
+	in             <-chan []core.DiscoveryMessage
+	queue          []core.DiscoveryMessage
+	activeSnapshot *snapshotBuffer
+	// Events are deferred while snapshot is in progress
+	defferedEvents []core.DiscoveryEvent
 }
 
 // NewTargetApplier wires a TargetApplier instance
@@ -28,47 +39,43 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 		scheme:       s,
 		targetSource: ts,
 		in:           in,
-		collected:    make(map[string][]core.DiscoveredTarget),
 	}
 }
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (m *TargetApplier) Run(ctx context.Context) error {
+func (a *TargetApplier) Run(ctx context.Context) error {
 	logger := log.FromContext(ctx).
 		WithValues(
-			"name", m.targetSource.Name,
-			"namespace", m.targetSource.Namespace,
+			"name", a.targetSource.Name,
+			"namespace", a.targetSource.Namespace,
 		)
-
 	logger.Info("target applier started")
 
-	queue := []core.DiscoveryMessage{}
-
 	for ctx.Err() == nil {
 		select {
-		case batch, ok := <-m.in:
+		case batch, ok := <-a.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
 				logger.Info("input channel closed, stopping target applier")
 				return nil
 			}
-			queue = append(queue, batch...)
+			a.queue = append(a.queue, batch...)
 
 		case <-ctx.Done():
 			logger.Info("context canceled, stopping target applier")
 			return nil
 		}
 
-		for len(queue) > 0 {
+		for len(a.queue) > 0 {
 			if ctx.Err() != nil {
-				return ctx.Err()
+				return nil // why return nil?
 			}
 
-			msg := queue[0]
-			queue = queue[1:]
+			msg := a.queue[0]
+			a.queue = a.queue[1:]
 
-			if err := m.handleMessage(ctx, msg, logger); err != nil {
+			if err := a.processMessage(ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?
@@ -82,7 +89,7 @@ func (m *TargetApplier) Run(ctx context.Context) error {
 	return nil
 }
 
-func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -94,12 +101,10 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover
 		logger.Info(
 			"received snapshot chunk",
 			"snapshotID", msg.SnapshotID,
+			"index", msg.ChunkIndex,
 			"targetCount", len(msg.Targets),
 		)
-		m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...)
-		if msg.IsLastChunk {
-			m.processSnapshot(msg.SnapshotID, logger)
-		}
+		return a.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
@@ -107,31 +112,155 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover
 			"received discovery event",
 			"target", msg.Target.Name,
 		)
-		switch msg.Event {
-		case core.CREATE:
-			logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
-		case core.UPDATE:
-			logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels)
-		case core.DELETE:
-			logger.Info("Would delete target", "name", msg.Target.Name)
+		return a.processEvent(ctx, msg, logger)
+
+	default:
+		return fmt.Errorf("unknonw discovery message type %T", msg)
+	}
+}
+
+// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
+func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	if a.activeSnapshot == nil {
+		a.startNewSnapshot(chunk, logger)
+		return nil
+	}
+
+	snapshot := a.activeSnapshot
+	// Check if a new snapshot arrived
+	if snapshot.snapshotID != chunk.SnapshotID {
+		// If current snapshot is complete apply it first
+		if snapshot.complete {
+			if err := a.applySnapshot(ctx, snapshot, logger); err != nil {
+				return err
+			}
+		} else {
+			// If a new snapshot is started before the old one completed
+			// the old one can be discarded
+			logger.Info(
+				"discarding incomplete snapshot",
+				"snapshotID", snapshot.snapshotID,
+			)
 		}
+
+		// Start collecting the new snapshot
+		a.startNewSnapshot(chunk, logger)
+		return nil
+	}
+
+	return a.collectSnapshot(chunk, logger)
+}
+
+func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+	a.activeSnapshot = &snapshotBuffer{
+		snapshotID:  chunk.SnapshotID,
+		totalChunks: chunk.TotalChunks,
+		received:    make(map[int][]core.DiscoveredTarget),
+		complete:    false,
+	}
+	// Delete buffered events that will be current with new snapshot
+	a.defferedEvents = nil
+
+	a.collectSnapshot(chunk, logger)
+}
+
+func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	snapshot := a.activeSnapshot
+
+	if chunk.TotalChunks != snapshot.totalChunks {
+		logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID)
+	}
+	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
+		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
+		a.activeSnapshot = nil
+		return nil
+	}
+	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
+		logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex)
+		a.activeSnapshot = nil
+		return nil
+	}
+
+	snapshot.received[chunk.ChunkIndex] = chunk.Targets
+
+	if len(snapshot.received) == snapshot.totalChunks {
+		snapshot.complete = true
 	}
 
 	return nil
 }
 
-// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) {
-	targets := m.collected[snapshotID]
-	delete(m.collected, snapshotID)
+func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
+	select {
+	case <-ctx.Done():
+		a.activeSnapshot = nil
+		return nil
+	default:
+	}
 
-	logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets))
+	var allTargets []core.DiscoveredTarget
+	for i := 0; i < snapshot.totalChunks; i++ {
+		select {
+		case <-ctx.Done():
+			a.activeSnapshot = nil
+			return nil
+		default:
+		}
 
-	if m.targetSource.Spec.Provider.HTTP != nil {
-		logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name)
+		chunk, ok := snapshot.received[i]
+		if !ok {
+			logger.Error(nil, "missing snapshot chunk", "index", i)
+			a.activeSnapshot = nil
+			return nil
+		}
+		allTargets = append(allTargets, chunk...)
 	}
 
-	for _, target := range targets {
-		logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels)
+	logger.Info(
+		"applying snapshot",
+		"snapshotID", snapshot.snapshotID,
+		"targetCount", len(allTargets),
+	)
+
+	// apply all targets
+	// a.applyTargets
+
+	// Replay deffered events
+	for _, event := range a.defferedEvents {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+		if err := a.applyEvent(ctx, event, logger); err != nil {
+			return err
+		}
 	}
+
+	a.activeSnapshot = nil
+	a.defferedEvents = nil
+	return nil
+}
+
+func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+	// If snapshot collecting is active defer events
+	if a.activeSnapshot != nil {
+		a.defferedEvents = append(a.defferedEvents, event)
+		return nil
+	}
+
+	// Apply events
+	return a.applyEvent(ctx, event, logger)
+}
+
+func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+	switch event.Event {
+	case core.CREATE:
+		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
+	case core.UPDATE:
+		logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
+	case core.DELETE:
+		logger.Info("Would delete target", "name", event.Target.Name)
+	}
+	return nil
 }

From a66accbbcac43a0cdbefa4f59231ca57fca1635f Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Sun, 26 Apr 2026 18:23:19 -0600
Subject: [PATCH 036/165] moved finalizer label into const file

---
 internal/controller/const.go                   |  2 ++
 internal/controller/targetsource_controller.go | 10 ++++------
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/internal/controller/const.go b/internal/controller/const.go
index b5196b8..5ef2e8f 100644
--- a/internal/controller/const.go
+++ b/internal/controller/const.go
@@ -21,6 +21,8 @@ const (
 	LabelCertType            = "operator.gnmic.dev/cert-type"
 	LabelValueCertTypeClient = "client"
 	LabelValueCertTypeTunnel = "tunnel"
+
+	LabelTargetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
 )
 
 const (
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index f04eced..232c624 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -37,8 +37,6 @@ import (
 )
 
 const (
-	targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer"
-
 	pipelineMaxRestarts = 5
 	pipelineBackoff     = 3 * time.Second
 )
@@ -138,8 +136,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type
 	r.stopDiscoveryPipeline(key)
 
 	// Remove finalizer if exists
-	if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) {
-		controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer)
+	if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) {
+		controllerutil.RemoveFinalizer(targetSource, LabelTargetSourceFinalizer)
 		if err := r.Update(ctx, targetSource); err != nil {
 			return ctrl.Result{}, err
 		}
@@ -150,11 +148,11 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type
 
 // ensureFinalizer adds the finalizer if not present and updates the TargetSource
 func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error {
-	if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) {
+	if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) {
 		return nil
 	}
 
-	controllerutil.AddFinalizer(targetSource, targetSourceFinalizer)
+	controllerutil.AddFinalizer(targetSource, LabelTargetSourceFinalizer)
 	if err := r.Update(ctx, targetSource); err != nil {
 		return err
 	}

From 3b2d9258a06116738be182e567ee6f275c9ad0e4 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Sun, 26 Apr 2026 18:29:05 -0600
Subject: [PATCH 037/165] fixed typo

---
 internal/controller/discovery/target_applier.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go
index ee127c5..3c714bd 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_applier.go
@@ -29,7 +29,7 @@ type TargetApplier struct {
 	queue          []core.DiscoveryMessage
 	activeSnapshot *snapshotBuffer
 	// Events are deferred while snapshot is in progress
-	defferedEvents []core.DiscoveryEvent
+	deferredEvents []core.DiscoveryEvent
 }
 
 // NewTargetApplier wires a TargetApplier instance
@@ -159,7 +159,7 @@ func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger lo
 		complete:    false,
 	}
 	// Delete buffered events that will be current with new snapshot
-	a.defferedEvents = nil
+	a.deferredEvents = nil
 
 	a.collectSnapshot(chunk, logger)
 }
@@ -225,8 +225,8 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	// apply all targets
 	// a.applyTargets
 
-	// Replay deffered events
-	for _, event := range a.defferedEvents {
+	// Replay deferred events
+	for _, event := range a.deferredEvents {
 		select {
 		case <-ctx.Done():
 			return nil
@@ -238,14 +238,14 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	}
 
 	a.activeSnapshot = nil
-	a.defferedEvents = nil
+	a.deferredEvents = nil
 	return nil
 }
 
 func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	// If snapshot collecting is active defer events
 	if a.activeSnapshot != nil {
-		a.defferedEvents = append(a.defferedEvents, event)
+		a.deferredEvents = append(a.deferredEvents, event)
 		return nil
 	}
 

From 3ba86cb63c45a7f042a2051faca5f8ddfdc5b2ad Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Sun, 26 Apr 2026 19:21:01 -0600
Subject: [PATCH 038/165] restructured loaders package

---
 .../controller/discovery/loaders/http/{loader.go => http.go}   | 0
 .../discovery/loaders/http/{loader_test.go => http_test.go}    | 0
 .../controller/discovery/{loader.go => loaders/loaders.go}     | 2 +-
 internal/controller/targetsource_controller.go                 | 3 ++-
 4 files changed, 3 insertions(+), 2 deletions(-)
 rename internal/controller/discovery/loaders/http/{loader.go => http.go} (100%)
 rename internal/controller/discovery/loaders/http/{loader_test.go => http_test.go} (100%)
 rename internal/controller/discovery/{loader.go => loaders/loaders.go} (97%)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/http.go
similarity index 100%
rename from internal/controller/discovery/loaders/http/loader.go
rename to internal/controller/discovery/loaders/http/http.go
diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/http_test.go
similarity index 100%
rename from internal/controller/discovery/loaders/http/loader_test.go
rename to internal/controller/discovery/loaders/http/http_test.go
diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loaders/loaders.go
similarity index 97%
rename from internal/controller/discovery/loader.go
rename to internal/controller/discovery/loaders/loaders.go
index 0d8ddd3..45bf9c1 100644
--- a/internal/controller/discovery/loader.go
+++ b/internal/controller/discovery/loaders/loaders.go
@@ -1,4 +1,4 @@
-package discovery
+package loaders
 
 import (
 	"fmt"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 232c624..77a3a35 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -31,6 +31,7 @@ import (
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/gnmic/operator/internal/controller/discovery/loaders"
 	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/all"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	"github.com/go-logr/logr"
@@ -207,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	loaderConfigured := targetSource.Spec.Provider != nil
 	webhookConfigured := targetSource.Spec.Webhook.Enabled != nil
 	if loaderConfigured {
-		loader, err := discovery.NewLoader(
+		loader, err := loaders.NewLoader(
 			key,
 			targetSource.Spec,
 			core.LoaderConfig{ChunkSize: r.ChunkSize},

From d0ac86be2e389e91ef833bf5c278324af2df59bb Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Sun, 26 Apr 2026 19:21:13 -0600
Subject: [PATCH 039/165] restructured target handler

---
 internal/controller/discovery/client.go       |  27 ----
 .../{target_applier.go => target_handler.go}  | 121 ++++++++++--------
 .../controller/targetsource_controller.go     |  20 +--
 3 files changed, 80 insertions(+), 88 deletions(-)
 delete mode 100644 internal/controller/discovery/client.go
 rename internal/controller/discovery/{target_applier.go => target_handler.go} (66%)

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
deleted file mode 100644
index 3bc7ef7..0000000
--- a/internal/controller/discovery/client.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package discovery
-
-// File may become obsolete, depends on how the logic to compare desired vs. existing state will get implemented
-
-import (
-	"context"
-
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-)
-
-func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) {
-	var targetList gnmicv1alpha1.TargetList
-
-	err := c.List(ctx, &targetList,
-		client.InNamespace(ts.Namespace),
-		client.MatchingLabels{
-			"gnmic.io/source": ts.Name,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return targetList.Items, nil
-}
diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_handler.go
similarity index 66%
rename from internal/controller/discovery/target_applier.go
rename to internal/controller/discovery/target_handler.go
index 3c714bd..e8c0308 100644
--- a/internal/controller/discovery/target_applier.go
+++ b/internal/controller/discovery/target_handler.go
@@ -20,8 +20,9 @@ type snapshotBuffer struct {
 	complete    bool
 }
 
-// TargetApplier consumes discovered targets and applies them to Kubernetes
-type TargetApplier struct {
+// TargetHandler consumes discovered targets and applies them to Kubernetes
+type TargetHandler struct {
+	ctx            context.Context
 	client         client.Client
 	scheme         *runtime.Scheme
 	targetSource   *gnmicv1alpha1.TargetSource
@@ -32,9 +33,9 @@ type TargetApplier struct {
 	deferredEvents []core.DiscoveryEvent
 }
 
-// NewTargetApplier wires a TargetApplier instance
-func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier {
-	return &TargetApplier{
+// NewTargetHandler wires a TargetHandler instance
+func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler {
+	return &TargetHandler{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
@@ -44,38 +45,40 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (a *TargetApplier) Run(ctx context.Context) error {
-	logger := log.FromContext(ctx).
+func (c *TargetHandler) Run(ctx context.Context) error {
+	c.ctx = ctx
+
+	logger := log.FromContext(c.ctx).
 		WithValues(
-			"name", a.targetSource.Name,
-			"namespace", a.targetSource.Namespace,
+			"name", c.targetSource.Name,
+			"namespace", c.targetSource.Namespace,
 		)
-	logger.Info("target applier started")
+	logger.Info("target handler started")
 
-	for ctx.Err() == nil {
+	for c.ctx.Err() == nil {
 		select {
-		case batch, ok := <-a.in:
+		case batch, ok := <-c.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
-				logger.Info("input channel closed, stopping target applier")
+				logger.Info("input channel closed, stopping target handler")
 				return nil
 			}
-			a.queue = append(a.queue, batch...)
+			c.queue = append(c.queue, batch...)
 
 		case <-ctx.Done():
-			logger.Info("context canceled, stopping target applier")
+			logger.Info("context canceled, stopping target handler")
 			return nil
 		}
 
-		for len(a.queue) > 0 {
+		for len(c.queue) > 0 {
 			if ctx.Err() != nil {
 				return nil // why return nil?
 			}
 
-			msg := a.queue[0]
-			a.queue = a.queue[1:]
+			msg := c.queue[0]
+			c.queue = c.queue[1:]
 
-			if err := a.processMessage(ctx, msg, logger); err != nil {
+			if err := c.processMessage(c.ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?
@@ -85,11 +88,11 @@ func (a *TargetApplier) Run(ctx context.Context) error {
 		}
 	}
 
-	logger.Info("target applier stopped")
+	logger.Info("target handler stopped")
 	return nil
 }
 
-func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -104,7 +107,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove
 			"index", msg.ChunkIndex,
 			"targetCount", len(msg.Targets),
 		)
-		return a.processSnapshot(ctx, msg, logger)
+		return c.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
@@ -112,7 +115,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove
 			"received discovery event",
 			"target", msg.Target.Name,
 		)
-		return a.processEvent(ctx, msg, logger)
+		return c.processEvent(ctx, msg, logger)
 
 	default:
 		return fmt.Errorf("unknonw discovery message type %T", msg)
@@ -120,18 +123,18 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	if a.activeSnapshot == nil {
-		a.startNewSnapshot(chunk, logger)
+func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	if c.activeSnapshot == nil {
+		c.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	snapshot := a.activeSnapshot
+	snapshot := c.activeSnapshot
 	// Check if a new snapshot arrived
 	if snapshot.snapshotID != chunk.SnapshotID {
 		// If current snapshot is complete apply it first
 		if snapshot.complete {
-			if err := a.applySnapshot(ctx, snapshot, logger); err != nil {
+			if err := c.applySnapshot(ctx, snapshot, logger); err != nil {
 				return err
 			}
 		} else {
@@ -144,40 +147,40 @@ func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.Discover
 		}
 
 		// Start collecting the new snapshot
-		a.startNewSnapshot(chunk, logger)
+		c.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	return a.collectSnapshot(chunk, logger)
+	return c.collectSnapshot(chunk, logger)
 }
 
-func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
-	a.activeSnapshot = &snapshotBuffer{
+func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+	c.activeSnapshot = &snapshotBuffer{
 		snapshotID:  chunk.SnapshotID,
 		totalChunks: chunk.TotalChunks,
 		received:    make(map[int][]core.DiscoveredTarget),
 		complete:    false,
 	}
 	// Delete buffered events that will be current with new snapshot
-	a.deferredEvents = nil
+	c.deferredEvents = nil
 
-	a.collectSnapshot(chunk, logger)
+	c.collectSnapshot(chunk, logger)
 }
 
-func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	snapshot := a.activeSnapshot
+func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	snapshot := c.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
 		logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID)
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
 		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
-		a.activeSnapshot = nil
+		c.activeSnapshot = nil
 		return nil
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
 		logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex)
-		a.activeSnapshot = nil
+		c.activeSnapshot = nil
 		return nil
 	}
 
@@ -190,10 +193,10 @@ func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger log
 	return nil
 }
 
-func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
+func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
 	select {
 	case <-ctx.Done():
-		a.activeSnapshot = nil
+		c.activeSnapshot = nil
 		return nil
 	default:
 	}
@@ -202,7 +205,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	for i := 0; i < snapshot.totalChunks; i++ {
 		select {
 		case <-ctx.Done():
-			a.activeSnapshot = nil
+			c.activeSnapshot = nil
 			return nil
 		default:
 		}
@@ -210,7 +213,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 		chunk, ok := snapshot.received[i]
 		if !ok {
 			logger.Error(nil, "missing snapshot chunk", "index", i)
-			a.activeSnapshot = nil
+			c.activeSnapshot = nil
 			return nil
 		}
 		allTargets = append(allTargets, chunk...)
@@ -226,34 +229,34 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	// a.applyTargets
 
 	// Replay deferred events
-	for _, event := range a.deferredEvents {
+	for _, event := range c.deferredEvents {
 		select {
 		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err := a.applyEvent(ctx, event, logger); err != nil {
+		if err := c.applyEvent(ctx, event, logger); err != nil {
 			return err
 		}
 	}
 
-	a.activeSnapshot = nil
-	a.deferredEvents = nil
+	c.activeSnapshot = nil
+	c.deferredEvents = nil
 	return nil
 }
 
-func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	// If snapshot collecting is active defer events
-	if a.activeSnapshot != nil {
-		a.deferredEvents = append(a.deferredEvents, event)
+	if c.activeSnapshot != nil {
+		c.deferredEvents = append(c.deferredEvents, event)
 		return nil
 	}
 
 	// Apply events
-	return a.applyEvent(ctx, event, logger)
+	return c.applyEvent(ctx, event, logger)
 }
 
-func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
 	case core.CREATE:
 		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
@@ -264,3 +267,19 @@ func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEven
 	}
 	return nil
 }
+
+func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) {
+	var targetList gnmicv1alpha1.TargetList
+
+	err := c.client.List(c.ctx, &targetList,
+		client.InNamespace(c.targetSource.Namespace),
+		client.MatchingLabels{
+			"gnmic.io/source": c.targetSource.Name,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return targetList.Items, nil
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 77a3a35..4d5f400 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource
 //
 // Pipeline semantics:
-// 1. target-applier is mandatory and must start first
+// 1. target-handler is mandatory and must start first
 // 2. loader is optional and conditional on spec
 // 3. Permanent failure of required components shuts down the pipeline
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
@@ -176,30 +176,30 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		return err
 	}
 
-	// Create target applier instance
-	applier := discovery.NewTargetApplier(
+	// Create target targetHandler instance
+	targetHandler := discovery.NewTargetHandler(
 		r.Client,
 		r.Scheme,
 		targetSource,
 		targetChannel,
 	)
-	// Start target applier
-	applierReady := make(chan struct{})
+	// Start target handler
+	handlerReady := make(chan struct{})
 	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
-		Name: "target-applier",
+		Name: "target-handler",
 		Policy: discovery.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
 		EscalatesOnFailure: true,
 		Run: func(ctx context.Context) error {
-			close(applierReady) // Signals that applier started successfully
-			return applier.Run(ctx)
+			close(handlerReady) // Signals that handler started successfully
+			return targetHandler.Run(ctx)
 		},
 	})
-	// Wait for applier to be ready before starting loader
+	// Wait for handler to be ready before starting loader
 	select {
-	case <-applierReady:
+	case <-handlerReady:
 	case <-supervisor.Done():
 		return nil
 	}

From 240a2bc382c5133829d327bde1cebfb4fd1530e9 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Sun, 26 Apr 2026 19:59:29 -0600
Subject: [PATCH 040/165] ran go mod tidy

---
 go.mod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index f236ded..827da2a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.25.5
 require (
 	github.com/cert-manager/cert-manager v1.19.3
 	github.com/go-logr/logr v1.4.3
+	github.com/google/uuid v1.6.0
 	github.com/onsi/ginkgo/v2 v2.27.3
 	github.com/onsi/gomega v1.38.3
 	github.com/openconfig/gnmic/pkg/api v0.1.10
@@ -47,7 +48,6 @@ require (
 	github.com/google/gnostic-models v0.7.1 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect

From 7ef1281a7b37bd8b9a845501f7011c615710429b Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Mon, 27 Apr 2026 10:10:08 -0600
Subject: [PATCH 041/165] renamed target applier to message processor & created
 client.go for generic functions

---
 internal/controller/discovery/client.go       |  25 ++++
 ...target_handler.go => message_processor.go} | 112 ++++++++----------
 .../controller/targetsource_controller.go     |   2 +-
 3 files changed, 74 insertions(+), 65 deletions(-)
 create mode 100644 internal/controller/discovery/client.go
 rename internal/controller/discovery/{target_handler.go => message_processor.go} (63%)

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
new file mode 100644
index 0000000..72147b7
--- /dev/null
+++ b/internal/controller/discovery/client.go
@@ -0,0 +1,25 @@
+package discovery
+
+import (
+	"context"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+)
+
+func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) {
+	var targetList gnmicv1alpha1.TargetList
+
+	err := c.List(ctx, &targetList,
+		client.InNamespace(ts.Namespace),
+		client.MatchingLabels{
+			"gnmic.io/source": ts.Name,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return targetList.Items, nil
+}
diff --git a/internal/controller/discovery/target_handler.go b/internal/controller/discovery/message_processor.go
similarity index 63%
rename from internal/controller/discovery/target_handler.go
rename to internal/controller/discovery/message_processor.go
index e8c0308..65c8b44 100644
--- a/internal/controller/discovery/target_handler.go
+++ b/internal/controller/discovery/message_processor.go
@@ -20,8 +20,8 @@ type snapshotBuffer struct {
 	complete    bool
 }
 
-// TargetHandler consumes discovered targets and applies them to Kubernetes
-type TargetHandler struct {
+// MessageProcessor consumes discovered targets and applies them to Kubernetes
+type MessageProcessor struct {
 	ctx            context.Context
 	client         client.Client
 	scheme         *runtime.Scheme
@@ -33,9 +33,9 @@ type TargetHandler struct {
 	deferredEvents []core.DiscoveryEvent
 }
 
-// NewTargetHandler wires a TargetHandler instance
-func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler {
-	return &TargetHandler{
+// NewMessageProcessor wires a MessageProcessor instance
+func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor {
+	return &MessageProcessor{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
@@ -45,40 +45,40 @@ func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (c *TargetHandler) Run(ctx context.Context) error {
-	c.ctx = ctx
+func (m *MessageProcessor) Run(ctx context.Context) error {
+	m.ctx = ctx
 
-	logger := log.FromContext(c.ctx).
+	logger := log.FromContext(m.ctx).
 		WithValues(
-			"name", c.targetSource.Name,
-			"namespace", c.targetSource.Namespace,
+			"name", m.targetSource.Name,
+			"namespace", m.targetSource.Namespace,
 		)
 	logger.Info("target handler started")
 
-	for c.ctx.Err() == nil {
+	for m.ctx.Err() == nil {
 		select {
-		case batch, ok := <-c.in:
+		case batch, ok := <-m.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
 				logger.Info("input channel closed, stopping target handler")
 				return nil
 			}
-			c.queue = append(c.queue, batch...)
+			m.queue = append(m.queue, batch...)
 
 		case <-ctx.Done():
 			logger.Info("context canceled, stopping target handler")
 			return nil
 		}
 
-		for len(c.queue) > 0 {
+		for len(m.queue) > 0 {
 			if ctx.Err() != nil {
 				return nil // why return nil?
 			}
 
-			msg := c.queue[0]
-			c.queue = c.queue[1:]
+			msg := m.queue[0]
+			m.queue = m.queue[1:]
 
-			if err := c.processMessage(c.ctx, msg, logger); err != nil {
+			if err := m.processMessage(m.ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?
@@ -92,7 +92,7 @@ func (c *TargetHandler) Run(ctx context.Context) error {
 	return nil
 }
 
-func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -107,7 +107,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove
 			"index", msg.ChunkIndex,
 			"targetCount", len(msg.Targets),
 		)
-		return c.processSnapshot(ctx, msg, logger)
+		return m.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
@@ -115,7 +115,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove
 			"received discovery event",
 			"target", msg.Target.Name,
 		)
-		return c.processEvent(ctx, msg, logger)
+		return m.processEvent(ctx, msg, logger)
 
 	default:
 		return fmt.Errorf("unknonw discovery message type %T", msg)
@@ -123,18 +123,18 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	if c.activeSnapshot == nil {
-		c.startNewSnapshot(chunk, logger)
+func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	if m.activeSnapshot == nil {
+		m.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	snapshot := c.activeSnapshot
+	snapshot := m.activeSnapshot
 	// Check if a new snapshot arrived
 	if snapshot.snapshotID != chunk.SnapshotID {
 		// If current snapshot is complete apply it first
 		if snapshot.complete {
-			if err := c.applySnapshot(ctx, snapshot, logger); err != nil {
+			if err := m.applySnapshot(ctx, snapshot, logger); err != nil {
 				return err
 			}
 		} else {
@@ -147,40 +147,40 @@ func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.Discover
 		}
 
 		// Start collecting the new snapshot
-		c.startNewSnapshot(chunk, logger)
+		m.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	return c.collectSnapshot(chunk, logger)
+	return m.collectSnapshot(chunk, logger)
 }
 
-func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
-	c.activeSnapshot = &snapshotBuffer{
+func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+	m.activeSnapshot = &snapshotBuffer{
 		snapshotID:  chunk.SnapshotID,
 		totalChunks: chunk.TotalChunks,
 		received:    make(map[int][]core.DiscoveredTarget),
 		complete:    false,
 	}
 	// Delete buffered events that will be current with new snapshot
-	c.deferredEvents = nil
+	m.deferredEvents = nil
 
-	c.collectSnapshot(chunk, logger)
+	m.collectSnapshot(chunk, logger)
 }
 
-func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	snapshot := c.activeSnapshot
+func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	snapshot := m.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
 		logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID)
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
 		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
-		c.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
 		logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex)
-		c.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	}
 
@@ -193,10 +193,10 @@ func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger log
 	return nil
 }
 
-func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
+func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
 	select {
 	case <-ctx.Done():
-		c.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	default:
 	}
@@ -205,7 +205,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	for i := 0; i < snapshot.totalChunks; i++ {
 		select {
 		case <-ctx.Done():
-			c.activeSnapshot = nil
+			m.activeSnapshot = nil
 			return nil
 		default:
 		}
@@ -213,7 +213,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 		chunk, ok := snapshot.received[i]
 		if !ok {
 			logger.Error(nil, "missing snapshot chunk", "index", i)
-			c.activeSnapshot = nil
+			m.activeSnapshot = nil
 			return nil
 		}
 		allTargets = append(allTargets, chunk...)
@@ -229,34 +229,34 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf
 	// a.applyTargets
 
 	// Replay deferred events
-	for _, event := range c.deferredEvents {
+	for _, event := range m.deferredEvents {
 		select {
 		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err := c.applyEvent(ctx, event, logger); err != nil {
+		if err := m.applyEvent(ctx, event, logger); err != nil {
 			return err
 		}
 	}
 
-	c.activeSnapshot = nil
-	c.deferredEvents = nil
+	m.activeSnapshot = nil
+	m.deferredEvents = nil
 	return nil
 }
 
-func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	// If snapshot collecting is active defer events
-	if c.activeSnapshot != nil {
-		c.deferredEvents = append(c.deferredEvents, event)
+	if m.activeSnapshot != nil {
+		m.deferredEvents = append(m.deferredEvents, event)
 		return nil
 	}
 
 	// Apply events
-	return c.applyEvent(ctx, event, logger)
+	return m.applyEvent(ctx, event, logger)
 }
 
-func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
 	case core.CREATE:
 		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
@@ -267,19 +267,3 @@ func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEven
 	}
 	return nil
 }
-
-func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) {
-	var targetList gnmicv1alpha1.TargetList
-
-	err := c.client.List(c.ctx, &targetList,
-		client.InNamespace(c.targetSource.Namespace),
-		client.MatchingLabels{
-			"gnmic.io/source": c.targetSource.Name,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return targetList.Items, nil
-}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 4d5f400..8070a3a 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -177,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Create target targetHandler instance
-	targetHandler := discovery.NewTargetHandler(
+	targetHandler := discovery.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,

From 7bcbcc023ff39e36a565e9235f503a98375f3327 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Fri, 24 Apr 2026 15:15:56 -0600
Subject: [PATCH 042/165] added const file for common labels

---
 internal/controller/discovery/client.go     | 3 ++-
 internal/controller/discovery/core/const.go | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 internal/controller/discovery/core/const.go

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
index 3bc7ef7..d23c043 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/client.go
@@ -8,6 +8,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
 )
 
 func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) {
@@ -16,7 +17,7 @@ func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1
 	err := c.List(ctx, &targetList,
 		client.InNamespace(ts.Namespace),
 		client.MatchingLabels{
-			"gnmic.io/source": ts.Name,
+			core.LabelTargetSourceName: ts.Name,
 		},
 	)
 	if err != nil {
diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/core/const.go
new file mode 100644
index 0000000..82a5962
--- /dev/null
+++ b/internal/controller/discovery/core/const.go
@@ -0,0 +1,6 @@
+package core
+
+const (
+	// Labels
+	LabelTargetSourceName = "operator.gnmic.dev/targetsource"
+)

From d10fc9ac868d50be64c123cbc619b2f4eb189682 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Mon, 27 Apr 2026 11:26:10 -0600
Subject: [PATCH 043/165] removed all package

---
 internal/controller/discovery/loaders/all/all.go | 5 -----
 internal/controller/targetsource_controller.go   | 1 -
 2 files changed, 6 deletions(-)
 delete mode 100644 internal/controller/discovery/loaders/all/all.go

diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go
deleted file mode 100644
index 3590cda..0000000
--- a/internal/controller/discovery/loaders/all/all.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package all
-
-import (
-	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
-)
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 8070a3a..49f9683 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -32,7 +32,6 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/loaders"
-	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/all"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	"github.com/go-logr/logr"
 )

From 108bd2dc3f58b2193535c8eadf6c30ee1d6d0dad Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Mon, 27 Apr 2026 11:34:11 -0600
Subject: [PATCH 044/165] changed error lookup to apierrors

---
 internal/controller/targetsource_controller.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 232c624..2f198a6 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -81,13 +82,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		WithValues("targetsource", req.NamespacedName)
 
 	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
-	if err != nil {
-		// If the TargetSource no longer exists, ensure runtime cleanup
-		if client.IgnoreNotFound(err) == nil {
-			logger.Info("TargetSource not found; stopping discovery pipeline")
-			r.stopDiscoveryPipeline(req.NamespacedName)
-			return ctrl.Result{}, nil
-		}
+	// If the TargetSource no longer exists, ensure runtime cleanup
+	if apierrors.IsNotFound(err) {
+		logger.Info("TargetSource not found; stopping discovery pipeline")
+		r.stopDiscoveryPipeline(req.NamespacedName)
+		return ctrl.Result{}, nil
+	} else if err != nil {
 		return ctrl.Result{}, err
 	}
 

From b7dd0367e99a0c5435db00092c83e1bc01ab439b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 08:53:30 +0000
Subject: [PATCH 045/165] remove unused fiels

---
 internal/controller/discovery/mapper.go      | 4 ----
 internal/controller/discovery/mapper_test.go | 1 -
 2 files changed, 5 deletions(-)
 delete mode 100644 internal/controller/discovery/mapper.go
 delete mode 100644 internal/controller/discovery/mapper_test.go

diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go
deleted file mode 100644
index 18470b2..0000000
--- a/internal/controller/discovery/mapper.go
+++ /dev/null
@@ -1,4 +0,0 @@
-package discovery
-
-// This file makes diff between existing and new targets
-// file decides which targets to create/update/delete
diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go
deleted file mode 100644
index 5844159..0000000
--- a/internal/controller/discovery/mapper_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package discovery

From d3a9b5ca3021c9f0485698c1d1c54bbd3562bb9b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 11:56:56 +0000
Subject: [PATCH 046/165] rename files and restructure packages

---
 .../core/{loader_interface.go => loader.go}     |  0
 .../core/{message_interface.go => message.go}   |  0
 .../discovery/core/{helpers.go => send.go}      |  0
 internal/controller/discovery/core/types.go     |  4 ++--
 internal/controller/discovery/discovery.go      | 17 +++++++++++++++++
 .../loaders/{loaders.go => factory.go}          |  0
 .../loaders/http/{http.go => loader.go}         |  0
 .../http/{http_test.go => loader_test.go}       |  0
 .../discovery/{ => pipeline}/supervisor.go      |  5 +++--
 .../discovery/{ => reconciler}/client.go        | 13 ++++++++++---
 .../{ => reconciler}/message_processor.go       |  2 +-
 internal/controller/targetsource_controller.go  | 15 ++++++++-------
 12 files changed, 41 insertions(+), 15 deletions(-)
 rename internal/controller/discovery/core/{loader_interface.go => loader.go} (100%)
 rename internal/controller/discovery/core/{message_interface.go => message.go} (100%)
 rename internal/controller/discovery/core/{helpers.go => send.go} (100%)
 create mode 100644 internal/controller/discovery/discovery.go
 rename internal/controller/discovery/loaders/{loaders.go => factory.go} (100%)
 rename internal/controller/discovery/loaders/http/{http.go => loader.go} (100%)
 rename internal/controller/discovery/loaders/http/{http_test.go => loader_test.go} (100%)
 rename internal/controller/discovery/{ => pipeline}/supervisor.go (95%)
 rename internal/controller/discovery/{ => reconciler}/client.go (68%)
 rename internal/controller/discovery/{ => reconciler}/message_processor.go (99%)

diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader.go
similarity index 100%
rename from internal/controller/discovery/core/loader_interface.go
rename to internal/controller/discovery/core/loader.go
diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message.go
similarity index 100%
rename from internal/controller/discovery/core/message_interface.go
rename to internal/controller/discovery/core/message.go
diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/send.go
similarity index 100%
rename from internal/controller/discovery/core/helpers.go
rename to internal/controller/discovery/core/send.go
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 3f6957a..28ec503 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -12,14 +12,14 @@ type DiscoveredTarget struct {
 	Labels  map[string]string
 }
 
+type EventAction int
+
 const (
 	DELETE EventAction = 0
 	CREATE EventAction = 1
 	UPDATE EventAction = 2
 )
 
-type EventAction int
-
 type DiscoveryEvent struct {
 	Target DiscoveredTarget
 	Event  EventAction
diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go
new file mode 100644
index 0000000..3dc51bd
--- /dev/null
+++ b/internal/controller/discovery/discovery.go
@@ -0,0 +1,17 @@
+package discovery
+
+// Package discovery implements the discovery runtime subsystem.
+//
+// The discovery subsystem is responsible for:
+// - Receiving discovery data from external providers (loaders, webhooks).
+// - Supervising discovery pipelines and restart semantics.
+// - Applying discovered state to Kubernetes Targets.
+//
+// The package is structured into the following subpackages:
+// - core: message contracts, snapshot/event types, and transport helpers.
+// - pipeline: supervision, restart policies, and lifecycle control.
+// - reconciler: snapshot + event target state application logic.
+// - loaders: target discovery providers (HTTP, webhook, etc.).
+// - registry: key -> channel registry.
+//
+// At the moment, the targetsource controller imports specific subpackages explicitly.
diff --git a/internal/controller/discovery/loaders/loaders.go b/internal/controller/discovery/loaders/factory.go
similarity index 100%
rename from internal/controller/discovery/loaders/loaders.go
rename to internal/controller/discovery/loaders/factory.go
diff --git a/internal/controller/discovery/loaders/http/http.go b/internal/controller/discovery/loaders/http/loader.go
similarity index 100%
rename from internal/controller/discovery/loaders/http/http.go
rename to internal/controller/discovery/loaders/http/loader.go
diff --git a/internal/controller/discovery/loaders/http/http_test.go b/internal/controller/discovery/loaders/http/loader_test.go
similarity index 100%
rename from internal/controller/discovery/loaders/http/http_test.go
rename to internal/controller/discovery/loaders/http/loader_test.go
diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/pipeline/supervisor.go
similarity index 95%
rename from internal/controller/discovery/supervisor.go
rename to internal/controller/discovery/pipeline/supervisor.go
index 710381e..042d305 100644
--- a/internal/controller/discovery/supervisor.go
+++ b/internal/controller/discovery/pipeline/supervisor.go
@@ -1,4 +1,4 @@
-package discovery
+package pipeline
 
 import (
 	"context"
@@ -25,12 +25,13 @@ type Supervisor struct {
 	stopped bool
 }
 
-// RestartPolicy defines the restart behavior for a component
+// RestartPolicy defines restart behavior of a component
 type RestartPolicy struct {
 	MaxRestarts int
 	Backoff     time.Duration
 }
 
+// ComponentSpec defines a supervised component
 type ComponentSpec struct {
 	Name   string
 	Run    func(ctx context.Context) error
diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/reconciler/client.go
similarity index 68%
rename from internal/controller/discovery/client.go
rename to internal/controller/discovery/reconciler/client.go
index 25100bd..4bbbbc1 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/reconciler/client.go
@@ -1,4 +1,4 @@
-package discovery
+package reconciler
 
 import (
 	"context"
@@ -9,10 +9,17 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 )
 
-func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) {
+func fetchExistingTargets(
+	ctx context.Context,
+	c client.Client,
+	ts *gnmicv1alpha1.TargetSource,
+) ([]gnmicv1alpha1.Target, error) {
+
 	var targetList gnmicv1alpha1.TargetList
 
-	err := c.List(ctx, &targetList,
+	err := c.List(
+		ctx,
+		&targetList,
 		client.InNamespace(ts.Namespace),
 		client.MatchingLabels{
 			core.LabelTargetSourceName: ts.Name,
diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go
similarity index 99%
rename from internal/controller/discovery/message_processor.go
rename to internal/controller/discovery/reconciler/message_processor.go
index 65c8b44..0c205bd 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/reconciler/message_processor.go
@@ -1,4 +1,4 @@
-package discovery
+package reconciler
 
 import (
 	"context"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 49f9683..35946d2 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -29,9 +29,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/loaders"
+	"github.com/gnmic/operator/internal/controller/discovery/pipeline"
+	"github.com/gnmic/operator/internal/controller/discovery/reconciler"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	"github.com/go-logr/logr"
 )
@@ -168,7 +169,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 // 3. Permanent failure of required components shuts down the pipeline
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
-	supervisor := discovery.NewSupervisor(context.Background())
+	supervisor := pipeline.NewSupervisor(context.Background())
 
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 	if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil {
@@ -176,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Create target targetHandler instance
-	targetHandler := discovery.NewMessageProcessor(
+	targetHandler := reconciler.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,
@@ -184,9 +185,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	)
 	// Start target handler
 	handlerReady := make(chan struct{})
-	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
+	supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
 		Name: "target-handler",
-		Policy: discovery.RestartPolicy{
+		Policy: pipeline.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
@@ -217,9 +218,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 			return err
 		}
 
-		supervisor.StartSupervisedComponent(discovery.ComponentSpec{
+		supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
 			Name: "loader",
-			Policy: discovery.RestartPolicy{
+			Policy: pipeline.RestartPolicy{
 				MaxRestarts: pipelineMaxRestarts,
 				Backoff:     pipelineBackoff,
 			},

From 0c80394ab358c662fe519b872ed7219c2f7e384c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 12:01:40 +0000
Subject: [PATCH 047/165] rename target handler to target reconciler

---
 .../discovery/reconciler/message_processor.go          |  8 ++++----
 internal/controller/targetsource_controller.go         | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go
index 0c205bd..2c4632c 100644
--- a/internal/controller/discovery/reconciler/message_processor.go
+++ b/internal/controller/discovery/reconciler/message_processor.go
@@ -53,20 +53,20 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 			"name", m.targetSource.Name,
 			"namespace", m.targetSource.Namespace,
 		)
-	logger.Info("target handler started")
+	logger.Info("target reconciler started")
 
 	for m.ctx.Err() == nil {
 		select {
 		case batch, ok := <-m.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
-				logger.Info("input channel closed, stopping target handler")
+				logger.Info("input channel closed, stopping target reconciler")
 				return nil
 			}
 			m.queue = append(m.queue, batch...)
 
 		case <-ctx.Done():
-			logger.Info("context canceled, stopping target handler")
+			logger.Info("context canceled, stopping target reconciler")
 			return nil
 		}
 
@@ -88,7 +88,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 		}
 	}
 
-	logger.Info("target handler stopped")
+	logger.Info("target reconciler stopped")
 	return nil
 }
 
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 35946d2..6c9ad31 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource
 //
 // Pipeline semantics:
-// 1. target-handler is mandatory and must start first
+// 1. target reconciler is mandatory and must start first
 // 2. loader is optional and conditional on spec
 // 3. Permanent failure of required components shuts down the pipeline
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
@@ -176,14 +176,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		return err
 	}
 
-	// Create target targetHandler instance
-	targetHandler := reconciler.NewMessageProcessor(
+	// Create target reconciler instance
+	targetReconciler := reconciler.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,
 		targetChannel,
 	)
-	// Start target handler
+	// Start target reconciler
 	handlerReady := make(chan struct{})
 	supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
 		Name: "target-handler",
@@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		EscalatesOnFailure: true,
 		Run: func(ctx context.Context) error {
 			close(handlerReady) // Signals that handler started successfully
-			return targetHandler.Run(ctx)
+			return targetReconciler.Run(ctx)
 		},
 	})
 	// Wait for handler to be ready before starting loader

From 04208bf078b170160a6ef72eda6b6ddaa3630070 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 12:20:58 +0000
Subject: [PATCH 048/165] rename handler to reconciler

---
 internal/controller/targetsource_controller.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 6c9ad31..9078af2 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -184,22 +184,22 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		targetChannel,
 	)
 	// Start target reconciler
-	handlerReady := make(chan struct{})
+	reconcilerReady := make(chan struct{})
 	supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
-		Name: "target-handler",
+		Name: "target-reconciler",
 		Policy: pipeline.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
 		EscalatesOnFailure: true,
 		Run: func(ctx context.Context) error {
-			close(handlerReady) // Signals that handler started successfully
+			close(reconcilerReady) // Signals that reconciler started successfully
 			return targetReconciler.Run(ctx)
 		},
 	})
-	// Wait for handler to be ready before starting loader
+	// Wait for reconciler to be ready before starting loader
 	select {
-	case <-handlerReady:
+	case <-reconcilerReady:
 	case <-supervisor.Done():
 		return nil
 	}

From c3818ce6f7693360496866d7ba1694f7ce702f32 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 12:21:46 +0000
Subject: [PATCH 049/165] clarify interface files

---
 .../discovery/core/{loader.go => loader_interface.go}        | 2 +-
 internal/controller/discovery/core/message.go                | 4 ----
 internal/controller/discovery/core/message_interface.go      | 5 +++++
 3 files changed, 6 insertions(+), 5 deletions(-)
 rename internal/controller/discovery/core/{loader.go => loader_interface.go} (91%)
 create mode 100644 internal/controller/discovery/core/message_interface.go

diff --git a/internal/controller/discovery/core/loader.go b/internal/controller/discovery/core/loader_interface.go
similarity index 91%
rename from internal/controller/discovery/core/loader.go
rename to internal/controller/discovery/core/loader_interface.go
index 8964be8..72f1898 100644
--- a/internal/controller/discovery/core/loader.go
+++ b/internal/controller/discovery/core/loader_interface.go
@@ -14,7 +14,7 @@ type Loader interface {
 	Name() string
 
 	// Start begins discovery and pushes target snapshots or events into the out channel
-	// The loader must stop cleanly when ctx is cancelled
+	// The loader must stop cleanly when ctx is canceled
 	Start(
 		ctx context.Context,
 		targetsourceName types.NamespacedName,
diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go
index 0836bc6..af4f6c1 100644
--- a/internal/controller/discovery/core/message.go
+++ b/internal/controller/discovery/core/message.go
@@ -1,8 +1,4 @@
 package core
 
-type DiscoveryMessage interface {
-	isDiscoveryMessage()
-}
-
 func (DiscoveryEvent) isDiscoveryMessage()    {}
 func (DiscoverySnapshot) isDiscoveryMessage() {}
diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go
new file mode 100644
index 0000000..07b819e
--- /dev/null
+++ b/internal/controller/discovery/core/message_interface.go
@@ -0,0 +1,5 @@
+package core
+
+type DiscoveryMessage interface {
+	isDiscoveryMessage()
+}

From e4df0d4a6245d71d48539414b0f3ab45136de874 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 12:35:14 +0000
Subject: [PATCH 050/165] define EventAction to be go idomatic

---
 internal/controller/discovery/core/types.go   | 20 +++++++++++--------
 .../discovery/reconciler/message_processor.go |  6 +++---
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 28ec503..1ae2f7a 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -4,6 +4,18 @@ type LoaderConfig struct {
 	ChunkSize int
 }
 
+// EventAction represents the type of a discovery event
+type EventAction int
+
+const (
+	// EventDelete indicates that a target should be removed
+	EventDelete EventAction = iota
+	// EventCreate indicates that a target should be created
+	EventCreate
+	// EventUpdate indicates that a target should be updated
+	EventUpdate
+)
+
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
@@ -12,14 +24,6 @@ type DiscoveredTarget struct {
 	Labels  map[string]string
 }
 
-type EventAction int
-
-const (
-	DELETE EventAction = 0
-	CREATE EventAction = 1
-	UPDATE EventAction = 2
-)
-
 type DiscoveryEvent struct {
 	Target DiscoveredTarget
 	Event  EventAction
diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go
index 2c4632c..a0e91e5 100644
--- a/internal/controller/discovery/reconciler/message_processor.go
+++ b/internal/controller/discovery/reconciler/message_processor.go
@@ -258,11 +258,11 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover
 
 func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
-	case core.CREATE:
+	case core.EventCreate:
 		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
-	case core.UPDATE:
+	case core.EventUpdate:
 		logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
-	case core.DELETE:
+	case core.EventDelete:
 		logger.Info("Would delete target", "name", event.Target.Name)
 	}
 	return nil

From 86c0af066faef2af3e75d68d3285c16dc6978bbe Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 29 Apr 2026 13:49:19 +0000
Subject: [PATCH 051/165] add webhook activation info to metadata of
 DiscoveryRegistry

---
 cmd/main.go                                    |  2 +-
 internal/apiserver/apiserver.go                |  2 +-
 internal/controller/discovery/core/types.go    |  5 +++++
 .../controller/discovery/registry/registry.go  | 14 +++++++-------
 internal/controller/targetsource_controller.go | 18 +++++++++++-------
 5 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index e4bad31..4cf6e94 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -86,7 +86,7 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]()
+	discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go
index 17e5c82..a7ca16a 100644
--- a/internal/apiserver/apiserver.go
+++ b/internal/apiserver/apiserver.go
@@ -14,7 +14,7 @@ type APIServer struct {
 	Server            *http.Server
 	clusterReconciler *controller.ClusterReconciler
 
-	DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage]
+	DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
 }
 
 func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer {
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 1ae2f7a..68c9c7e 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -1,5 +1,10 @@
 package core
 
+type DiscoveryRegistryValue struct {
+	Channel        chan<- []DiscoveryMessage
+	WebhookEnabled bool
+}
+
 type LoaderConfig struct {
 	ChunkSize int
 }
diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go
index 093bd2c..f2630e8 100644
--- a/internal/controller/discovery/registry/registry.go
+++ b/internal/controller/discovery/registry/registry.go
@@ -10,20 +10,20 @@ import (
 // DO NOT USE a pointer type as K
 type Registry[K comparable, V any] struct {
 	mu sync.RWMutex
-	m  map[K]chan<- V
+	m  map[K]V
 }
 
 func NewRegistry[K comparable, V any]() *Registry[K, V] {
-	return &Registry[K, V]{m: make(map[K]chan<- V)}
+	return &Registry[K, V]{m: make(map[K]V)}
 }
 
-func (r *Registry[K, V]) Register(key K, ch chan<- V) error {
+func (r *Registry[K, V]) Register(key K, value V) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if _, exists := r.m[key]; exists {
 		return fmt.Errorf("already registered: %v", key)
 	}
-	r.m[key] = ch
+	r.m[key] = value
 	return nil
 }
 
@@ -33,9 +33,9 @@ func (r *Registry[K, V]) Unregister(key K) {
 	r.mu.Unlock()
 }
 
-func (r *Registry[K, V]) Get(key K) (chan<- V, bool) {
+func (r *Registry[K, V]) Get(key K) (V, bool) {
 	r.mu.RLock()
-	ch, ok := r.m[key]
+	value, ok := r.m[key]
 	r.mu.RUnlock()
-	return ch, ok
+	return value, ok
 }
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 9078af2..c7e6460 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -66,7 +66,7 @@ type TargetSourceReconciler struct {
 	BufferSize int
 	ChunkSize  int
 
-	DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage]
+	DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -108,7 +108,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	logger.Info("Discover pipeline started")
+	logger.Info("Discovery pipeline started")
 	return ctrl.Result{}, nil
 }
 
@@ -161,7 +161,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 	return nil
 }
 
-// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource
+// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource
 //
 // Pipeline semantics:
 // 1. target reconciler is mandatory and must start first
@@ -169,10 +169,16 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 // 3. Permanent failure of required components shuts down the pipeline
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
+	loaderConfigured := targetSource.Spec.Provider != nil
+	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
+
 	supervisor := pipeline.NewSupervisor(context.Background())
 
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
-	if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil {
+	if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{
+		Channel:        targetChannel,
+		WebhookEnabled: webhookActivated,
+	}); err != nil {
 		return err
 	}
 
@@ -205,8 +211,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Create loader instance
-	loaderConfigured := targetSource.Spec.Provider != nil
-	webhookConfigured := targetSource.Spec.Webhook.Enabled != nil
 	if loaderConfigured {
 		loader, err := loaders.NewLoader(
 			key,
@@ -224,7 +228,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 				MaxRestarts: pipelineMaxRestarts,
 				Backoff:     pipelineBackoff,
 			},
-			EscalatesOnFailure: !webhookConfigured,
+			EscalatesOnFailure: !webhookActivated,
 			Run: func(ctx context.Context) error {
 				return loader.Start(ctx, key, targetSource.Spec, targetChannel)
 			},

From 284b1f290bd7f1c33f6213bba5399fb16ac0dae9 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:33:40 -0600
Subject: [PATCH 052/165] moved reconciler files to discovery

---
 internal/controller/discovery/{reconciler => }/client.go      | 2 +-
 .../discovery/{reconciler => }/message_processor.go           | 2 +-
 internal/controller/targetsource_controller.go                | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename internal/controller/discovery/{reconciler => }/client.go (96%)
 rename internal/controller/discovery/{reconciler => }/message_processor.go (99%)

diff --git a/internal/controller/discovery/reconciler/client.go b/internal/controller/discovery/client.go
similarity index 96%
rename from internal/controller/discovery/reconciler/client.go
rename to internal/controller/discovery/client.go
index 4bbbbc1..2deb477 100644
--- a/internal/controller/discovery/reconciler/client.go
+++ b/internal/controller/discovery/client.go
@@ -1,4 +1,4 @@
-package reconciler
+package discovery
 
 import (
 	"context"
diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/message_processor.go
similarity index 99%
rename from internal/controller/discovery/reconciler/message_processor.go
rename to internal/controller/discovery/message_processor.go
index a0e91e5..6e69c99 100644
--- a/internal/controller/discovery/reconciler/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -1,4 +1,4 @@
-package reconciler
+package discovery
 
 import (
 	"context"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index c7e6460..84f9a6f 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -29,10 +29,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/loaders"
 	"github.com/gnmic/operator/internal/controller/discovery/pipeline"
-	"github.com/gnmic/operator/internal/controller/discovery/reconciler"
 	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	"github.com/go-logr/logr"
 )
@@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Create target reconciler instance
-	targetReconciler := reconciler.NewMessageProcessor(
+	targetReconciler := discovery.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,

From b59897c253b5db8858a03026ae187ac6c8959d19 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:34:55 -0600
Subject: [PATCH 053/165] renamed messageProcessor to targetReconciler

---
 ...sage_processor.go => target_reconciler.go} | 96 +++++++++----------
 .../controller/targetsource_controller.go     |  2 +-
 2 files changed, 49 insertions(+), 49 deletions(-)
 rename internal/controller/discovery/{message_processor.go => target_reconciler.go} (72%)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/target_reconciler.go
similarity index 72%
rename from internal/controller/discovery/message_processor.go
rename to internal/controller/discovery/target_reconciler.go
index 6e69c99..4f3711c 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -20,8 +20,8 @@ type snapshotBuffer struct {
 	complete    bool
 }
 
-// MessageProcessor consumes discovered targets and applies them to Kubernetes
-type MessageProcessor struct {
+// TargetReconciler consumes discovered targets and applies them to Kubernetes
+type TargetReconciler struct {
 	ctx            context.Context
 	client         client.Client
 	scheme         *runtime.Scheme
@@ -33,9 +33,9 @@ type MessageProcessor struct {
 	deferredEvents []core.DiscoveryEvent
 }
 
-// NewMessageProcessor wires a MessageProcessor instance
-func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor {
-	return &MessageProcessor{
+// NewTargetReconciler wires a TargetReconciler instance
+func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler {
+	return &TargetReconciler{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
@@ -45,40 +45,40 @@ func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (m *MessageProcessor) Run(ctx context.Context) error {
-	m.ctx = ctx
+func (r *TargetReconciler) Run(ctx context.Context) error {
+	r.ctx = ctx
 
-	logger := log.FromContext(m.ctx).
+	logger := log.FromContext(r.ctx).
 		WithValues(
-			"name", m.targetSource.Name,
-			"namespace", m.targetSource.Namespace,
+			"name", r.targetSource.Name,
+			"namespace", r.targetSource.Namespace,
 		)
 	logger.Info("target reconciler started")
 
-	for m.ctx.Err() == nil {
+	for r.ctx.Err() == nil {
 		select {
-		case batch, ok := <-m.in:
+		case batch, ok := <-r.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
 				logger.Info("input channel closed, stopping target reconciler")
 				return nil
 			}
-			m.queue = append(m.queue, batch...)
+			r.queue = append(r.queue, batch...)
 
 		case <-ctx.Done():
 			logger.Info("context canceled, stopping target reconciler")
 			return nil
 		}
 
-		for len(m.queue) > 0 {
+		for len(r.queue) > 0 {
 			if ctx.Err() != nil {
 				return nil // why return nil?
 			}
 
-			msg := m.queue[0]
-			m.queue = m.queue[1:]
+			msg := r.queue[0]
+			r.queue = r.queue[1:]
 
-			if err := m.processMessage(m.ctx, msg, logger); err != nil {
+			if err := r.processMessage(r.ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?
@@ -92,7 +92,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 	return nil
 }
 
-func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -107,7 +107,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc
 			"index", msg.ChunkIndex,
 			"targetCount", len(msg.Targets),
 		)
-		return m.processSnapshot(ctx, msg, logger)
+		return r.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
@@ -115,7 +115,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc
 			"received discovery event",
 			"target", msg.Target.Name,
 		)
-		return m.processEvent(ctx, msg, logger)
+		return r.processEvent(ctx, msg, logger)
 
 	default:
 		return fmt.Errorf("unknonw discovery message type %T", msg)
@@ -123,18 +123,18 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	if m.activeSnapshot == nil {
-		m.startNewSnapshot(chunk, logger)
+func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	if r.activeSnapshot == nil {
+		r.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	snapshot := m.activeSnapshot
+	snapshot := r.activeSnapshot
 	// Check if a new snapshot arrived
 	if snapshot.snapshotID != chunk.SnapshotID {
 		// If current snapshot is complete apply it first
 		if snapshot.complete {
-			if err := m.applySnapshot(ctx, snapshot, logger); err != nil {
+			if err := r.applySnapshot(ctx, snapshot, logger); err != nil {
 				return err
 			}
 		} else {
@@ -147,40 +147,40 @@ func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.Disco
 		}
 
 		// Start collecting the new snapshot
-		m.startNewSnapshot(chunk, logger)
+		r.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	return m.collectSnapshot(chunk, logger)
+	return r.collectSnapshot(chunk, logger)
 }
 
-func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
-	m.activeSnapshot = &snapshotBuffer{
+func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+	r.activeSnapshot = &snapshotBuffer{
 		snapshotID:  chunk.SnapshotID,
 		totalChunks: chunk.TotalChunks,
 		received:    make(map[int][]core.DiscoveredTarget),
 		complete:    false,
 	}
 	// Delete buffered events that will be current with new snapshot
-	m.deferredEvents = nil
+	r.deferredEvents = nil
 
-	m.collectSnapshot(chunk, logger)
+	r.collectSnapshot(chunk, logger)
 }
 
-func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	snapshot := m.activeSnapshot
+func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	snapshot := r.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
 		logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID)
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
 		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
-		m.activeSnapshot = nil
+		r.activeSnapshot = nil
 		return nil
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
 		logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex)
-		m.activeSnapshot = nil
+		r.activeSnapshot = nil
 		return nil
 	}
 
@@ -193,10 +193,10 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger
 	return nil
 }
 
-func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
+func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
 	select {
 	case <-ctx.Done():
-		m.activeSnapshot = nil
+		r.activeSnapshot = nil
 		return nil
 	default:
 	}
@@ -205,7 +205,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 	for i := 0; i < snapshot.totalChunks; i++ {
 		select {
 		case <-ctx.Done():
-			m.activeSnapshot = nil
+			r.activeSnapshot = nil
 			return nil
 		default:
 		}
@@ -213,7 +213,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 		chunk, ok := snapshot.received[i]
 		if !ok {
 			logger.Error(nil, "missing snapshot chunk", "index", i)
-			m.activeSnapshot = nil
+			r.activeSnapshot = nil
 			return nil
 		}
 		allTargets = append(allTargets, chunk...)
@@ -229,34 +229,34 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 	// a.applyTargets
 
 	// Replay deferred events
-	for _, event := range m.deferredEvents {
+	for _, event := range r.deferredEvents {
 		select {
 		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err := m.applyEvent(ctx, event, logger); err != nil {
+		if err := r.applyEvent(ctx, event, logger); err != nil {
 			return err
 		}
 	}
 
-	m.activeSnapshot = nil
-	m.deferredEvents = nil
+	r.activeSnapshot = nil
+	r.deferredEvents = nil
 	return nil
 }
 
-func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	// If snapshot collecting is active defer events
-	if m.activeSnapshot != nil {
-		m.deferredEvents = append(m.deferredEvents, event)
+	if r.activeSnapshot != nil {
+		r.deferredEvents = append(r.deferredEvents, event)
 		return nil
 	}
 
 	// Apply events
-	return m.applyEvent(ctx, event, logger)
+	return r.applyEvent(ctx, event, logger)
 }
 
-func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
 	case core.EventCreate:
 		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 84f9a6f..65a4cf9 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	}
 
 	// Create target reconciler instance
-	targetReconciler := discovery.NewMessageProcessor(
+	targetReconciler := discovery.NewTargetReconciler(
 		r.Client,
 		r.Scheme,
 		targetSource,

From c268808d67eb8df1d7328c0658b36bd369eda489 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:38:23 -0600
Subject: [PATCH 054/165] moved registry.go to discovery

---
 internal/controller/discovery/{registry => }/registry.go | 2 +-
 internal/controller/targetsource_controller.go           | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)
 rename internal/controller/discovery/{registry => }/registry.go (97%)

diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry.go
similarity index 97%
rename from internal/controller/discovery/registry/registry.go
rename to internal/controller/discovery/registry.go
index f2630e8..0afa2b2 100644
--- a/internal/controller/discovery/registry/registry.go
+++ b/internal/controller/discovery/registry.go
@@ -1,4 +1,4 @@
-package registry
+package discovery
 
 import (
 	"fmt"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 65a4cf9..3b62b6d 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -33,7 +33,6 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/loaders"
 	"github.com/gnmic/operator/internal/controller/discovery/pipeline"
-	"github.com/gnmic/operator/internal/controller/discovery/registry"
 	"github.com/go-logr/logr"
 )
 
@@ -66,7 +65,7 @@ type TargetSourceReconciler struct {
 	BufferSize int
 	ChunkSize  int
 
-	DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
+	DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete

From 02958966b77f80ee3fc1f0e447b98967e54e9c2a Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:39:32 -0600
Subject: [PATCH 055/165] moved supervisor to discovery

---
 .../controller/discovery/{pipeline => }/supervisor.go |  2 +-
 internal/controller/targetsource_controller.go        | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)
 rename internal/controller/discovery/{pipeline => }/supervisor.go (99%)

diff --git a/internal/controller/discovery/pipeline/supervisor.go b/internal/controller/discovery/supervisor.go
similarity index 99%
rename from internal/controller/discovery/pipeline/supervisor.go
rename to internal/controller/discovery/supervisor.go
index 042d305..56fa687 100644
--- a/internal/controller/discovery/pipeline/supervisor.go
+++ b/internal/controller/discovery/supervisor.go
@@ -1,4 +1,4 @@
-package pipeline
+package discovery
 
 import (
 	"context"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 3b62b6d..301e421 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -32,7 +32,6 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery/loaders"
-	"github.com/gnmic/operator/internal/controller/discovery/pipeline"
 	"github.com/go-logr/logr"
 )
 
@@ -171,7 +170,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	loaderConfigured := targetSource.Spec.Provider != nil
 	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
 
-	supervisor := pipeline.NewSupervisor(context.Background())
+	supervisor := discovery.NewSupervisor(context.Background())
 
 	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
 	if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{
@@ -190,9 +189,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	)
 	// Start target reconciler
 	reconcilerReady := make(chan struct{})
-	supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
+	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
 		Name: "target-reconciler",
-		Policy: pipeline.RestartPolicy{
+		Policy: discovery.RestartPolicy{
 			MaxRestarts: pipelineMaxRestarts,
 			Backoff:     pipelineBackoff,
 		},
@@ -221,9 +220,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 			return err
 		}
 
-		supervisor.StartSupervisedComponent(pipeline.ComponentSpec{
+		supervisor.StartSupervisedComponent(discovery.ComponentSpec{
 			Name: "loader",
-			Policy: pipeline.RestartPolicy{
+			Policy: discovery.RestartPolicy{
 				MaxRestarts: pipelineMaxRestarts,
 				Backoff:     pipelineBackoff,
 			},

From 4d32c40fb2e319fa2ff77a9c05f576ba6e0dba4d Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:40:26 -0600
Subject: [PATCH 056/165] moved factory.go to discovery/loaders.go

---
 .../controller/discovery/{loaders/factory.go => loaders.go}    | 2 +-
 internal/controller/targetsource_controller.go                 | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)
 rename internal/controller/discovery/{loaders/factory.go => loaders.go} (97%)

diff --git a/internal/controller/discovery/loaders/factory.go b/internal/controller/discovery/loaders.go
similarity index 97%
rename from internal/controller/discovery/loaders/factory.go
rename to internal/controller/discovery/loaders.go
index 45bf9c1..0d8ddd3 100644
--- a/internal/controller/discovery/loaders/factory.go
+++ b/internal/controller/discovery/loaders.go
@@ -1,4 +1,4 @@
-package loaders
+package discovery
 
 import (
 	"fmt"
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 301e421..9ba2c94 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -31,7 +31,6 @@ import (
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
-	"github.com/gnmic/operator/internal/controller/discovery/loaders"
 	"github.com/go-logr/logr"
 )
 
@@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 
 	// Create loader instance
 	if loaderConfigured {
-		loader, err := loaders.NewLoader(
+		loader, err := discovery.NewLoader(
 			key,
 			targetSource.Spec,
 			core.LoaderConfig{ChunkSize: r.ChunkSize},

From 7671c1a20aa7a48a26cf306c55ef0698c1ec448f Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:54:58 -0600
Subject: [PATCH 057/165] moved send.go to loaders package

---
 .../discovery/loaders/http/loader.go          |  3 ++-
 .../discovery/{core => loaders}/send.go       | 22 ++++++++++---------
 2 files changed, 14 insertions(+), 11 deletions(-)
 rename internal/controller/discovery/{core => loaders}/send.go (67%)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 09bb7d6..1e5fc37 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -10,6 +10,7 @@ import (
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/gnmic/operator/internal/controller/discovery/loaders"
 	"github.com/google/uuid"
 )
 
@@ -66,7 +67,7 @@ func (l *Loader) Start(
 				},
 			}
 
-			if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
+			if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
 				return err
 			}
 		}
diff --git a/internal/controller/discovery/core/send.go b/internal/controller/discovery/loaders/send.go
similarity index 67%
rename from internal/controller/discovery/core/send.go
rename to internal/controller/discovery/loaders/send.go
index f24b50c..1377432 100644
--- a/internal/controller/discovery/core/send.go
+++ b/internal/controller/discovery/loaders/send.go
@@ -1,12 +1,14 @@
-package core
+package loaders
 
 import (
 	"context"
 	"fmt"
+
+	"github.com/gnmic/operator/internal/controller/discovery/core"
 )
 
 // sendMessages sends discovery messages over a channel in a context-aware manner
-func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error {
+func sendMessages(ctx context.Context, out chan<- []core.DiscoveryMessage, messages []core.DiscoveryMessage) error {
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
@@ -30,14 +32,14 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error {
 }
 
 // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots
-func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot {
-	var snapshots []DiscoverySnapshot
+func createDiscoverySnapshots(targets []core.DiscoveredTarget, snapshotID string, chunkSize int) []core.DiscoverySnapshot {
+	var snapshots []core.DiscoverySnapshot
 	totalTargets := len(targets)
 	totalChunks := (totalTargets + chunkSize - 1) / chunkSize
 
 	_ = forEachChunk(totalTargets, chunkSize, func(i, end int) error {
 		chunk := targets[i:end]
-		snapshots = append(snapshots, DiscoverySnapshot{
+		snapshots = append(snapshots, core.DiscoverySnapshot{
 			Targets:     chunk,
 			SnapshotID:  snapshotID,
 			ChunkIndex:  i / chunkSize,
@@ -50,7 +52,7 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu
 }
 
 // SendSnapshot sends discovered targets as a snapshot over a channel in chunks
-func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error {
+func SendSnapshot(ctx context.Context, out chan<- []core.DiscoveryMessage, targets []core.DiscoveredTarget, snapshotID string, chunkSize int) error {
 	if len(targets) == 0 {
 		return fmt.Errorf("no targets in Snapshot")
 	}
@@ -58,7 +60,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []
 	snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize)
 	for _, snapshot := range snapshots {
 		// Convert DiscoverySnapshot to DiscoveryMessage
-		messages := make([]DiscoveryMessage, 1)
+		messages := make([]core.DiscoveryMessage, 1)
 		messages[0] = snapshot
 
 		if err := sendMessages(ctx, out, messages); err != nil {
@@ -69,8 +71,8 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []
 	return nil
 }
 
-func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage {
-	message := make([]DiscoveryMessage, len(events))
+func eventsToMessages(events []core.DiscoveryEvent) []core.DiscoveryMessage {
+	message := make([]core.DiscoveryMessage, len(events))
 	for i, event := range events {
 		message[i] = event
 	}
@@ -78,7 +80,7 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage {
 }
 
 // SendEvents sends discovery messages over channel in a context-aware manner
-func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error {
+func SendEvents(ctx context.Context, out chan<- []core.DiscoveryMessage, events []core.DiscoveryEvent, chunkSize int) error {
 	if len(events) == 0 {
 		return fmt.Errorf("no events to process")
 	}

From 5f1e9cbe91d28e837ff7fbfae4029df45f27c001 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:55:59 -0600
Subject: [PATCH 058/165] eliminated message.go

---
 internal/controller/discovery/core/message.go           | 4 ----
 internal/controller/discovery/core/message_interface.go | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)
 delete mode 100644 internal/controller/discovery/core/message.go

diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go
deleted file mode 100644
index af4f6c1..0000000
--- a/internal/controller/discovery/core/message.go
+++ /dev/null
@@ -1,4 +0,0 @@
-package core
-
-func (DiscoveryEvent) isDiscoveryMessage()    {}
-func (DiscoverySnapshot) isDiscoveryMessage() {}
diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go
index 07b819e..0836bc6 100644
--- a/internal/controller/discovery/core/message_interface.go
+++ b/internal/controller/discovery/core/message_interface.go
@@ -3,3 +3,6 @@ package core
 type DiscoveryMessage interface {
 	isDiscoveryMessage()
 }
+
+func (DiscoveryEvent) isDiscoveryMessage()    {}
+func (DiscoverySnapshot) isDiscoveryMessage() {}

From 6d6753731ca36cdafa5a251e164ed1b70eafd3dc Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 10:56:39 -0600
Subject: [PATCH 059/165] moved const.go to discovery.go

---
 internal/controller/discovery/client.go           | 3 +--
 internal/controller/discovery/{core => }/const.go | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
 rename internal/controller/discovery/{core => }/const.go (81%)

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
index 2deb477..cb02161 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/client.go
@@ -6,7 +6,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-	"github.com/gnmic/operator/internal/controller/discovery/core"
 )
 
 func fetchExistingTargets(
@@ -22,7 +21,7 @@ func fetchExistingTargets(
 		&targetList,
 		client.InNamespace(ts.Namespace),
 		client.MatchingLabels{
-			core.LabelTargetSourceName: ts.Name,
+			LabelTargetSourceName: ts.Name,
 		},
 	)
 	if err != nil {
diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/const.go
similarity index 81%
rename from internal/controller/discovery/core/const.go
rename to internal/controller/discovery/const.go
index 82a5962..ac7a57f 100644
--- a/internal/controller/discovery/core/const.go
+++ b/internal/controller/discovery/const.go
@@ -1,4 +1,4 @@
-package core
+package discovery
 
 const (
 	// Labels

From 391463097c6caab4b89c72de9789efe8b346e8bf Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 11:28:29 -0600
Subject: [PATCH 060/165] renamed core package within targetsource controller

---
 internal/controller/targetsource_controller.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 9ba2c94..e52b02b 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -30,7 +30,7 @@ import (
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery"
-	"github.com/gnmic/operator/internal/controller/discovery/core"
+	discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/go-logr/logr"
 )
 
@@ -63,7 +63,7 @@ type TargetSourceReconciler struct {
 	BufferSize int
 	ChunkSize  int
 
-	DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
+	DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -171,8 +171,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 
 	supervisor := discovery.NewSupervisor(context.Background())
 
-	targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize)
-	if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{
+	targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize)
+	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
 		Channel:        targetChannel,
 		WebhookEnabled: webhookActivated,
 	}); err != nil {
@@ -212,7 +212,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		loader, err := discovery.NewLoader(
 			key,
 			targetSource.Spec,
-			core.LoaderConfig{ChunkSize: r.ChunkSize},
+			discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize},
 		)
 		if err != nil {
 			supervisor.Stop()

From 46a201fc1d9f0dc9cc73825477f789fc3cb3e860 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 11:34:42 -0600
Subject: [PATCH 061/165] changed events to delete / apply

---
 internal/controller/discovery/core/types.go        | 6 ++----
 internal/controller/discovery/target_reconciler.go | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 68c9c7e..2c37fc7 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -15,10 +15,8 @@ type EventAction int
 const (
 	// EventDelete indicates that a target should be removed
 	EventDelete EventAction = iota
-	// EventCreate indicates that a target should be created
-	EventCreate
-	// EventUpdate indicates that a target should be updated
-	EventUpdate
+	// EventApply indicates that a target should be applied (created or updated)
+	EventApply
 )
 
 // DiscoveredTarget represents a target discovered from an external source
diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go
index 4f3711c..86470c6 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -258,12 +258,10 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover
 
 func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
-	case core.EventCreate:
-		logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
-	case core.EventUpdate:
-		logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
 	case core.EventDelete:
 		logger.Info("Would delete target", "name", event.Target.Name)
+	case core.EventApply:
+		logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
 	}
 	return nil
 }

From 7b17f7e77644abff70f5796704e36b10bf03da15 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 11:37:39 -0600
Subject: [PATCH 062/165] moved send.go into separate utils for loaders

---
 internal/controller/discovery/loaders/http/loader.go      | 4 ++--
 internal/controller/discovery/loaders/{ => utils}/send.go | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename internal/controller/discovery/loaders/{ => utils}/send.go (99%)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 1e5fc37..d7d5961 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -10,7 +10,7 @@ import (
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
-	"github.com/gnmic/operator/internal/controller/discovery/loaders"
+	loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils"
 	"github.com/google/uuid"
 )
 
@@ -67,7 +67,7 @@ func (l *Loader) Start(
 				},
 			}
 
-			if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
+			if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
 				return err
 			}
 		}
diff --git a/internal/controller/discovery/loaders/send.go b/internal/controller/discovery/loaders/utils/send.go
similarity index 99%
rename from internal/controller/discovery/loaders/send.go
rename to internal/controller/discovery/loaders/utils/send.go
index 1377432..3cfba8d 100644
--- a/internal/controller/discovery/loaders/send.go
+++ b/internal/controller/discovery/loaders/utils/send.go
@@ -1,4 +1,4 @@
-package loaders
+package utils
 
 import (
 	"context"

From 4540163d4137a27a291846a5960ecf09844bf5f8 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 29 Apr 2026 11:45:43 -0600
Subject: [PATCH 063/165] replaced legacy registry package

---
 cmd/main.go                     | 4 ++--
 internal/apiserver/apiserver.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cmd/main.go b/cmd/main.go
index 4cf6e94..aaf398a 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -42,7 +42,7 @@ import (
 	"github.com/gnmic/operator/internal/apiserver"
 	"github.com/gnmic/operator/internal/controller"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
-	"github.com/gnmic/operator/internal/controller/discovery/registry"
+	"github.com/gnmic/operator/internal/controller/discovery"
 	webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1"
 	//+kubebuilder:scaffold:imports
 )
@@ -86,7 +86,7 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]()
+	discoveryRegistry := discovery.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]()
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go
index a7ca16a..705b277 100644
--- a/internal/apiserver/apiserver.go
+++ b/internal/apiserver/apiserver.go
@@ -6,7 +6,7 @@ import (
 
 	"github.com/gnmic/operator/internal/controller"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
-	"github.com/gnmic/operator/internal/controller/discovery/registry"
+	"github.com/gnmic/operator/internal/controller/discovery"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -14,7 +14,7 @@ type APIServer struct {
 	Server            *http.Server
 	clusterReconciler *controller.ClusterReconciler
 
-	DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
+	DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue]
 }
 
 func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer {

From c728fa2f340066c1f261769ab379ba223e12d62c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 08:21:11 +0000
Subject: [PATCH 064/165] add supervisor restart policy to targetsource spec
 configuration

---
 api/v1alpha1/targetsource_types.go            | 13 ++++-
 api/v1alpha1/zz_generated.deepcopy.go         | 30 ++++++++++++
 .../operator.gnmic.dev_targetsources.yaml     |  7 +++
 internal/controller/discovery/defaults.go     | 12 +++++
 .../controller/targetsource_controller.go     | 49 ++++++++++++-------
 5 files changed, 92 insertions(+), 19 deletions(-)
 create mode 100644 internal/controller/discovery/defaults.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index a936e66..7c8f74c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -26,9 +26,12 @@ type TargetSourceSpec struct {
 	Provider *ProviderSpec `json:"provider"`
 	// +kubebuilder:validation:Optional
 	Webhook WebhookSpec `json:"webhook,omitempty"`
-	//
+	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"`
+
 	// +kubebuilder:validation:MinLength=1
 	TargetProfile string `json:"targetProfile"`
 }
@@ -54,6 +57,14 @@ type ConsulConfig struct {
 	URL string `json:"url,omitempty"`
 }
 
+type RestartPolicySpec struct {
+	// +kubebuilder:validation:Optional
+	MaxRestarts *int `json:"maxRestarts,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	BackoffSeconds *int `json:"backoffSeconds,omitempty"`
+}
+
 // TargetSourceStatus defines the observed state of TargetSource
 type TargetSourceStatus struct {
 	Status       string      `json:"status"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 608d47e..df08573 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -843,6 +843,31 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) {
+	*out = *in
+	if in.MaxRestarts != nil {
+		in, out := &in.MaxRestarts, &out.MaxRestarts
+		*out = new(int)
+		**out = **in
+	}
+	if in.BackoffSeconds != nil {
+		in, out := &in.BackoffSeconds, &out.BackoffSeconds
+		*out = new(int)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec.
+func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(RestartPolicySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) {
 	*out = *in
@@ -1300,6 +1325,11 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.RestartPolicy != nil {
+		in, out := &in.RestartPolicy, &out.RestartPolicy
+		*out = new(RestartPolicySpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec.
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index b385d8e..6464ea2 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -60,6 +60,13 @@ spec:
                 - message: exactly one of the fields in [http consul] must be set
                   rule: '[has(self.http),has(self.consul)].filter(x,x==true).size()
                     == 1'
+              restartPolicy:
+                properties:
+                  backoffSeconds:
+                    type: integer
+                  maxRestarts:
+                    type: integer
+                type: object
               targetLabels:
                 additionalProperties:
                   type: string
diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go
new file mode 100644
index 0000000..dc6f046
--- /dev/null
+++ b/internal/controller/discovery/defaults.go
@@ -0,0 +1,12 @@
+package discovery
+
+import "time"
+
+// DefaultRestartPolicy defines the default restart behavior
+// for the discovery components
+func DefaultRestartPolicy() RestartPolicy {
+	return RestartPolicy{
+		MaxRestarts: 5,
+		Backoff:     3 * time.Second,
+	}
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 06b4fac..fddebda 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -35,11 +35,6 @@ import (
 	"github.com/go-logr/logr"
 )
 
-const (
-	pipelineMaxRestarts = 5
-	pipelineBackoff     = 3 * time.Second
-)
-
 // pipelineHandle represents a controller-owned handle to a running pipeline
 // The controller never manipulates internals; it only invokes cancel()
 type pipelineHandle struct {
@@ -158,6 +153,29 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 	return nil
 }
 
+// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy
+func resolveRestartPolicy(
+	override *gnmicv1alpha1.RestartPolicySpec,
+) discovery.RestartPolicy {
+	defaults := discovery.DefaultRestartPolicy()
+
+	if override == nil {
+		return defaults
+	}
+
+	resolved := defaults
+
+	if override.MaxRestarts != nil {
+		resolved.MaxRestarts = *override.MaxRestarts
+	}
+
+	if override.BackoffSeconds != nil {
+		resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second
+	}
+
+	return resolved
+}
+
 // startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource
 //
 // Pipeline semantics:
@@ -168,6 +186,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
 	loaderConfigured := targetSource.Spec.Provider != nil
 	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
+	restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy)
 
 	supervisor := discovery.NewSupervisor(context.Background())
 
@@ -187,22 +206,19 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		targetChannel,
 	)
 	// Start target reconciler
-	reconcilerReady := make(chan struct{})
+	targetReconcilerReady := make(chan struct{})
 	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
-		Name: "target-reconciler",
-		Policy: discovery.RestartPolicy{
-			MaxRestarts: pipelineMaxRestarts,
-			Backoff:     pipelineBackoff,
-		},
+		Name:               "target-reconciler",
+		Policy:             restartPolicy,
 		EscalatesOnFailure: true,
 		Run: func(ctx context.Context) error {
-			close(reconcilerReady) // Signals that reconciler started successfully
+			close(targetReconcilerReady) // Signals that reconciler started successfully
 			return targetReconciler.Run(ctx)
 		},
 	})
 	// Wait for reconciler to be ready before starting loader
 	select {
-	case <-reconcilerReady:
+	case <-targetReconcilerReady:
 	case <-supervisor.Done():
 		return nil
 	}
@@ -220,11 +236,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		}
 
 		supervisor.StartSupervisedComponent(discovery.ComponentSpec{
-			Name: "loader",
-			Policy: discovery.RestartPolicy{
-				MaxRestarts: pipelineMaxRestarts,
-				Backoff:     pipelineBackoff,
-			},
+			Name:               "loader",
+			Policy:             restartPolicy,
 			EscalatesOnFailure: !webhookActivated,
 			Run: func(ctx context.Context) error {
 				return loader.Start(ctx, key, targetSource.Spec, targetChannel)

From 589bc9f8cf0643af82f40c4e126ec2e72fc7e67e Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 08:31:37 +0000
Subject: [PATCH 065/165] add targetsource example for lab

---
 lab/dev/resources/targetsources/cts1.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 lab/dev/resources/targetsources/cts1.yml

diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml
new file mode 100644
index 0000000..682930c
--- /dev/null
+++ b/lab/dev/resources/targetsources/cts1.yml
@@ -0,0 +1,18 @@
+apiVersion: operator.gnmic.dev/v1alpha1
+kind: TargetSource
+metadata:
+  name: http-discovery
+spec:
+  provider:
+    http:
+      url: http://srbsci-121:8081/api/dcim/devices/?export=test
+  webhook:
+    enabled: true
+  targetLabels:
+    source: inventory
+    site: siteA
+    tags: "inventory,siteA,http-discovery"
+  restartPolicy:
+    maxRestarts: 2
+    backoffSeconds: 4
+  targetProfile: eos
\ No newline at end of file

From a5dde06e8df6dafe7a72f5650e35584ef22b2662 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 08:37:04 +0000
Subject: [PATCH 066/165] remove targetsource example to not add unnecassary
 logging to main

---
 lab/dev/resources/targetsources/cts1.yml | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 lab/dev/resources/targetsources/cts1.yml

diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml
deleted file mode 100644
index 682930c..0000000
--- a/lab/dev/resources/targetsources/cts1.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-apiVersion: operator.gnmic.dev/v1alpha1
-kind: TargetSource
-metadata:
-  name: http-discovery
-spec:
-  provider:
-    http:
-      url: http://srbsci-121:8081/api/dcim/devices/?export=test
-  webhook:
-    enabled: true
-  targetLabels:
-    source: inventory
-    site: siteA
-    tags: "inventory,siteA,http-discovery"
-  restartPolicy:
-    maxRestarts: 2
-    backoffSeconds: 4
-  targetProfile: eos
\ No newline at end of file

From 4be9c27a8a547ded1d79f9d6a542da2ad148fe2b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 08:42:06 +0000
Subject: [PATCH 067/165] update gitignore to not push targetsources in order
 to prevent logging in main branch

---
 .gitignore | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 29d31af..7515fa3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,4 +33,9 @@ notes/
 docs/public
 docs/resources/_gen/
 docs/.hugo_build.lock
-test/integration/clab-*
\ No newline at end of file
+test/integration/clab-*
+
+# Only for development and testing purposes
+# To be removed after development of targetsource
+# ignored in order to not add unnecassary logging messages
+lab/dev/resources/targetsources
\ No newline at end of file

From 7337541e70e7bbf0867eb2a1e66a7c6ffacc3799 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 09:56:06 +0000
Subject: [PATCH 068/165] add component info to logging

---
 internal/controller/discovery/target_reconciler.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go
index 86470c6..3a9f327 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -50,6 +50,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error {
 
 	logger := log.FromContext(r.ctx).
 		WithValues(
+			"component", "target reconciler",
 			"name", r.targetSource.Name,
 			"namespace", r.targetSource.Namespace,
 		)

From 41d54987415e9dc7c31096b12eff8e9022680405 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 10:02:20 +0000
Subject: [PATCH 069/165] make snapshot id a bit smaller

---
 internal/controller/discovery/loaders/http/loader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index bc87855..84cb70b 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -77,7 +77,7 @@ func (l *Loader) Start(
 			return
 		}
 
-		snapshotID := fmt.Sprintf("snapshot-%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString())
+		snapshotID := fmt.Sprintf("%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString())
 		if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
 			logger.Error(err, "failed to send discovery snapshot")
 			return

From 3ec3203efa7a6e484902ba17dd4eed9085c52277 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 12:21:19 +0000
Subject: [PATCH 070/165] if context is canceled return with ctx.Err() not a
 clean exit

---
 internal/controller/discovery/target_reconciler.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go
index 3a9f327..2f623c3 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -73,7 +73,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error {
 
 		for len(r.queue) > 0 {
 			if ctx.Err() != nil {
-				return nil // why return nil?
+				return ctx.Err()
 			}
 
 			msg := r.queue[0]

From 0eaffdcfc63c32bd6d63e46f3081a12015bd76e4 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 12:34:18 +0000
Subject: [PATCH 071/165] applied kubebuilder best-practise logging

---
 .../discovery/loaders/http/loader.go          |  6 +-
 internal/controller/discovery/supervisor.go   | 22 ++++---
 .../controller/discovery/target_reconciler.go | 59 ++++++++++++++-----
 .../controller/targetsource_controller.go     | 31 ++++++++--
 4 files changed, 89 insertions(+), 29 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index d7d5961..67c61e1 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -39,7 +39,11 @@ func (l *Loader) Start(
 		"targetsource", targetsourceNN,
 	)
 
-	logger.Info("HTTP loader started")
+	logger.Info(
+		"HTTP loader started",
+		"targetsource", targetsourceNN.Name,
+		"namespace", targetsourceNN.Namespace,
+	)
 
 	// Only for debugging: emit a static snapshot every 30 seconds
 	ticker := time.NewTicker(30 * time.Second)
diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
index 56fa687..22ec227 100644
--- a/internal/controller/discovery/supervisor.go
+++ b/internal/controller/discovery/supervisor.go
@@ -78,7 +78,10 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) {
 		failures := 0
 
 		for {
-			logger.Info("starting component")
+			logger.Info(
+				"Starting supervised component",
+				"component", component.Name,
+			)
 			err := component.Run(s.ctx)
 
 			if s.ctx.Err() != nil {
@@ -87,21 +90,26 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) {
 			}
 
 			failures++
-			logger.Error(err,
-				"component failed to run",
+			logger.Error(
+				err,
+				"Supervised component failed",
+				"component", component.Name,
 				"attempt", failures,
-				"max", component.Policy.MaxRestarts,
+				"maxRestarts", component.Policy.MaxRestarts,
 			)
 
 			if failures >= component.Policy.MaxRestarts {
 				if component.EscalatesOnFailure {
-					logger.Error(err,
-						"component permanently failed; shutting down pipeline",
+					logger.Error(
+						err,
+						"Supervised component permanently failed; stopped discovery pipeline",
+						"component", component.Name,
 					)
 					s.Stop()
 				} else {
 					logger.Info(
-						"optional component permanently failed; continuing without it",
+						"Optional component permanently failed; continuing without it",
+						"component", component.Name,
 					)
 				}
 				return
diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go
index 2f623c3..67d9611 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -54,20 +54,30 @@ func (r *TargetReconciler) Run(ctx context.Context) error {
 			"name", r.targetSource.Name,
 			"namespace", r.targetSource.Namespace,
 		)
-	logger.Info("target reconciler started")
+	logger.Info(
+		"Target reconciler started",
+		"targetsource", r.targetSource.Name,
+		"namespace", r.targetSource.Namespace,
+	)
 
 	for r.ctx.Err() == nil {
 		select {
 		case batch, ok := <-r.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
-				logger.Info("input channel closed, stopping target reconciler")
+				logger.Info(
+					"Input channel closed; stopping target reconciler",
+					"targetsource", r.targetSource.Name,
+				)
 				return nil
 			}
 			r.queue = append(r.queue, batch...)
 
 		case <-ctx.Done():
-			logger.Info("context canceled, stopping target reconciler")
+			logger.Info(
+				"Context was canceled; stopping target reconciler",
+				"targetsource", r.targetSource.Name,
+			)
 			return nil
 		}
 
@@ -103,23 +113,24 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc
 	case core.DiscoverySnapshot:
 		// Collect snapshot chunks
 		logger.Info(
-			"received snapshot chunk",
+			"Received discovery snapshot chunk",
 			"snapshotID", msg.SnapshotID,
-			"index", msg.ChunkIndex,
-			"targetCount", len(msg.Targets),
+			"chunkIndex", msg.ChunkIndex,
+			"targets", len(msg.Targets),
 		)
 		return r.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
 		logger.Info(
-			"received discovery event",
+			"Received discovery event",
+			"event", msg.Event,
 			"target", msg.Target.Name,
 		)
 		return r.processEvent(ctx, msg, logger)
 
 	default:
-		return fmt.Errorf("unknonw discovery message type %T", msg)
+		return fmt.Errorf("Unknown discovery message type %T", msg)
 	}
 }
 
@@ -142,7 +153,7 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco
 			// If a new snapshot is started before the old one completed
 			// the old one can be discarded
 			logger.Info(
-				"discarding incomplete snapshot",
+				"Discarded incomplete discovery snapshot",
 				"snapshotID", snapshot.snapshotID,
 			)
 		}
@@ -172,7 +183,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 	snapshot := r.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
-		logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID)
+		logger.Error(
+			nil,
+			"Snapshot totalChunks mismatch",
+			"snapshotID", snapshot.snapshotID,
+		)
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
 		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
@@ -180,7 +195,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 		return nil
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
-		logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex)
+		logger.Error(
+			nil,
+			"Duplicate snapshot chunk received",
+			"chunkIndex", chunk.ChunkIndex,
+		)
 		r.activeSnapshot = nil
 		return nil
 	}
@@ -221,9 +240,9 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot
 	}
 
 	logger.Info(
-		"applying snapshot",
+		"Applying discovery snapshot",
 		"snapshotID", snapshot.snapshotID,
-		"targetCount", len(allTargets),
+		"targets", len(allTargets),
 	)
 
 	// apply all targets
@@ -260,9 +279,19 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover
 func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
 	case core.EventDelete:
-		logger.Info("Would delete target", "name", event.Target.Name)
+		logger.Info(
+			"Deleting Target",
+			"target", event.Target.Name,
+			"targetsource", r.targetSource.Name,
+		)
 	case core.EventApply:
-		logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels)
+		logger.Info(
+			"Applying Target",
+			"target", event.Target.Name,
+			"address", event.Target.Address,
+			"labels", event.Target.Labels,
+			"targetsource", r.targetSource.Name,
+		)
 	}
 	return nil
 }
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index fddebda..c82ad08 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -71,13 +71,20 @@ type TargetSourceReconciler struct {
 // move the current state of the cluster closer to the desired state.
 func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx).
-		WithName("targetsource controller").
-		WithValues("targetsource", req.NamespacedName)
+		WithName("targetsource-controller").
+		WithValues(
+			"targetsource", req.NamespacedName.Name,
+			"namespace", req.NamespacedName.Namespace,
+		)
 
 	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
 	// If the TargetSource no longer exists, ensure runtime cleanup
 	if apierrors.IsNotFound(err) {
-		logger.Info("TargetSource not found; stopping discovery pipeline")
+		logger.Info(
+			"TargetSource not found; stopped discovery pipeline",
+			"targetsource", req.NamespacedName.Name,
+			"namespace", req.NamespacedName.Namespace,
+		)
 		r.stopDiscoveryPipeline(req.NamespacedName)
 		return ctrl.Result{}, nil
 	} else if err != nil {
@@ -100,7 +107,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	logger.Info("Discovery pipeline started")
+	logger.Info(
+		"Started discovery pipeline",
+		"targetsource", req.NamespacedName.Name,
+		"namespace", req.NamespacedName.Namespace,
+	)
 	return ctrl.Result{}, nil
 }
 
@@ -124,7 +135,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo
 // reconcileDeletion stops the discovery pipeline and removes the finalizer
 func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
-	logger.Info("TargetSource is being deleted, stopping pipeline", "name", key)
+	logger.Info(
+		"TargetSource was marked for deletion; stopping discovery pipeline",
+		"targetsource", key.Name,
+		"namespace", key.Namespace,
+	)
 
 	r.stopDiscoveryPipeline(key)
 
@@ -250,10 +265,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		<-supervisor.Done()
 		supervisor.Wait() // Wait for components to exit
 
-		logger.Info("Pipeline stopped; cleaning up")
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
 		r.stopDiscoveryPipeline(key)
+		logger.Info(
+			"Discovery pipeline stopped; cleaned up resources",
+			"targetsource", key.Name,
+			"namespace", key.Namespace,
+		)
 	}()
 
 	r.mu.Lock()

From e447b3bd18867bd9d5cd1df19262dec14d0785dd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 12:36:18 +0000
Subject: [PATCH 072/165] improved logging

---
 .../controller/discovery/loaders/http/loader.go | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index c95f1b3..4b58223 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -77,13 +77,20 @@ func (l *Loader) Start(
 			spec.Provider.HTTP.Token,
 		)
 		if err != nil {
-			logger.Error(err, "failed to fetch targets from HTTP endpoint")
+			logger.Error(
+				err,
+				"Failed to fetch targets from HTTP endpoint",
+				"url", spec.Provider.HTTP.URL,
+			)
 			return
 		}
 
 		snapshotID := fmt.Sprintf("%s-%s-%s", targetsourceNN.Namespace, targetsourceNN.Name, uuid.NewString())
 		if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
-			logger.Error(err, "failed to send discovery snapshot")
+			logger.Error(
+				err,
+				"Failed to send discovery snapshot",
+			)
 			return
 		}
 	}
@@ -95,7 +102,11 @@ func (l *Loader) Start(
 	for {
 		select {
 		case <-ctx.Done():
-			logger.Info("HTTP loader stopped")
+			logger.Info(
+				"HTTP loader stopped",
+				"targetsource", targetsourceNN.Name,
+				"namespace", targetsourceNN.Namespace,
+			)
 			return nil
 
 		case <-ticker.C:

From fca37e0e87076946731fe0918846d75ab20d0356 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 12:39:48 +0000
Subject: [PATCH 073/165] improved logging

---
 .../discovery/loaders/http/loader.go          | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 4b58223..95470fc 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -44,15 +44,12 @@ func (l *Loader) Start(
 	logger := log.FromContext(ctx).WithValues(
 		"component", "loader",
 		"name", l.Name(),
-		"targetsource", targetsourceNN,
-	)
-
-	logger.Info(
-		"HTTP loader started",
 		"targetsource", targetsourceNN.Name,
 		"namespace", targetsourceNN.Namespace,
 	)
 
+	logger.Info("HTTP loader started")
+
 	// Input Validation of spec
 	if spec.Provider == nil || spec.Provider.HTTP == nil {
 		return errors.New("HTTP loader requires spec.provider.http to be set")
@@ -66,7 +63,11 @@ func (l *Loader) Start(
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
-	logger.Info("HTTP pull loader started", "interval", interval.String())
+	logger.Info(
+		"HTTP polling discovery started",
+		"interval", interval.String(),
+		"url", spec.Provider.HTTP.URL,
+	)
 
 	// helper function to fetch targets and emit discovery messages
 	fetchAndEmit := func() {
@@ -90,9 +91,17 @@ func (l *Loader) Start(
 			logger.Error(
 				err,
 				"Failed to send discovery snapshot",
+				"snapshotID", snapshotID,
+				"targets", len(targets),
 			)
 			return
 		}
+
+		logger.Info(
+			"Discovery snapshot sent",
+			"snapshotID", snapshotID,
+			"targets", len(targets),
+		)
 	}
 
 	// Immediate fetch on startup
@@ -102,11 +111,7 @@ func (l *Loader) Start(
 	for {
 		select {
 		case <-ctx.Done():
-			logger.Info(
-				"HTTP loader stopped",
-				"targetsource", targetsourceNN.Name,
-				"namespace", targetsourceNN.Namespace,
-			)
+			logger.Info("HTTP loader stopped")
 			return nil
 
 		case <-ticker.C:

From fd4abe7f086416c0bd4b52249a03625ac6d72124 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 12:55:58 +0000
Subject: [PATCH 074/165] improved logging

---
 .../controller/discovery/target_reconciler.go | 34 +++++++++----------
 .../controller/targetsource_controller.go     | 29 ++++++++--------
 2 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go
index 67d9611..39382ab 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/target_reconciler.go
@@ -48,36 +48,26 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T
 func (r *TargetReconciler) Run(ctx context.Context) error {
 	r.ctx = ctx
 
-	logger := log.FromContext(r.ctx).
-		WithValues(
-			"component", "target reconciler",
-			"name", r.targetSource.Name,
-			"namespace", r.targetSource.Namespace,
-		)
-	logger.Info(
-		"Target reconciler started",
+	logger := log.FromContext(ctx).WithValues(
+		"component", "target-reconciler",
 		"targetsource", r.targetSource.Name,
 		"namespace", r.targetSource.Namespace,
 	)
 
+	logger.Info("Target reconciler started")
+
 	for r.ctx.Err() == nil {
 		select {
 		case batch, ok := <-r.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
-				logger.Info(
-					"Input channel closed; stopping target reconciler",
-					"targetsource", r.targetSource.Name,
-				)
+				logger.Info("Input channel closed; stopping target reconciler")
 				return nil
 			}
 			r.queue = append(r.queue, batch...)
 
 		case <-ctx.Done():
-			logger.Info(
-				"Context was canceled; stopping target reconciler",
-				"targetsource", r.targetSource.Name,
-			)
+			logger.Info("Context was canceled; stopping target reconciler")
 			return nil
 		}
 
@@ -190,7 +180,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 		)
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
-		logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex)
+		logger.Error(
+			nil,
+			"Snapshot chunk index out of range",
+			"chunkIndex", chunk.ChunkIndex,
+		)
 		r.activeSnapshot = nil
 		return nil
 	}
@@ -232,7 +226,11 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot
 
 		chunk, ok := snapshot.received[i]
 		if !ok {
-			logger.Error(nil, "missing snapshot chunk", "index", i)
+			logger.Error(
+				nil,
+				"Missing snapshot chunk",
+				"chunkIndex", i,
+			)
 			r.activeSnapshot = nil
 			return nil
 		}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index c82ad08..f36d47d 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -80,11 +80,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
 	// If the TargetSource no longer exists, ensure runtime cleanup
 	if apierrors.IsNotFound(err) {
-		logger.Info(
-			"TargetSource not found; stopped discovery pipeline",
-			"targetsource", req.NamespacedName.Name,
-			"namespace", req.NamespacedName.Namespace,
-		)
+		logger.Info("TargetSource not found; stopped discovery pipeline")
 		r.stopDiscoveryPipeline(req.NamespacedName)
 		return ctrl.Result{}, nil
 	} else if err != nil {
@@ -100,6 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	if r.hasPipelineRunning(req.NamespacedName) {
+		logger.Info("Discovery pipeline already running; reconciliation completed")
 		return ctrl.Result{}, nil
 	}
 
@@ -107,11 +104,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	logger.Info(
-		"Started discovery pipeline",
-		"targetsource", req.NamespacedName.Name,
-		"namespace", req.NamespacedName.Namespace,
-	)
+	logger.Info("Started discovery pipeline")
 	return ctrl.Result{}, nil
 }
 
@@ -134,13 +127,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo
 
 // reconcileDeletion stops the discovery pipeline and removes the finalizer
 func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
-	logger := log.FromContext(ctx)
-	logger.Info(
-		"TargetSource was marked for deletion; stopping discovery pipeline",
+	logger := log.FromContext(ctx).WithValues(
 		"targetsource", key.Name,
 		"namespace", key.Namespace,
 	)
-
+	logger.Info("TargetSource was marked for deletion; stopping discovery pipeline")
 	r.stopDiscoveryPipeline(key)
 
 	// Remove finalizer if exists
@@ -149,6 +140,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type
 		if err := r.Update(ctx, targetSource); err != nil {
 			return ctrl.Result{}, err
 		}
+
+		logger.Info("Removed TargetSource finalizer")
 	}
 
 	return ctrl.Result{}, nil
@@ -165,6 +158,12 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 		return err
 	}
 
+	log.FromContext(ctx).Info(
+		"Added TargetSource finalizer",
+		"targetsource", targetSource.Name,
+		"namespace", targetSource.Namespace,
+	)
+
 	return nil
 }
 
@@ -234,7 +233,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	// Wait for reconciler to be ready before starting loader
 	select {
 	case <-targetReconcilerReady:
+		logger.Info("Target reconciler started")
 	case <-supervisor.Done():
+		logger.Info("Supervisor stopped before target reconciler became ready")
 		return nil
 	}
 

From a6bc11447919eac352164b959bcac482c2b0a115 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 13:23:47 +0000
Subject: [PATCH 075/165] simplified pipeline context handling

---
 internal/controller/discovery/core/types.go   | 12 +++-
 internal/controller/discovery/registry.go     |  8 +++
 .../controller/targetsource_controller.go     | 68 ++++++-------------
 3 files changed, 38 insertions(+), 50 deletions(-)

diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 2c37fc7..2f89fdf 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -1,8 +1,18 @@
 package core
 
+import "context"
+
+// DiscoveryRegistryValue represents the controller-owned runtime state
+// of a discovery pipeline for a single TargetSource
 type DiscoveryRegistryValue struct {
-	Channel        chan<- []DiscoveryMessage
+	// Channel is the outbound communication channel used by discovery
+	// components (loaders, webhooks, etc.) to emit discovery messages
+	Channel chan<- []DiscoveryMessage
+	// WebhookEnabled indicates whether webhook-based discovery is enabled
+	// for this TargetSource
 	WebhookEnabled bool
+	// Stop cancels the discovery pipeline associated with this registry entry
+	Stop context.CancelFunc
 }
 
 type LoaderConfig struct {
diff --git a/internal/controller/discovery/registry.go b/internal/controller/discovery/registry.go
index 0afa2b2..2193665 100644
--- a/internal/controller/discovery/registry.go
+++ b/internal/controller/discovery/registry.go
@@ -39,3 +39,11 @@ func (r *Registry[K, V]) Get(key K) (V, bool) {
 	r.mu.RUnlock()
 	return value, ok
 }
+
+func (r *Registry[K, V]) Exists(key K) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	_, exists := r.m[key]
+	return exists
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index f36d47d..dca0570 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -18,7 +18,6 @@ package controller
 
 import (
 	"context"
-	"sync"
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -35,12 +34,6 @@ import (
 	"github.com/go-logr/logr"
 )
 
-// pipelineHandle represents a controller-owned handle to a running pipeline
-// The controller never manipulates internals; it only invokes cancel()
-type pipelineHandle struct {
-	cancel context.CancelFunc
-}
-
 // TargetSourceReconciler reconciles a TargetSource object
 //
 // Responsibilities:
@@ -52,14 +45,13 @@ type TargetSourceReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 
-	mu sync.Mutex
-	// runningPipelines tracks currently active pipelines by NamespacedName
-	runningPipelines map[types.NamespacedName]pipelineHandle
-
 	BufferSize int
 	ChunkSize  int
 
-	DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue]
+	DiscoveryRegistry *discovery.Registry[
+		types.NamespacedName,
+		discoveryTypes.DiscoveryRegistryValue,
+	]
 }
 
 // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete
@@ -80,8 +72,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
 	// If the TargetSource no longer exists, ensure runtime cleanup
 	if apierrors.IsNotFound(err) {
+		if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok {
+			pipeline.Stop()
+			r.DiscoveryRegistry.Unregister(req.NamespacedName)
+		}
 		logger.Info("TargetSource not found; stopped discovery pipeline")
-		r.stopDiscoveryPipeline(req.NamespacedName)
 		return ctrl.Result{}, nil
 	} else if err != nil {
 		return ctrl.Result{}, err
@@ -95,7 +90,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	if r.hasPipelineRunning(req.NamespacedName) {
+	if r.DiscoveryRegistry.Exists(req.NamespacedName) {
 		logger.Info("Discovery pipeline already running; reconciliation completed")
 		return ctrl.Result{}, nil
 	}
@@ -117,14 +112,6 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type
 	return &targetSource, nil
 }
 
-// hasPipelineRunning checks if a discovery pipeline is already running for the given key
-func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	_, exists := r.runningPipelines[key]
-	return exists
-}
-
 // reconcileDeletion stops the discovery pipeline and removes the finalizer
 func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
 	logger := log.FromContext(ctx).WithValues(
@@ -132,7 +119,10 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type
 		"namespace", key.Namespace,
 	)
 	logger.Info("TargetSource was marked for deletion; stopping discovery pipeline")
-	r.stopDiscoveryPipeline(key)
+	if pipeline, ok := r.DiscoveryRegistry.Get(key); ok {
+		pipeline.Stop()
+		r.DiscoveryRegistry.Unregister(key)
+	}
 
 	// Remove finalizer if exists
 	if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) {
@@ -197,7 +187,11 @@ func resolveRestartPolicy(
 // 2. loader is optional and conditional on spec
 // 3. Permanent failure of required components shuts down the pipeline
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
-func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error {
+func (r *TargetSourceReconciler) startDiscoveryPipeline(
+	key types.NamespacedName,
+	targetSource *gnmicv1alpha1.TargetSource,
+	logger logr.Logger,
+) error {
 	loaderConfigured := targetSource.Spec.Provider != nil
 	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
 	restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy)
@@ -208,6 +202,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
 		Channel:        targetChannel,
 		WebhookEnabled: webhookActivated,
+		Stop:           supervisor.Stop,
 	}); err != nil {
 		return err
 	}
@@ -268,7 +263,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
-		r.stopDiscoveryPipeline(key)
 		logger.Info(
 			"Discovery pipeline stopped; cleaned up resources",
 			"targetsource", key.Name,
@@ -276,35 +270,11 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName
 		)
 	}()
 
-	r.mu.Lock()
-	r.runningPipelines[key] = pipelineHandle{
-		cancel: func() {
-			supervisor.Stop()
-		},
-	}
-	r.mu.Unlock()
-
 	return nil
 }
 
-// stopDiscoveryPipeline stops and removes a running discovery pipeline
-func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) {
-	r.mu.Lock()
-	running, ok := r.runningPipelines[key]
-	if ok {
-		delete(r.runningPipelines, key)
-	}
-	r.mu.Unlock()
-
-	if ok {
-		running.cancel()
-	}
-}
-
 // SetupWithManager sets up the controller with the Manager.
 func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.runningPipelines = make(map[types.NamespacedName]pipelineHandle)
-
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&gnmicv1alpha1.TargetSource{}).
 		Named("targetsource").

From 54c41fdf1b68f5168dad66c7cb5c7adca29016ed Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 30 Apr 2026 14:36:23 +0000
Subject: [PATCH 076/165] add timeout as a const

---
 internal/controller/discovery/loaders/http/loader.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 95470fc..9a7a949 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -18,7 +18,8 @@ import (
 )
 
 const (
-	defaultPollInterval = 30 * time.Second
+	defaultPollInterval   = 30 * time.Second
+	defaultTimeoutSeconds = 30
 )
 
 // Loader implements the HTTP pull discovery mechanism
@@ -56,9 +57,8 @@ func (l *Loader) Start(
 	}
 
 	client := &http.Client{
-		Timeout: 30 * time.Second,
+		Timeout: defaultTimeoutSeconds * time.Second,
 	}
-
 	interval := defaultPollInterval
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()

From 535ee49438fb9f4a3b45449a8321d6ab92966e42 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 07:06:36 +0000
Subject: [PATCH 077/165] rename target reconciler to message processor

---
 ...get_reconciler.go => message_processor.go} | 108 +++++++++---------
 .../controller/targetsource_controller.go     |   2 +-
 2 files changed, 55 insertions(+), 55 deletions(-)
 rename internal/controller/discovery/{target_reconciler.go => message_processor.go} (68%)

diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/message_processor.go
similarity index 68%
rename from internal/controller/discovery/target_reconciler.go
rename to internal/controller/discovery/message_processor.go
index 39382ab..ed66940 100644
--- a/internal/controller/discovery/target_reconciler.go
+++ b/internal/controller/discovery/message_processor.go
@@ -20,8 +20,8 @@ type snapshotBuffer struct {
 	complete    bool
 }
 
-// TargetReconciler consumes discovered targets and applies them to Kubernetes
-type TargetReconciler struct {
+// MessageProcessor consumes discovery messages and applies them to Kubernetes
+type MessageProcessor struct {
 	ctx            context.Context
 	client         client.Client
 	scheme         *runtime.Scheme
@@ -33,9 +33,9 @@ type TargetReconciler struct {
 	deferredEvents []core.DiscoveryEvent
 }
 
-// NewTargetReconciler wires a TargetReconciler instance
-func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler {
-	return &TargetReconciler{
+// NewMessageProcessor wires a MessageProcessor instance
+func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor {
+	return &MessageProcessor{
 		client:       c,
 		scheme:       s,
 		targetSource: ts,
@@ -45,41 +45,41 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T
 
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
-func (r *TargetReconciler) Run(ctx context.Context) error {
-	r.ctx = ctx
+func (m *MessageProcessor) Run(ctx context.Context) error {
+	m.ctx = ctx
 
 	logger := log.FromContext(ctx).WithValues(
-		"component", "target-reconciler",
-		"targetsource", r.targetSource.Name,
-		"namespace", r.targetSource.Namespace,
+		"component", "message-processor",
+		"targetsource", m.targetSource.Name,
+		"namespace", m.targetSource.Namespace,
 	)
 
-	logger.Info("Target reconciler started")
+	logger.Info("Message processor started")
 
-	for r.ctx.Err() == nil {
+	for m.ctx.Err() == nil {
 		select {
-		case batch, ok := <-r.in:
+		case batch, ok := <-m.in:
 			if !ok {
 				// Channel closed, pipeline is shutting down
-				logger.Info("Input channel closed; stopping target reconciler")
+				logger.Info("Input channel closed; stopping message processor")
 				return nil
 			}
-			r.queue = append(r.queue, batch...)
+			m.queue = append(m.queue, batch...)
 
 		case <-ctx.Done():
-			logger.Info("Context was canceled; stopping target reconciler")
+			logger.Info("Context was canceled; stopping message processor")
 			return nil
 		}
 
-		for len(r.queue) > 0 {
+		for len(m.queue) > 0 {
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
 
-			msg := r.queue[0]
-			r.queue = r.queue[1:]
+			msg := m.queue[0]
+			m.queue = m.queue[1:]
 
-			if err := r.processMessage(r.ctx, msg, logger); err != nil {
+			if err := m.processMessage(m.ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?
@@ -89,11 +89,11 @@ func (r *TargetReconciler) Run(ctx context.Context) error {
 		}
 	}
 
-	logger.Info("target reconciler stopped")
+	logger.Info("Message processor stopped")
 	return nil
 }
 
-func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
+func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
@@ -108,7 +108,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc
 			"chunkIndex", msg.ChunkIndex,
 			"targets", len(msg.Targets),
 		)
-		return r.processSnapshot(ctx, msg, logger)
+		return m.processSnapshot(ctx, msg, logger)
 
 	case core.DiscoveryEvent:
 		// Process individual event-driven update
@@ -117,7 +117,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc
 			"event", msg.Event,
 			"target", msg.Target.Name,
 		)
-		return r.processEvent(ctx, msg, logger)
+		return m.processEvent(ctx, msg, logger)
 
 	default:
 		return fmt.Errorf("Unknown discovery message type %T", msg)
@@ -125,18 +125,18 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc
 }
 
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
-func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	if r.activeSnapshot == nil {
-		r.startNewSnapshot(chunk, logger)
+func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	if m.activeSnapshot == nil {
+		m.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	snapshot := r.activeSnapshot
+	snapshot := m.activeSnapshot
 	// Check if a new snapshot arrived
 	if snapshot.snapshotID != chunk.SnapshotID {
 		// If current snapshot is complete apply it first
 		if snapshot.complete {
-			if err := r.applySnapshot(ctx, snapshot, logger); err != nil {
+			if err := m.applySnapshot(ctx, snapshot, logger); err != nil {
 				return err
 			}
 		} else {
@@ -149,28 +149,28 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco
 		}
 
 		// Start collecting the new snapshot
-		r.startNewSnapshot(chunk, logger)
+		m.startNewSnapshot(chunk, logger)
 		return nil
 	}
 
-	return r.collectSnapshot(chunk, logger)
+	return m.collectSnapshot(chunk, logger)
 }
 
-func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
-	r.activeSnapshot = &snapshotBuffer{
+func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+	m.activeSnapshot = &snapshotBuffer{
 		snapshotID:  chunk.SnapshotID,
 		totalChunks: chunk.TotalChunks,
 		received:    make(map[int][]core.DiscoveredTarget),
 		complete:    false,
 	}
 	// Delete buffered events that will be current with new snapshot
-	r.deferredEvents = nil
+	m.deferredEvents = nil
 
-	r.collectSnapshot(chunk, logger)
+	m.collectSnapshot(chunk, logger)
 }
 
-func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
-	snapshot := r.activeSnapshot
+func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+	snapshot := m.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
 		logger.Error(
@@ -185,7 +185,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 			"Snapshot chunk index out of range",
 			"chunkIndex", chunk.ChunkIndex,
 		)
-		r.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
@@ -194,7 +194,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 			"Duplicate snapshot chunk received",
 			"chunkIndex", chunk.ChunkIndex,
 		)
-		r.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	}
 
@@ -207,10 +207,10 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger
 	return nil
 }
 
-func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
+func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
 	select {
 	case <-ctx.Done():
-		r.activeSnapshot = nil
+		m.activeSnapshot = nil
 		return nil
 	default:
 	}
@@ -219,7 +219,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot
 	for i := 0; i < snapshot.totalChunks; i++ {
 		select {
 		case <-ctx.Done():
-			r.activeSnapshot = nil
+			m.activeSnapshot = nil
 			return nil
 		default:
 		}
@@ -231,7 +231,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot
 				"Missing snapshot chunk",
 				"chunkIndex", i,
 			)
-			r.activeSnapshot = nil
+			m.activeSnapshot = nil
 			return nil
 		}
 		allTargets = append(allTargets, chunk...)
@@ -247,40 +247,40 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot
 	// a.applyTargets
 
 	// Replay deferred events
-	for _, event := range r.deferredEvents {
+	for _, event := range m.deferredEvents {
 		select {
 		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err := r.applyEvent(ctx, event, logger); err != nil {
+		if err := m.applyEvent(ctx, event, logger); err != nil {
 			return err
 		}
 	}
 
-	r.activeSnapshot = nil
-	r.deferredEvents = nil
+	m.activeSnapshot = nil
+	m.deferredEvents = nil
 	return nil
 }
 
-func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	// If snapshot collecting is active defer events
-	if r.activeSnapshot != nil {
-		r.deferredEvents = append(r.deferredEvents, event)
+	if m.activeSnapshot != nil {
+		m.deferredEvents = append(m.deferredEvents, event)
 		return nil
 	}
 
 	// Apply events
-	return r.applyEvent(ctx, event, logger)
+	return m.applyEvent(ctx, event, logger)
 }
 
-func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
+func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error {
 	switch event.Event {
 	case core.EventDelete:
 		logger.Info(
 			"Deleting Target",
 			"target", event.Target.Name,
-			"targetsource", r.targetSource.Name,
+			"targetsource", m.targetSource.Name,
 		)
 	case core.EventApply:
 		logger.Info(
@@ -288,7 +288,7 @@ func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryE
 			"target", event.Target.Name,
 			"address", event.Target.Address,
 			"labels", event.Target.Labels,
-			"targetsource", r.targetSource.Name,
+			"targetsource", m.targetSource.Name,
 		)
 	}
 	return nil
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index dca0570..b1755b0 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -208,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(
 	}
 
 	// Create target reconciler instance
-	targetReconciler := discovery.NewTargetReconciler(
+	targetReconciler := discovery.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,

From c09c68f5c0c8a0f042b3192458bd4a8f1fe671cf Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 07:16:44 +0000
Subject: [PATCH 078/165] rename pipeline to runtime

---
 .../controller/targetsource_controller.go     | 43 ++++++++-----------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index b1755b0..d4442cc 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -18,7 +18,6 @@ package controller
 
 import (
 	"context"
-	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -37,9 +36,9 @@ import (
 // TargetSourceReconciler reconciles a TargetSource object
 //
 // Responsibilities:
-// - Ensure at most one pipeline per TargetSource
-// - Start pipelines on reconcile
-// - Stop pipelines on deletion or NotFound
+// - Ensure at most one runtime per TargetSource
+// - Start runtimes on reconcile
+// - Stop runtimes on deletion or NotFound
 // - Delegate runtime failure handling to the Supervisor
 type TargetSourceReconciler struct {
 	client.Client
@@ -72,11 +71,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName)
 	// If the TargetSource no longer exists, ensure runtime cleanup
 	if apierrors.IsNotFound(err) {
-		if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok {
-			pipeline.Stop()
+		if runtime, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok {
+			runtime.Stop()
 			r.DiscoveryRegistry.Unregister(req.NamespacedName)
 		}
-		logger.Info("TargetSource not found; stopped discovery pipeline")
+		logger.Info("TargetSource not found; stopped discovery runtime")
 		return ctrl.Result{}, nil
 	} else if err != nil {
 		return ctrl.Result{}, err
@@ -91,15 +90,15 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	if r.DiscoveryRegistry.Exists(req.NamespacedName) {
-		logger.Info("Discovery pipeline already running; reconciliation completed")
+		logger.Info("Discovery runtime already running; reconciliation completed")
 		return ctrl.Result{}, nil
 	}
 
-	if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil {
+	if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil {
 		return ctrl.Result{}, err
 	}
 
-	logger.Info("Started discovery pipeline")
+	logger.Info("Started discovery runtime")
 	return ctrl.Result{}, nil
 }
 
@@ -112,15 +111,15 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type
 	return &targetSource, nil
 }
 
-// reconcileDeletion stops the discovery pipeline and removes the finalizer
+// reconcileDeletion stops the discovery runtime and removes the finalizer
 func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) {
 	logger := log.FromContext(ctx).WithValues(
 		"targetsource", key.Name,
 		"namespace", key.Namespace,
 	)
-	logger.Info("TargetSource was marked for deletion; stopping discovery pipeline")
-	if pipeline, ok := r.DiscoveryRegistry.Get(key); ok {
-		pipeline.Stop()
+	logger.Info("TargetSource was marked for deletion; stopping discovery runtime")
+	if runtime, ok := r.DiscoveryRegistry.Get(key); ok {
+		runtime.Stop()
 		r.DiscoveryRegistry.Unregister(key)
 	}
 
@@ -157,6 +156,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 	return nil
 }
 
+
 // resolveRestartPolicy merges an optional spec override with the controller’s default restart policy
 func resolveRestartPolicy(
 	override *gnmicv1alpha1.RestartPolicySpec,
@@ -179,24 +179,19 @@ func resolveRestartPolicy(
 
 	return resolved
 }
-
-// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource
+// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource
 //
-// Pipeline semantics:
+// Runtime semantics:
 // 1. target reconciler is mandatory and must start first
 // 2. loader is optional and conditional on spec
-// 3. Permanent failure of required components shuts down the pipeline
+// 3. Permanent failure of required components shuts down the runtime
 // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
-func (r *TargetSourceReconciler) startDiscoveryPipeline(
+func (r *TargetSourceReconciler) startDiscoveryRuntime(
 	key types.NamespacedName,
 	targetSource *gnmicv1alpha1.TargetSource,
 	logger logr.Logger,
 ) error {
-	loaderConfigured := targetSource.Spec.Provider != nil
 	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
-	restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy)
-
-	supervisor := discovery.NewSupervisor(context.Background())
 
 	targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize)
 	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
@@ -264,7 +259,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(
 		close(targetChannel)
 		r.DiscoveryRegistry.Unregister(key)
 		logger.Info(
-			"Discovery pipeline stopped; cleaned up resources",
+			"Discovery runtime stopped; cleaned up resources",
 			"targetsource", key.Name,
 			"namespace", key.Namespace,
 		)

From e4c01bac6d1abcdec28af44c20a65b6d4af477e0 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 08:00:34 +0000
Subject: [PATCH 079/165] removed supervisor

---
 api/v1alpha1/targetsource_types.go            |  11 --
 .../discovery/core/loader_interface.go        |   4 +-
 internal/controller/discovery/core/types.go   |   5 +-
 internal/controller/discovery/defaults.go     |  12 --
 .../discovery/loaders/http/loader.go          |   2 +-
 internal/controller/discovery/supervisor.go   | 125 ---------------
 .../controller/targetsource_controller.go     | 145 +++++++-----------
 7 files changed, 59 insertions(+), 245 deletions(-)
 delete mode 100644 internal/controller/discovery/defaults.go
 delete mode 100644 internal/controller/discovery/supervisor.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 7c8f74c..ae719c1 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -29,9 +29,6 @@ type TargetSourceSpec struct {
 	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
-	// +kubebuilder:validation:Optional
-	RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"`
-
 	// +kubebuilder:validation:MinLength=1
 	TargetProfile string `json:"targetProfile"`
 }
@@ -57,14 +54,6 @@ type ConsulConfig struct {
 	URL string `json:"url,omitempty"`
 }
 
-type RestartPolicySpec struct {
-	// +kubebuilder:validation:Optional
-	MaxRestarts *int `json:"maxRestarts,omitempty"`
-
-	// +kubebuilder:validation:Optional
-	BackoffSeconds *int `json:"backoffSeconds,omitempty"`
-}
-
 // TargetSourceStatus defines the observed state of TargetSource
 type TargetSourceStatus struct {
 	Status       string      `json:"status"`
diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go
index 72f1898..bebd725 100644
--- a/internal/controller/discovery/core/loader_interface.go
+++ b/internal/controller/discovery/core/loader_interface.go
@@ -13,9 +13,9 @@ type Loader interface {
 	// Name returns the unique loader identifier e.g. "pull"
 	Name() string
 
-	// Start begins discovery and pushes target snapshots or events into the out channel
+	// Run begins discovery and pushes target snapshots or events into the out channel
 	// The loader must stop cleanly when ctx is canceled
-	Start(
+	Run(
 		ctx context.Context,
 		targetsourceName types.NamespacedName,
 		spec gnmicv1alpha1.TargetSourceSpec,
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 2f89fdf..94b4e85 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -8,10 +8,7 @@ type DiscoveryRegistryValue struct {
 	// Channel is the outbound communication channel used by discovery
 	// components (loaders, webhooks, etc.) to emit discovery messages
 	Channel chan<- []DiscoveryMessage
-	// WebhookEnabled indicates whether webhook-based discovery is enabled
-	// for this TargetSource
-	WebhookEnabled bool
-	// Stop cancels the discovery pipeline associated with this registry entry
+	// Stop cancels the discovery context associated with this registry entry
 	Stop context.CancelFunc
 }
 
diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go
deleted file mode 100644
index dc6f046..0000000
--- a/internal/controller/discovery/defaults.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package discovery
-
-import "time"
-
-// DefaultRestartPolicy defines the default restart behavior
-// for the discovery components
-func DefaultRestartPolicy() RestartPolicy {
-	return RestartPolicy{
-		MaxRestarts: 5,
-		Backoff:     3 * time.Second,
-	}
-}
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 67c61e1..383e974 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -27,7 +27,7 @@ func (l *Loader) Name() string {
 	return "http"
 }
 
-func (l *Loader) Start(
+func (l *Loader) Run(
 	ctx context.Context,
 	targetsourceNN types.NamespacedName,
 	spec gnmicv1alpha1.TargetSourceSpec,
diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go
deleted file mode 100644
index 22ec227..0000000
--- a/internal/controller/discovery/supervisor.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"sigs.k8s.io/controller-runtime/pkg/log"
-)
-
-// Supervisor coordinates the runtime lifecycle of pipeline components
-//
-// Guarantees:
-// - Each component is restarted independently
-// - Permanent failure escalates according to policy
-// - Stop() cancels all components
-// - Wait() blocks until all goroutines exit
-type Supervisor struct {
-	ctx    context.Context
-	cancel context.CancelFunc
-
-	wg sync.WaitGroup
-
-	mu      sync.Mutex
-	stopped bool
-}
-
-// RestartPolicy defines restart behavior of a component
-type RestartPolicy struct {
-	MaxRestarts int
-	Backoff     time.Duration
-}
-
-// ComponentSpec defines a supervised component
-type ComponentSpec struct {
-	Name   string
-	Run    func(ctx context.Context) error
-	Policy RestartPolicy
-	// EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline
-	EscalatesOnFailure bool
-}
-
-// NewSupervisor creates a new Supervisor with a cancellable context
-func NewSupervisor(parentCtx context.Context) *Supervisor {
-	ctx, cancel := context.WithCancel(parentCtx)
-	return &Supervisor{
-		ctx:    ctx,
-		cancel: cancel,
-	}
-}
-
-// Stop signals all supervised components to stop by canceling the context
-func (s *Supervisor) Stop() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.stopped {
-		return
-	}
-	s.stopped = true
-	s.cancel()
-}
-
-// Done returns a channel that is closed when the pipeline is stopped
-func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() }
-
-// Wait blocks until all supervised components have exited
-func (s *Supervisor) Wait() { s.wg.Wait() }
-
-// StartSupervisedComponent starts and supervises a component
-func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) {
-	s.wg.Add(1)
-
-	go func() {
-		defer s.wg.Done()
-
-		logger := log.FromContext(s.ctx).WithValues("component", component.Name)
-		failures := 0
-
-		for {
-			logger.Info(
-				"Starting supervised component",
-				"component", component.Name,
-			)
-			err := component.Run(s.ctx)
-
-			if s.ctx.Err() != nil {
-				logger.Info("component stopped due to pipeline shutdown")
-				return
-			}
-
-			failures++
-			logger.Error(
-				err,
-				"Supervised component failed",
-				"component", component.Name,
-				"attempt", failures,
-				"maxRestarts", component.Policy.MaxRestarts,
-			)
-
-			if failures >= component.Policy.MaxRestarts {
-				if component.EscalatesOnFailure {
-					logger.Error(
-						err,
-						"Supervised component permanently failed; stopped discovery pipeline",
-						"component", component.Name,
-					)
-					s.Stop()
-				} else {
-					logger.Info(
-						"Optional component permanently failed; continuing without it",
-						"component", component.Name,
-					)
-				}
-				return
-			}
-
-			select {
-			case <-time.After(component.Policy.Backoff):
-			case <-s.ctx.Done():
-				return
-			}
-		}
-	}()
-}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index d4442cc..afc088b 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -36,10 +36,10 @@ import (
 // TargetSourceReconciler reconciles a TargetSource object
 //
 // Responsibilities:
-// - Ensure at most one runtime per TargetSource
-// - Start runtimes on reconcile
-// - Stop runtimes on deletion or NotFound
-// - Delegate runtime failure handling to the Supervisor
+// - Ensure at most one discovery runtime per TargetSource
+// - Start runtime on reconcile if not already running
+// - Restart runtime on reconcile if spec changed
+// - Stop runtime on deletion or NotFound
 type TargetSourceReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
@@ -94,7 +94,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, nil
 	}
 
-	if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil {
+	if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -156,113 +156,78 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 	return nil
 }
 
-
-// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy
-func resolveRestartPolicy(
-	override *gnmicv1alpha1.RestartPolicySpec,
-) discovery.RestartPolicy {
-	defaults := discovery.DefaultRestartPolicy()
-
-	if override == nil {
-		return defaults
-	}
-
-	resolved := defaults
-
-	if override.MaxRestarts != nil {
-		resolved.MaxRestarts = *override.MaxRestarts
-	}
-
-	if override.BackoffSeconds != nil {
-		resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second
-	}
-
-	return resolved
-}
-// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource
+// startDiscovery creates and starts a discovery runtime for a TargetSource
 //
-// Runtime semantics:
-// 1. target reconciler is mandatory and must start first
-// 2. loader is optional and conditional on spec
-// 3. Permanent failure of required components shuts down the runtime
-// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister
-func (r *TargetSourceReconciler) startDiscoveryRuntime(
+// Invariant:
+// - MessageProcessor and Loader must run for the lifetime of the TargetSource
+// - Any unexpected exit is treated as a bug and triggers full shutdown
+func (r *TargetSourceReconciler) startDiscovery(
 	key types.NamespacedName,
 	targetSource *gnmicv1alpha1.TargetSource,
 	logger logr.Logger,
 ) error {
-	webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
-
 	targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Register discovery runtime of targetsource
 	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
-		Channel:        targetChannel,
-		WebhookEnabled: webhookActivated,
-		Stop:           supervisor.Stop,
+		Channel: targetChannel,
+		Stop:    cancel,
 	}); err != nil {
 		return err
 	}
 
-	// Create target reconciler instance
-	targetReconciler := discovery.NewMessageProcessor(
+	// Cleanup function to cleanup discovery runtime of targetsource
+	cleanup := func() {
+		cancel()
+		r.DiscoveryRegistry.Unregister(key)
+		close(targetChannel)
+	}
+
+	// Start message processor
+	messageProcessor := discovery.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,
 		targetChannel,
 	)
-	// Start target reconciler
-	targetReconcilerReady := make(chan struct{})
-	supervisor.StartSupervisedComponent(discovery.ComponentSpec{
-		Name:               "target-reconciler",
-		Policy:             restartPolicy,
-		EscalatesOnFailure: true,
-		Run: func(ctx context.Context) error {
-			close(targetReconcilerReady) // Signals that reconciler started successfully
-			return targetReconciler.Run(ctx)
-		},
-	})
-	// Wait for reconciler to be ready before starting loader
-	select {
-	case <-targetReconcilerReady:
-		logger.Info("Target reconciler started")
-	case <-supervisor.Done():
-		logger.Info("Supervisor stopped before target reconciler became ready")
-		return nil
-	}
+	go func() {
+		logger.Info("Message processor started")
 
-	// Create loader instance
-	if loaderConfigured {
-		loader, err := discovery.NewLoader(
-			key,
-			targetSource.Spec,
-			discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize},
-		)
-		if err != nil {
-			supervisor.Stop()
-			return err
+		if err := messageProcessor.Run(ctx); err != nil {
+			logger.Error(err, "Message processor exited unecpectedly")
+		} else {
+			logger.Error(nil, "Message processor exited unexpectedly without error")
 		}
 
-		supervisor.StartSupervisedComponent(discovery.ComponentSpec{
-			Name:               "loader",
-			Policy:             restartPolicy,
-			EscalatesOnFailure: !webhookActivated,
-			Run: func(ctx context.Context) error {
-				return loader.Start(ctx, key, targetSource.Spec, targetChannel)
-			},
-		})
-	}
+		// Any exit is considered a bug that should stop the discovery runtime
+		cleanup()
+	}()
 
-	// Monitor supervisor in a separate goroutine to handle shutdown and cleanup
+	// Start target loader
+	// webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
+	loaderConfig := discoveryTypes.LoaderConfig{
+		ChunkSize: r.ChunkSize,
+	}
+	loader, err := discovery.NewLoader(
+		key,
+		targetSource.Spec,
+		loaderConfig,
+	)
+	if err != nil {
+		logger.Error(err, "Target loader could not be created")
+		cleanup()
+		return err
+	}
 	go func() {
-		<-supervisor.Done()
-		supervisor.Wait() // Wait for components to exit
+		if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil {
+			logger.Error(err, "Target loader exited unexpectedly")
+		} else {
+			logger.Error(nil, "Target loader exited unexpectedly without error")
+		}
 
-		close(targetChannel)
-		r.DiscoveryRegistry.Unregister(key)
-		logger.Info(
-			"Discovery runtime stopped; cleaned up resources",
-			"targetsource", key.Name,
-			"namespace", key.Namespace,
-		)
+		// Any exit is considered a bug that should stop the discovery runtime
+		cleanup()
 	}()
 
 	return nil

From 77dbd7e12dd19d33a38113d280b522a7fdea1d99 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 08:13:35 +0000
Subject: [PATCH 080/165] tidy loader configuration abstraction

---
 .../discovery/core/loader_interface.go        | 10 +--------
 internal/controller/discovery/core/types.go   | 11 ++++++++--
 internal/controller/discovery/loaders.go      | 12 +++++------
 .../discovery/loaders/http/loader.go          | 21 +++++++------------
 .../controller/targetsource_controller.go     | 12 +++++------
 5 files changed, 27 insertions(+), 39 deletions(-)

diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go
index bebd725..895258a 100644
--- a/internal/controller/discovery/core/loader_interface.go
+++ b/internal/controller/discovery/core/loader_interface.go
@@ -2,9 +2,6 @@ package core
 
 import (
 	"context"
-
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-	"k8s.io/apimachinery/pkg/types"
 )
 
 // Loader defines a pluggable TargetSource loader interface
@@ -15,10 +12,5 @@ type Loader interface {
 
 	// Run begins discovery and pushes target snapshots or events into the out channel
 	// The loader must stop cleanly when ctx is canceled
-	Run(
-		ctx context.Context,
-		targetsourceName types.NamespacedName,
-		spec gnmicv1alpha1.TargetSourceSpec,
-		out chan<- []DiscoveryMessage,
-	) error
+	Run(ctx context.Context, out chan<- []DiscoveryMessage) error
 }
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 94b4e85..5028972 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -1,6 +1,11 @@
 package core
 
-import "context"
+import (
+	"context"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/types"
+)
 
 // DiscoveryRegistryValue represents the controller-owned runtime state
 // of a discovery pipeline for a single TargetSource
@@ -13,7 +18,9 @@ type DiscoveryRegistryValue struct {
 }
 
 type LoaderConfig struct {
-	ChunkSize int
+	TargetsourceNN types.NamespacedName
+	Spec           *gnmicv1alpha1.TargetSourceSpec
+	ChunkSize      int
 }
 
 // EventAction represents the type of a discovery event
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 0d8ddd3..6c3e133 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -3,22 +3,20 @@ package discovery
 import (
 	"fmt"
 
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	http "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
-	"k8s.io/apimachinery/pkg/types"
 )
 
 // NewLoader creates a loader by name
-func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) {
+func NewLoader(cfg core.LoaderConfig) (core.Loader, error) {
 
 	switch {
-	case spec.Provider.HTTP != nil:
+	case cfg.Spec.Provider.HTTP != nil:
 		return http.New(cfg), nil
-	case spec.Provider.Consul != nil:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name)
+	case cfg.Spec.Provider.Consul != nil:
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name)
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 
 }
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 383e974..17812aa 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -5,10 +5,8 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils"
 	"github.com/google/uuid"
@@ -27,22 +25,17 @@ func (l *Loader) Name() string {
 	return "http"
 }
 
-func (l *Loader) Run(
-	ctx context.Context,
-	targetsourceNN types.NamespacedName,
-	spec gnmicv1alpha1.TargetSourceSpec,
-	out chan<- []core.DiscoveryMessage,
-) error {
+func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error {
 	logger := log.FromContext(ctx).WithValues(
 		"component", "loader",
 		"name", l.Name(),
-		"targetsource", targetsourceNN,
+		"targetsource", l.cfg.TargetsourceNN,
 	)
 
 	logger.Info(
 		"HTTP loader started",
-		"targetsource", targetsourceNN.Name,
-		"namespace", targetsourceNN.Namespace,
+		"targetsource", l.cfg.TargetsourceNN.Name,
+		"namespace", l.cfg.TargetsourceNN.Namespace,
 	)
 
 	// Only for debugging: emit a static snapshot every 30 seconds
@@ -57,17 +50,17 @@ func (l *Loader) Run(
 
 		case <-ticker.C:
 			// Example snapshot (placeholder)
-			snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString())
+			snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString())
 			targets := []core.DiscoveredTarget{
 				{
 					Name:    "ceos1",
 					Address: "clab-3-nodes-ceos1:6030",
-					Labels:  map[string]string{"TargetSource": targetsourceNN.String()},
+					Labels:  map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()},
 				},
 				{
 					Name:    "leaf1",
 					Address: "clab-3-nodes-leaf1:57400",
-					Labels:  map[string]string{"TargetSource": targetsourceNN.String()},
+					Labels:  map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()},
 				},
 			}
 
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index afc088b..0064570 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -207,20 +207,18 @@ func (r *TargetSourceReconciler) startDiscovery(
 	// Start target loader
 	// webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
 	loaderConfig := discoveryTypes.LoaderConfig{
-		ChunkSize: r.ChunkSize,
+		TargetsourceNN: key,
+		Spec:           &targetSource.Spec,
+		ChunkSize:      r.ChunkSize,
 	}
-	loader, err := discovery.NewLoader(
-		key,
-		targetSource.Spec,
-		loaderConfig,
-	)
+	loader, err := discovery.NewLoader(loaderConfig)
 	if err != nil {
 		logger.Error(err, "Target loader could not be created")
 		cleanup()
 		return err
 	}
 	go func() {
-		if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil {
+		if err := loader.Run(ctx, targetChannel); err != nil {
 			logger.Error(err, "Target loader exited unexpectedly")
 		} else {
 			logger.Error(nil, "Target loader exited unexpectedly without error")

From fe900e38774051956054d70af6ee24da88beb71f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 08:14:40 +0000
Subject: [PATCH 081/165] regenearte manifests without restartPolicy

---
 api/v1alpha1/zz_generated.deepcopy.go         | 30 -------------------
 .../operator.gnmic.dev_targetsources.yaml     |  7 -----
 2 files changed, 37 deletions(-)

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index df08573..608d47e 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -843,31 +843,6 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) {
-	*out = *in
-	if in.MaxRestarts != nil {
-		in, out := &in.MaxRestarts, &out.MaxRestarts
-		*out = new(int)
-		**out = **in
-	}
-	if in.BackoffSeconds != nil {
-		in, out := &in.BackoffSeconds, &out.BackoffSeconds
-		*out = new(int)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec.
-func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec {
-	if in == nil {
-		return nil
-	}
-	out := new(RestartPolicySpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) {
 	*out = *in
@@ -1325,11 +1300,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) {
 			(*out)[key] = val
 		}
 	}
-	if in.RestartPolicy != nil {
-		in, out := &in.RestartPolicy, &out.RestartPolicy
-		*out = new(RestartPolicySpec)
-		(*in).DeepCopyInto(*out)
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec.
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 6464ea2..b385d8e 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -60,13 +60,6 @@ spec:
                 - message: exactly one of the fields in [http consul] must be set
                   rule: '[has(self.http),has(self.consul)].filter(x,x==true).size()
                     == 1'
-              restartPolicy:
-                properties:
-                  backoffSeconds:
-                    type: integer
-                  maxRestarts:
-                    type: integer
-                type: object
               targetLabels:
                 additionalProperties:
                   type: string

From c1d7a91bc7e79f87736454dc31e530d3cf89b7fd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 08:29:54 +0000
Subject: [PATCH 082/165] tidy up comments

---
 internal/controller/discovery/discovery.go     | 4 +---
 internal/controller/discovery/loaders.go       | 1 +
 internal/controller/targetsource_controller.go | 1 -
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go
index 3dc51bd..491cdfb 100644
--- a/internal/controller/discovery/discovery.go
+++ b/internal/controller/discovery/discovery.go
@@ -4,13 +4,11 @@ package discovery
 //
 // The discovery subsystem is responsible for:
 // - Receiving discovery data from external providers (loaders, webhooks).
-// - Supervising discovery pipelines and restart semantics.
 // - Applying discovered state to Kubernetes Targets.
 //
 // The package is structured into the following subpackages:
 // - core: message contracts, snapshot/event types, and transport helpers.
-// - pipeline: supervision, restart policies, and lifecycle control.
-// - reconciler: snapshot + event target state application logic.
+// - message processor: snapshot + event target state application logic.
 // - loaders: target discovery providers (HTTP, webhook, etc.).
 // - registry: key -> channel registry.
 //
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 6c3e133..9704b16 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -12,6 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) {
 
 	switch {
 	case cfg.Spec.Provider.HTTP != nil:
+		// webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
 		return http.New(cfg), nil
 	case cfg.Spec.Provider.Consul != nil:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 0064570..2ba18a2 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -205,7 +205,6 @@ func (r *TargetSourceReconciler) startDiscovery(
 	}()
 
 	// Start target loader
-	// webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
 	loaderConfig := discoveryTypes.LoaderConfig{
 		TargetsourceNN: key,
 		Spec:           &targetSource.Spec,

From 05c7538ce47a4b81fd245b11435cb13481c4c671 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 13:23:34 +0000
Subject: [PATCH 083/165] move webhook spec into provider and rename it to
 acceptPush

---
 api/v1alpha1/targetsource_types.go          | 10 +++-------
 internal/controller/discovery/core/types.go |  5 ++++-
 internal/controller/discovery/loaders.go    |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index ae719c1..3d69743 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -24,8 +24,7 @@ import (
 // +kubebuilder:validation:Required
 type TargetSourceSpec struct {
 	Provider *ProviderSpec `json:"provider"`
-	// +kubebuilder:validation:Optional
-	Webhook WebhookSpec `json:"webhook,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
@@ -39,14 +38,11 @@ type ProviderSpec struct {
 	Consul *ConsulConfig `json:"consul,omitempty"`
 }
 
-type WebhookSpec struct {
-	// +kubebuilder:validation:Optional
-	Enabled *bool `json:"enabled,omitempty"`
-}
-
 type HTTPConfig struct {
 	// +kubebuilder:validation:MinLength=1
 	URL string `json:"url"`
+	// +kubebuilder:validation:Optional
+	AcceptPush bool `json:"acceptPush,omitempty"`
 }
 
 type ConsulConfig struct {
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 5028972..1dfcc9f 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -8,19 +8,22 @@ import (
 )
 
 // DiscoveryRegistryValue represents the controller-owned runtime state
-// of a discovery pipeline for a single TargetSource
+// with its configuration for a single TargetSource
 type DiscoveryRegistryValue struct {
 	// Channel is the outbound communication channel used by discovery
 	// components (loaders, webhooks, etc.) to emit discovery messages
 	Channel chan<- []DiscoveryMessage
 	// Stop cancels the discovery context associated with this registry entry
 	Stop context.CancelFunc
+
+	LoaderConfig *LoaderConfig
 }
 
 type LoaderConfig struct {
 	TargetsourceNN types.NamespacedName
 	Spec           *gnmicv1alpha1.TargetSourceSpec
 	ChunkSize      int
+	AcceptPush     bool
 }
 
 // EventAction represents the type of a discovery event
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 9704b16..487c76b 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -12,7 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) {
 
 	switch {
 	case cfg.Spec.Provider.HTTP != nil:
-		// webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled
+		cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush
 		return http.New(cfg), nil
 	case cfg.Spec.Provider.Consul != nil:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)

From 061d4b83daac3a5c26fbe12151aa353a88d1a213 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 13:24:00 +0000
Subject: [PATCH 084/165] regenerate manifests

---
 api/v1alpha1/zz_generated.deepcopy.go         | 21 -------------------
 .../operator.gnmic.dev_targetsources.yaml     |  7 ++-----
 2 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 608d47e..61e81fd 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -1292,7 +1292,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) {
 		*out = new(ProviderSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	in.Webhook.DeepCopyInto(&out.Webhook)
 	if in.TargetLabels != nil {
 		in, out := &in.TargetLabels, &out.TargetLabels
 		*out = make(map[string]string, len(*in))
@@ -1478,23 +1477,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
-	*out = *in
-	if in.Enabled != nil {
-		in, out := &in.Enabled, &out.Enabled
-		*out = new(bool)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
-func (in *WebhookSpec) DeepCopy() *WebhookSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookSpec)
-	in.DeepCopyInto(out)
-	return out
-}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index b385d8e..37d6919 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -49,6 +49,8 @@ spec:
                     type: object
                   http:
                     properties:
+                      acceptPush:
+                        type: boolean
                       url:
                         minLength: 1
                         type: string
@@ -67,11 +69,6 @@ spec:
               targetProfile:
                 minLength: 1
                 type: string
-              webhook:
-                properties:
-                  enabled:
-                    type: boolean
-                type: object
             required:
             - provider
             - targetProfile

From 41655a0d4e835bc2ff0b8a5a1cdaf55aa4bdfd7a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 14:12:05 +0000
Subject: [PATCH 085/165] remove spec from laoder config

---
 internal/controller/discovery/core/types.go    |  2 --
 internal/controller/discovery/loaders.go       |  9 +++++----
 internal/controller/targetsource_controller.go | 16 ++++++++--------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 1dfcc9f..993c84e 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -3,7 +3,6 @@ package core
 import (
 	"context"
 
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -21,7 +20,6 @@ type DiscoveryRegistryValue struct {
 
 type LoaderConfig struct {
 	TargetsourceNN types.NamespacedName
-	Spec           *gnmicv1alpha1.TargetSourceSpec
 	ChunkSize      int
 	AcceptPush     bool
 }
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 487c76b..d179d3e 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -3,18 +3,19 @@ package discovery
 import (
 	"fmt"
 
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	http "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
 
 // NewLoader creates a loader by name
-func NewLoader(cfg core.LoaderConfig) (core.Loader, error) {
+func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
 
 	switch {
-	case cfg.Spec.Provider.HTTP != nil:
-		cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush
+	case spec.Provider.HTTP != nil:
+		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
 		return http.New(cfg), nil
-	case cfg.Spec.Provider.Consul != nil:
+	case spec.Provider.Consul != nil:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
 		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 2ba18a2..1cf962d 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -168,11 +168,16 @@ func (r *TargetSourceReconciler) startDiscovery(
 ) error {
 	targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize)
 	ctx, cancel := context.WithCancel(context.Background())
+	loaderConfig := discoveryTypes.LoaderConfig{
+		TargetsourceNN: key,
+		ChunkSize:      r.ChunkSize,
+	}
 
 	// Register discovery runtime of targetsource
 	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
-		Channel: targetChannel,
-		Stop:    cancel,
+		Channel:      targetChannel,
+		Stop:         cancel,
+		LoaderConfig: &loaderConfig,
 	}); err != nil {
 		return err
 	}
@@ -205,12 +210,7 @@ func (r *TargetSourceReconciler) startDiscovery(
 	}()
 
 	// Start target loader
-	loaderConfig := discoveryTypes.LoaderConfig{
-		TargetsourceNN: key,
-		Spec:           &targetSource.Spec,
-		ChunkSize:      r.ChunkSize,
-	}
-	loader, err := discovery.NewLoader(loaderConfig)
+	loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec)
 	if err != nil {
 		logger.Error(err, "Target loader could not be created")
 		cleanup()

From 97849ae9d9a7afdc13aab966882433f0a59f0f7c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 1 May 2026 15:08:55 +0000
Subject: [PATCH 086/165] update LoaderConfig in registry

---
 internal/apiserver/apiserver.go               |  2 +-
 internal/controller/discovery/core/types.go   |  4 +-
 internal/controller/discovery/loaders.go      |  8 ++--
 .../discovery/loaders/http/loader.go          | 20 +++++-----
 .../controller/targetsource_controller.go     | 40 +++++++++++--------
 5 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go
index 705b277..5eb88b8 100644
--- a/internal/apiserver/apiserver.go
+++ b/internal/apiserver/apiserver.go
@@ -5,8 +5,8 @@ import (
 	"net/http"
 
 	"github.com/gnmic/operator/internal/controller"
-	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/gnmic/operator/internal/controller/discovery"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"k8s.io/apimachinery/pkg/types"
 )
 
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 993c84e..99605b9 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -15,10 +15,10 @@ type DiscoveryRegistryValue struct {
 	// Stop cancels the discovery context associated with this registry entry
 	Stop context.CancelFunc
 
-	LoaderConfig *LoaderConfig
+	CommonLoaderConfig *CommonLoaderConfig
 }
 
-type LoaderConfig struct {
+type CommonLoaderConfig struct {
 	TargetsourceNN types.NamespacedName
 	ChunkSize      int
 	AcceptPush     bool
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index d179d3e..7f2c656 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -9,16 +9,16 @@ import (
 )
 
 // NewLoader creates a loader by name
-func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
+func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, core.CommonLoaderConfig, error) {
 
 	switch {
 	case spec.Provider.HTTP != nil:
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
-		return http.New(cfg), nil
+		return http.New(cfg), cfg, nil
 	case spec.Provider.Consul != nil:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 
 }
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 17812aa..3325adb 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -13,12 +13,12 @@ import (
 )
 
 type Loader struct {
-	cfg core.LoaderConfig
+	commonCfg core.CommonLoaderConfig
 }
 
 // New instantiates the http loader with the provided config
-func New(cfg core.LoaderConfig) core.Loader {
-	return &Loader{cfg: cfg}
+func New(cfg core.CommonLoaderConfig) core.Loader {
+	return &Loader{commonCfg: cfg}
 }
 
 func (l *Loader) Name() string {
@@ -29,13 +29,13 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	logger := log.FromContext(ctx).WithValues(
 		"component", "loader",
 		"name", l.Name(),
-		"targetsource", l.cfg.TargetsourceNN,
+		"targetsource", l.commonCfg.TargetsourceNN,
 	)
 
 	logger.Info(
 		"HTTP loader started",
-		"targetsource", l.cfg.TargetsourceNN.Name,
-		"namespace", l.cfg.TargetsourceNN.Namespace,
+		"targetsource", l.commonCfg.TargetsourceNN.Name,
+		"namespace", l.commonCfg.TargetsourceNN.Namespace,
 	)
 
 	// Only for debugging: emit a static snapshot every 30 seconds
@@ -50,21 +50,21 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 
 		case <-ticker.C:
 			// Example snapshot (placeholder)
-			snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString())
+			snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString())
 			targets := []core.DiscoveredTarget{
 				{
 					Name:    "ceos1",
 					Address: "clab-3-nodes-ceos1:6030",
-					Labels:  map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()},
+					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 				{
 					Name:    "leaf1",
 					Address: "clab-3-nodes-leaf1:57400",
-					Labels:  map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()},
+					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 			}
 
-			if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil {
+			if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil {
 				return err
 			}
 		}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 1cf962d..8207103 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -168,20 +168,11 @@ func (r *TargetSourceReconciler) startDiscovery(
 ) error {
 	targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize)
 	ctx, cancel := context.WithCancel(context.Background())
-	loaderConfig := discoveryTypes.LoaderConfig{
+	loaderConfig := discoveryTypes.CommonLoaderConfig{
 		TargetsourceNN: key,
 		ChunkSize:      r.ChunkSize,
 	}
 
-	// Register discovery runtime of targetsource
-	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
-		Channel:      targetChannel,
-		Stop:         cancel,
-		LoaderConfig: &loaderConfig,
-	}); err != nil {
-		return err
-	}
-
 	// Cleanup function to cleanup discovery runtime of targetsource
 	cleanup := func() {
 		cancel()
@@ -189,13 +180,34 @@ func (r *TargetSourceReconciler) startDiscovery(
 		close(targetChannel)
 	}
 
-	// Start message processor
 	messageProcessor := discovery.NewMessageProcessor(
 		r.Client,
 		r.Scheme,
 		targetSource,
 		targetChannel,
 	)
+	loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{
+		TargetsourceNN: key,
+		ChunkSize:      r.ChunkSize,
+	},
+		&targetSource.Spec,
+	)
+	if err != nil {
+		logger.Error(err, "Target loader could not be created")
+		cleanup()
+		return err
+	}
+
+	// Register discovery runtime of targetsource
+	if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{
+		Channel:            targetChannel,
+		Stop:               cancel,
+		CommonLoaderConfig: &loaderConfig,
+	}); err != nil {
+		return err
+	}
+
+	// Start message processor
 	go func() {
 		logger.Info("Message processor started")
 
@@ -210,12 +222,6 @@ func (r *TargetSourceReconciler) startDiscovery(
 	}()
 
 	// Start target loader
-	loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec)
-	if err != nil {
-		logger.Error(err, "Target loader could not be created")
-		cleanup()
-		return err
-	}
 	go func() {
 		if err := loader.Run(ctx, targetChannel); err != nil {
 			logger.Error(err, "Target loader exited unexpectedly")

From 426e27ae1e39a33a963d6e24ea25362b56683f6f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 4 May 2026 18:15:49 +0000
Subject: [PATCH 087/165] fix: use defined variable

---
 internal/controller/targetsource_controller.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 8207103..c65e254 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -186,10 +186,7 @@ func (r *TargetSourceReconciler) startDiscovery(
 		targetSource,
 		targetChannel,
 	)
-	loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{
-		TargetsourceNN: key,
-		ChunkSize:      r.ChunkSize,
-	},
+	loader, loaderConfig, err := discovery.NewLoader(loaderConfig,
 		&targetSource.Spec,
 	)
 	if err != nil {

From a6e449d04266d5c71f0503ed5753b51835f3b58b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 6 May 2026 07:57:31 +0000
Subject: [PATCH 088/165] load spec into loader

---
 internal/controller/discovery/loaders.go             |  2 +-
 internal/controller/discovery/loaders/http/loader.go | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 7f2c656..c75c5fa 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -14,7 +14,7 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec
 	switch {
 	case spec.Provider.HTTP != nil:
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
-		return http.New(cfg), cfg, nil
+		return http.New(cfg, *spec.Provider.HTTP), cfg, nil
 	case spec.Provider.Consul != nil:
 		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index cf081b6..b6941e9 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -8,11 +8,14 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/google/uuid"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils"
-	"github.com/google/uuid"
 )
 
 const (
@@ -23,11 +26,12 @@ const (
 // Loader implements the HTTP pull discovery mechanism
 type Loader struct {
 	commonCfg core.CommonLoaderConfig
+	spec      *gnmicv1alpha1.HTTPConfig
 }
 
 // New instantiates the http loader with the provided config
-func New(cfg core.CommonLoaderConfig) core.Loader {
-	return &Loader{commonCfg: cfg}
+func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader {
+	return &Loader{commonCfg: cfg, spec: &httpConfig}
 }
 
 func (l *Loader) Name() string {
@@ -50,7 +54,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	logger.Info("HTTP loader started")
 
 	// Input Validation of spec
-	if spec.Provider == nil || spec.Provider.HTTP == nil {
+	if spec. == nil {
 		return errors.New("HTTP loader requires spec.provider.http to be set")
 	}
 

From e908953342531ffb2b0053e288d03095cca0a8dd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 6 May 2026 08:13:59 +0000
Subject: [PATCH 089/165] update httpconfig

---
 api/v1alpha1/targetsource_types.go            |  87 +++++-
 api/v1alpha1/zz_generated.deepcopy.go         | 261 +++++++++++++++--
 .../operator.gnmic.dev_targetsources.yaml     | 275 +++++++++++++++++-
 3 files changed, 588 insertions(+), 35 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 3d69743..ab48614 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -32,22 +33,98 @@ type TargetSourceSpec struct {
 	TargetProfile string `json:"targetProfile"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=http;consul
+// +kubebuilder:validation:ExactlyOneOf=http
 type ProviderSpec struct {
-	HTTP   *HTTPConfig   `json:"http,omitempty"`
-	Consul *ConsulConfig `json:"consul,omitempty"`
+	HTTP *HTTPConfig `json:"http,omitempty"`
+}
+
+type WebhookSpec struct {
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 type HTTPConfig struct {
 	// +kubebuilder:validation:MinLength=1
 	URL string `json:"url"`
 	// +kubebuilder:validation:Optional
+	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
+	// +kubebuilder:validation:Optional
+	PollInterval *metav1.Duration `json:"interval,omitempty"`
+	// +kubebuilder:validation:Optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
+	// +kubebuilder:validation:Optional
+	TLS *ClientTLSConfig `json:"tls,omitempty"`
+	// +kubebuilder:validation:Optional
+	Pagination *PaginationSpec `json:"pagination,omitempty"`
+	// +kubebuilder:validation:Optional
+	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
+	// +kubebuilder:validation:Optional
 	AcceptPush bool `json:"acceptPush,omitempty"`
 }
 
-type ConsulConfig struct {
+type ClientTLSConfig struct {
+	InsecureSkipVerify bool                      `json:"insecureSkipVerify,omitempty"`
+	CASecretRef        *corev1.SecretKeySelector `json:"caSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:ExactlyOneOf=basic;bearer;jwt;token
+type AuthorizationSpec struct {
+	Basic  *BasicAuthSpec  `json:"basic,omitempty"`
+	Bearer *BearerAuthSpec `json:"bearer,omitempty"`
+	Token  *TokenAuthSpec  `json:"token,omitempty"`
+	JWT    *JWTAuthSpec    `json:"jwt,omitempty"`
+}
+
+// Enforce EITHER inline creds OR secret ref
+// +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
+type BasicAuthSpec struct {
+	Username             string                    `json:"username,omitempty"`
+	Password             string                    `json:"password,omitempty"`
+	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:ExactlyOneOf=token;tokenSecretRef
+type BearerAuthSpec struct {
+	Token          string                    `json:"token,omitempty"`
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
+type TokenAuthSpec struct {
 	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url,omitempty"`
+	Scheme         string                    `json:"scheme"`
+	Token          string                    `json:"token,omitempty"`
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder:validation:XValidation:rule="!( (has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)) )",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != ”",message="algorithm must be specified when generating a JWT"
+type JWTAuthSpec struct {
+	// Static pre-generated JWT
+	Token          string                    `json:"token,omitempty"`
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+	// Optional: generate JWT dynamically
+	Claims              map[string]string         `json:"claims,omitempty"`
+	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
+	// HS256, RS256, ES256, etc.
+	Algorithm string           `json:"algorithm,omitempty"`
+	TTL       *metav1.Duration `json:"ttl,omitempty"`
+}
+
+type PaginationSpec struct {
+	Enabled bool `json:"enabled"`
+	// Example: "results"
+	ItemsField string `json:"itemsField,omitempty"`
+	// Example: "next"
+	NextField string `json:"nextField,omitempty"`
+}
+
+// JSONPath-style expressions
+type ResponseMappingSpec struct {
+	Name    string            `json:"name"`
+	Address string            `json:"address"`
+	Port    string            `json:"port,omitempty"`
+	Labels  map[string]string `json:"labels,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 61e81fd..602ede5 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -46,6 +46,101 @@ func (in *APIConfig) DeepCopy() *APIConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) {
+	*out = *in
+	if in.Basic != nil {
+		in, out := &in.Basic, &out.Basic
+		*out = new(BasicAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Bearer != nil {
+		in, out := &in.Bearer, &out.Bearer
+		*out = new(BearerAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Token != nil {
+		in, out := &in.Token, &out.Token
+		*out = new(TokenAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.JWT != nil {
+		in, out := &in.JWT, &out.JWT
+		*out = new(JWTAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationSpec.
+func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizationSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) {
+	*out = *in
+	if in.CredentialsSecretRef != nil {
+		in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuthSpec.
+func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(BasicAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BearerAuthSpec) DeepCopyInto(out *BearerAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BearerAuthSpec.
+func (in *BearerAuthSpec) DeepCopy() *BearerAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(BearerAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
+	*out = *in
+	if in.CASecretRef != nil {
+		in, out := &in.CASecretRef, &out.CASecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientTLSConfig.
+func (in *ClientTLSConfig) DeepCopy() *ClientTLSConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(ClientTLSConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Cluster) DeepCopyInto(out *Cluster) {
 	*out = *in
@@ -213,21 +308,6 @@ func (in *ClusterTargetState) DeepCopy() *ClusterTargetState {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ConsulConfig) DeepCopyInto(out *ConsulConfig) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsulConfig.
-func (in *ConsulConfig) DeepCopy() *ConsulConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(ConsulConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GRPCKeepAliveConfig) DeepCopyInto(out *GRPCKeepAliveConfig) {
 	*out = *in
@@ -273,6 +353,36 @@ func (in *GRPCTunnelConfig) DeepCopy() *GRPCTunnelConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	*out = *in
+	if in.Authorization != nil {
+		in, out := &in.Authorization, &out.Authorization
+		*out = new(AuthorizationSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PollInterval != nil {
+		in, out := &in.PollInterval, &out.PollInterval
+		*out = new(metav1.Duration)
+		**out = **in
+	}
+	if in.Timeout != nil {
+		in, out := &in.Timeout, &out.Timeout
+		*out = new(metav1.Duration)
+		**out = **in
+	}
+	if in.TLS != nil {
+		in, out := &in.TLS, &out.TLS
+		*out = new(ClientTLSConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Pagination != nil {
+		in, out := &in.Pagination, &out.Pagination
+		*out = new(PaginationSpec)
+		**out = **in
+	}
+	if in.ResponseMapping != nil {
+		in, out := &in.ResponseMapping, &out.ResponseMapping
+		*out = new(ResponseMappingSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig.
@@ -414,6 +524,43 @@ func (in *InputStatus) DeepCopy() *InputStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Claims != nil {
+		in, out := &in.Claims, &out.Claims
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.SigningKeySecretRef != nil {
+		in, out := &in.SigningKeySecretRef, &out.SigningKeySecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.TTL != nil {
+		in, out := &in.TTL, &out.TTL
+		*out = new(metav1.Duration)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthSpec.
+func (in *JWTAuthSpec) DeepCopy() *JWTAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(JWTAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Output) DeepCopyInto(out *Output) {
 	*out = *in
@@ -587,6 +734,21 @@ func (in *OutputStatus) DeepCopy() *OutputStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec.
+func (in *PaginationSpec) DeepCopy() *PaginationSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PaginationSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pipeline) DeepCopyInto(out *Pipeline) {
 	*out = *in
@@ -824,12 +986,7 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) {
 	if in.HTTP != nil {
 		in, out := &in.HTTP, &out.HTTP
 		*out = new(HTTPConfig)
-		**out = **in
-	}
-	if in.Consul != nil {
-		in, out := &in.Consul, &out.Consul
-		*out = new(ConsulConfig)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -843,6 +1000,28 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResponseMappingSpec.
+func (in *ResponseMappingSpec) DeepCopy() *ResponseMappingSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ResponseMappingSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) {
 	*out = *in
@@ -1384,6 +1563,26 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenAuthSpec.
+func (in *TokenAuthSpec) DeepCopy() *TokenAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(TokenAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TunnelTargetPolicy) DeepCopyInto(out *TunnelTargetPolicy) {
 	*out = *in
@@ -1477,3 +1676,23 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
+	*out = *in
+	if in.Enabled != nil {
+		in, out := &in.Enabled, &out.Enabled
+		*out = new(bool)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
+func (in *WebhookSpec) DeepCopy() *WebhookSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookSpec)
+	in.DeepCopyInto(out)
+	return out
+}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 37d6919..23360c5 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -41,16 +41,274 @@ spec:
             properties:
               provider:
                 properties:
-                  consul:
-                    properties:
-                      url:
-                        minLength: 1
-                        type: string
-                    type: object
                   http:
                     properties:
                       acceptPush:
                         type: boolean
+                      authorization:
+                        properties:
+                          basic:
+                            description: Enforce EITHER inline creds OR secret ref
+                            properties:
+                              credentialsSecretRef:
+                                description: SecretKeySelector selects a key of a
+                                  Secret.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                              password:
+                                type: string
+                              username:
+                                type: string
+                            type: object
+                            x-kubernetes-validations:
+                            - message: either credentialsSecretRef OR both username
+                                and password must be set, but not a mix
+                              rule: (has(self.credentialsSecretRef) && !has(self.username)
+                                && !has(self.password)) || (!has(self.credentialsSecretRef)
+                                && has(self.username) && has(self.password))
+                          bearer:
+                            properties:
+                              token:
+                                type: string
+                              tokenSecretRef:
+                                description: SecretKeySelector selects a key of a
+                                  Secret.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                            type: object
+                            x-kubernetes-validations:
+                            - message: exactly one of the fields in [token tokenSecretRef]
+                                must be set
+                              rule: '[has(self.token),has(self.tokenSecretRef)].filter(x,x==true).size()
+                                == 1'
+                          jwt:
+                            properties:
+                              algorithm:
+                                description: HS256, RS256, ES256, etc.
+                                type: string
+                              claims:
+                                additionalProperties:
+                                  type: string
+                                description: 'Optional: generate JWT dynamically'
+                                type: object
+                              signingKeySecretRef:
+                                description: SecretKeySelector selects a key of a
+                                  Secret.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                              token:
+                                description: Static pre-generated JWT
+                                type: string
+                              tokenSecretRef:
+                                description: SecretKeySelector selects a key of a
+                                  Secret.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                              ttl:
+                                type: string
+                            type: object
+                            x-kubernetes-validations:
+                            - message: static JWT token and generated JWT configuration
+                                cannot be combined
+                              rule: '!((has(self.token) || has(self.tokenSecretRef))
+                                && (has(self.signingKeySecretRef) || has(self.claims)))'
+                            - message: static JWT token and generated JWT configuration
+                                cannot be combined
+                              rule: '!( (has(self.token) || has(self.tokenSecretRef))
+                                && (has(self.signingKeySecretRef) || has(self.claims))
+                                )'
+                            - message: algorithm must be specified when generating
+                                a JWT
+                              rule: '!has(self.signingKeySecretRef) || self.algorithm
+                                != ”'
+                          token:
+                            properties:
+                              scheme:
+                                minLength: 1
+                                type: string
+                              token:
+                                type: string
+                              tokenSecretRef:
+                                description: SecretKeySelector selects a key of a
+                                  Secret.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                            required:
+                            - scheme
+                            type: object
+                            x-kubernetes-validations:
+                            - message: either token or tokenSecretRef must be set,
+                                but not both
+                              rule: has(self.token) != has(self.tokenSecretRef)
+                        type: object
+                        x-kubernetes-validations:
+                        - message: exactly one of the fields in [basic bearer jwt
+                            token] must be set
+                          rule: '[has(self.basic),has(self.bearer),has(self.jwt),has(self.token)].filter(x,x==true).size()
+                            == 1'
+                      interval:
+                        type: string
+                      mapping:
+                        description: JSONPath-style expressions
+                        properties:
+                          address:
+                            type: string
+                          labels:
+                            additionalProperties:
+                              type: string
+                            type: object
+                          name:
+                            type: string
+                          port:
+                            type: string
+                        required:
+                        - address
+                        - name
+                        type: object
+                      pagination:
+                        properties:
+                          enabled:
+                            type: boolean
+                          itemsField:
+                            description: 'Example: "results"'
+                            type: string
+                          nextField:
+                            description: 'Example: "next"'
+                            type: string
+                        required:
+                        - enabled
+                        type: object
+                      timeout:
+                        type: string
+                      tls:
+                        properties:
+                          caSecretRef:
+                            description: SecretKeySelector selects a key of a Secret.
+                            properties:
+                              key:
+                                description: The key of the secret to select from.  Must
+                                  be a valid secret key.
+                                type: string
+                              name:
+                                default: ""
+                                description: |-
+                                  Name of the referent.
+                                  This field is effectively required, but due to backwards compatibility is
+                                  allowed to be empty. Instances of this type with an empty value here are
+                                  almost certainly wrong.
+                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                type: string
+                              optional:
+                                description: Specify whether the Secret or its key
+                                  must be defined
+                                type: boolean
+                            required:
+                            - key
+                            type: object
+                            x-kubernetes-map-type: atomic
+                          insecureSkipVerify:
+                            type: boolean
+                        type: object
                       url:
                         minLength: 1
                         type: string
@@ -59,9 +317,8 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-validations:
-                - message: exactly one of the fields in [http consul] must be set
-                  rule: '[has(self.http),has(self.consul)].filter(x,x==true).size()
-                    == 1'
+                - message: exactly one of the fields in [http] must be set
+                  rule: '[has(self.http)].filter(x,x==true).size() == 1'
               targetLabels:
                 additionalProperties:
                   type: string

From 85278df46d3d8f217928f9783d631b78cf1daec8 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 6 May 2026 08:14:09 +0000
Subject: [PATCH 090/165] use httpconfig within loader

---
 internal/controller/discovery/loaders.go      |  2 --
 .../discovery/loaders/http/loader.go          | 20 +++++++++----------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index c75c5fa..646fb0a 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -15,8 +15,6 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec
 	case spec.Provider.HTTP != nil:
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
 		return http.New(cfg, *spec.Provider.HTTP), cfg, nil
-	case spec.Provider.Consul != nil:
-		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
 		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index b6941e9..a009b76 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -3,13 +3,11 @@ package http
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/google/uuid"
-	"k8s.io/kube-openapi/pkg/validation/spec"
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -54,9 +52,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	logger.Info("HTTP loader started")
 
 	// Input Validation of spec
-	if spec. == nil {
-		return errors.New("HTTP loader requires spec.provider.http to be set")
-	}
+	// if l.spec.URL == "nil" {
+	// 	return errors.New("HTTP loader requires spec.provider.http to be set")
+	// }
 
 	client := &http.Client{
 		Timeout: defaultTimeoutSeconds * time.Second,
@@ -68,7 +66,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	logger.Info(
 		"HTTP polling discovery started",
 		"interval", interval.String(),
-		"url", spec.Provider.HTTP.URL,
+		"url", l.spec.URL,
 	)
 
 	// helper function to fetch targets and emit discovery messages
@@ -76,14 +74,15 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 		targets, err := l.fetchTargetsFromHTTPEndpoint(
 			ctx,
 			client,
-			spec.Provider.HTTP.URL,
-			spec.Provider.HTTP.Token,
+			l.spec.URL,
+			l.spec.Authorization.Token.Scheme,
+			l.spec.Authorization.Token.Token,
 		)
 		if err != nil {
 			logger.Error(
 				err,
 				"Failed to fetch targets from HTTP endpoint",
-				"url", spec.Provider.HTTP.URL,
+				"url", l.spec.URL,
 			)
 			return
 		}
@@ -126,6 +125,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,
 	url string,
+	scheme string,
 	token string,
 ) ([]core.DiscoveredTarget, error) {
 
@@ -135,7 +135,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	}
 
 	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Authorization", "Token "+token)
+	req.Header.Set("Authorization", fmt.Sprintf("%s %s", scheme, token))
 
 	resp, err := client.Do(req)
 	if err != nil {

From 6c82320ff05d704910675eb011e6e05c882b0d4a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 6 May 2026 08:27:17 +0000
Subject: [PATCH 091/165] refactor

---
 api/v1alpha1/targetsource_types.go                     | 4 ++--
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 7 +++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index ab48614..44e605e 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -97,8 +97,8 @@ type TokenAuthSpec struct {
 }
 
 // +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder:validation:XValidation:rule="!( (has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)) )",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != ”",message="algorithm must be specified when generating a JWT"
+// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
 type JWTAuthSpec struct {
 	// Static pre-generated JWT
 	Token          string                    `json:"token,omitempty"`
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 23360c5..65c48c0 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -194,13 +194,12 @@ spec:
                                 && (has(self.signingKeySecretRef) || has(self.claims)))'
                             - message: static JWT token and generated JWT configuration
                                 cannot be combined
-                              rule: '!( (has(self.token) || has(self.tokenSecretRef))
-                                && (has(self.signingKeySecretRef) || has(self.claims))
-                                )'
+                              rule: '!((has(self.token) || has(self.tokenSecretRef))
+                                && (has(self.signingKeySecretRef) || has(self.claims)))'
                             - message: algorithm must be specified when generating
                                 a JWT
                               rule: '!has(self.signingKeySecretRef) || self.algorithm
-                                != ”'
+                                != ""'
                           token:
                             properties:
                               scheme:

From deb9e90802f6138f1f71eb25ea4531bd2812d15b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 8 May 2026 12:37:00 +0000
Subject: [PATCH 092/165] git ignore sonar scanner

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 7515fa3..ef68c8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,4 +38,5 @@ test/integration/clab-*
 # Only for development and testing purposes
 # To be removed after development of targetsource
 # ignored in order to not add unnecassary logging messages
-lab/dev/resources/targetsources
\ No newline at end of file
+lab/dev/resources/targetsources
+.scannerwork/
\ No newline at end of file

From b088db2eb0cae0fd5f9cc5142b42961d29365954 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 8 May 2026 15:27:15 +0000
Subject: [PATCH 093/165] add defaulting for targetsource crd

---
 api/v1alpha1/targetsource_types.go            |  55 ++++----
 api/v1alpha1/zz_generated.deepcopy.go         | 122 +++++++++++-------
 .../operator.gnmic.dev_targetsources.yaml     |  53 ++------
 internal/controller/discovery/loaders.go      |   2 +-
 .../discovery/loaders/http/const.go           |   9 ++
 .../discovery/loaders/http/loader.go          |  27 ++--
 .../webhook/v1alpha1/targetsource_webhook.go  |  17 ++-
 7 files changed, 145 insertions(+), 140 deletions(-)
 create mode 100644 internal/controller/discovery/loaders/http/const.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 44e605e..78a2f74 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -38,13 +38,10 @@ type ProviderSpec struct {
 	HTTP *HTTPConfig `json:"http,omitempty"`
 }
 
-type WebhookSpec struct {
-	Enabled *bool `json:"enabled,omitempty"`
-}
-
+// +kubebuilder:validation:AtLeastOneOf=url;acceptPush
 type HTTPConfig struct {
-	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url"`
+	// +kubebuilder:validation:Optional
+	URL string `json:"url,omitempty"`
 	// +kubebuilder:validation:Optional
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
 	// +kubebuilder:validation:Optional
@@ -58,41 +55,34 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
 	// +kubebuilder:validation:Optional
-	AcceptPush bool `json:"acceptPush,omitempty"`
+	AcceptPush *bool `json:"acceptPush,omitempty"`
 }
 
 type ClientTLSConfig struct {
-	InsecureSkipVerify bool                      `json:"insecureSkipVerify,omitempty"`
+	InsecureSkipVerify *bool                     `json:"insecureSkipVerify,omitempty"`
 	CASecretRef        *corev1.SecretKeySelector `json:"caSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=basic;bearer;jwt;token
+// +kubebuilder:validation:ExactlyOneOf=basic;jwt;token
 type AuthorizationSpec struct {
-	Basic  *BasicAuthSpec  `json:"basic,omitempty"`
-	Bearer *BearerAuthSpec `json:"bearer,omitempty"`
-	Token  *TokenAuthSpec  `json:"token,omitempty"`
-	JWT    *JWTAuthSpec    `json:"jwt,omitempty"`
+	Basic *BasicAuthSpec `json:"basic,omitempty"`
+	Token *TokenAuthSpec `json:"token,omitempty"`
+	JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
 }
 
 // Enforce EITHER inline creds OR secret ref
 // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
-	Username             string                    `json:"username,omitempty"`
-	Password             string                    `json:"password,omitempty"`
+	Username             *string                   `json:"username,omitempty"`
+	Password             *string                   `json:"password,omitempty"`
 	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=token;tokenSecretRef
-type BearerAuthSpec struct {
-	Token          string                    `json:"token,omitempty"`
-	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
-}
-
 // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
 type TokenAuthSpec struct {
 	// +kubebuilder:validation:MinLength=1
-	Scheme         string                    `json:"scheme"`
-	Token          string                    `json:"token,omitempty"`
+	Scheme         *string                   `json:"scheme"`
+	Token          *string                   `json:"token,omitempty"`
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
@@ -101,36 +91,35 @@ type TokenAuthSpec struct {
 // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
 type JWTAuthSpec struct {
 	// Static pre-generated JWT
-	Token          string                    `json:"token,omitempty"`
+	Token          *string                   `json:"token,omitempty"`
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 	// Optional: generate JWT dynamically
 	Claims              map[string]string         `json:"claims,omitempty"`
 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
 	// HS256, RS256, ES256, etc.
-	Algorithm string           `json:"algorithm,omitempty"`
+	Algorithm *string          `json:"algorithm,omitempty"`
 	TTL       *metav1.Duration `json:"ttl,omitempty"`
 }
 
 type PaginationSpec struct {
-	Enabled bool `json:"enabled"`
 	// Example: "results"
-	ItemsField string `json:"itemsField,omitempty"`
+	ItemsField *string `json:"itemsField,omitempty"`
 	// Example: "next"
-	NextField string `json:"nextField,omitempty"`
+	NextField *string `json:"nextField,omitempty"`
 }
 
 // JSONPath-style expressions
 type ResponseMappingSpec struct {
-	Name    string            `json:"name"`
-	Address string            `json:"address"`
-	Port    string            `json:"port,omitempty"`
+	Name    *string           `json:"name"`
+	Address *string           `json:"address"`
+	Port    *string           `json:"port,omitempty"`
 	Labels  map[string]string `json:"labels,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
 type TargetSourceStatus struct {
-	Status       string      `json:"status"`
-	TargetsCount int32       `json:"targetsCount"`
+	Status       *string     `json:"status"`
+	TargetsCount *int32      `json:"targetsCount"`
 	LastSync     metav1.Time `json:"lastSync"`
 }
 
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 602ede5..453dcc9 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -54,11 +54,6 @@ func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) {
 		*out = new(BasicAuthSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.Bearer != nil {
-		in, out := &in.Bearer, &out.Bearer
-		*out = new(BearerAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
 	if in.Token != nil {
 		in, out := &in.Token, &out.Token
 		*out = new(TokenAuthSpec)
@@ -84,6 +79,16 @@ func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) {
 	*out = *in
+	if in.Username != nil {
+		in, out := &in.Username, &out.Username
+		*out = new(string)
+		**out = **in
+	}
+	if in.Password != nil {
+		in, out := &in.Password, &out.Password
+		*out = new(string)
+		**out = **in
+	}
 	if in.CredentialsSecretRef != nil {
 		in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef
 		*out = new(v1.SecretKeySelector)
@@ -101,29 +106,14 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *BearerAuthSpec) DeepCopyInto(out *BearerAuthSpec) {
-	*out = *in
-	if in.TokenSecretRef != nil {
-		in, out := &in.TokenSecretRef, &out.TokenSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BearerAuthSpec.
-func (in *BearerAuthSpec) DeepCopy() *BearerAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(BearerAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
+	if in.InsecureSkipVerify != nil {
+		in, out := &in.InsecureSkipVerify, &out.InsecureSkipVerify
+		*out = new(bool)
+		**out = **in
+	}
 	if in.CASecretRef != nil {
 		in, out := &in.CASecretRef, &out.CASecretRef
 		*out = new(v1.SecretKeySelector)
@@ -376,13 +366,18 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	if in.Pagination != nil {
 		in, out := &in.Pagination, &out.Pagination
 		*out = new(PaginationSpec)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.ResponseMapping != nil {
 		in, out := &in.ResponseMapping, &out.ResponseMapping
 		*out = new(ResponseMappingSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.AcceptPush != nil {
+		in, out := &in.AcceptPush, &out.AcceptPush
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig.
@@ -527,6 +522,11 @@ func (in *InputStatus) DeepCopy() *InputStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
 	*out = *in
+	if in.Token != nil {
+		in, out := &in.Token, &out.Token
+		*out = new(string)
+		**out = **in
+	}
 	if in.TokenSecretRef != nil {
 		in, out := &in.TokenSecretRef, &out.TokenSecretRef
 		*out = new(v1.SecretKeySelector)
@@ -544,6 +544,11 @@ func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
 		*out = new(v1.SecretKeySelector)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Algorithm != nil {
+		in, out := &in.Algorithm, &out.Algorithm
+		*out = new(string)
+		**out = **in
+	}
 	if in.TTL != nil {
 		in, out := &in.TTL, &out.TTL
 		*out = new(metav1.Duration)
@@ -737,6 +742,16 @@ func (in *OutputStatus) DeepCopy() *OutputStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) {
 	*out = *in
+	if in.ItemsField != nil {
+		in, out := &in.ItemsField, &out.ItemsField
+		*out = new(string)
+		**out = **in
+	}
+	if in.NextField != nil {
+		in, out := &in.NextField, &out.NextField
+		*out = new(string)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec.
@@ -1003,6 +1018,21 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
 	*out = *in
+	if in.Name != nil {
+		in, out := &in.Name, &out.Name
+		*out = new(string)
+		**out = **in
+	}
+	if in.Address != nil {
+		in, out := &in.Address, &out.Address
+		*out = new(string)
+		**out = **in
+	}
+	if in.Port != nil {
+		in, out := &in.Port, &out.Port
+		*out = new(string)
+		**out = **in
+	}
 	if in.Labels != nil {
 		in, out := &in.Labels, &out.Labels
 		*out = make(map[string]string, len(*in))
@@ -1493,6 +1523,16 @@ func (in *TargetSourceSpec) DeepCopy() *TargetSourceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TargetSourceStatus) DeepCopyInto(out *TargetSourceStatus) {
 	*out = *in
+	if in.Status != nil {
+		in, out := &in.Status, &out.Status
+		*out = new(string)
+		**out = **in
+	}
+	if in.TargetsCount != nil {
+		in, out := &in.TargetsCount, &out.TargetsCount
+		*out = new(int32)
+		**out = **in
+	}
 	in.LastSync.DeepCopyInto(&out.LastSync)
 }
 
@@ -1566,6 +1606,16 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) {
 	*out = *in
+	if in.Scheme != nil {
+		in, out := &in.Scheme, &out.Scheme
+		*out = new(string)
+		**out = **in
+	}
+	if in.Token != nil {
+		in, out := &in.Token, &out.Token
+		*out = new(string)
+		**out = **in
+	}
 	if in.TokenSecretRef != nil {
 		in, out := &in.TokenSecretRef, &out.TokenSecretRef
 		*out = new(v1.SecretKeySelector)
@@ -1676,23 +1726,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
-	*out = *in
-	if in.Enabled != nil {
-		in, out := &in.Enabled, &out.Enabled
-		*out = new(bool)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
-func (in *WebhookSpec) DeepCopy() *WebhookSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookSpec)
-	in.DeepCopyInto(out)
-	return out
-}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 65c48c0..588bccf 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -86,41 +86,6 @@ spec:
                               rule: (has(self.credentialsSecretRef) && !has(self.username)
                                 && !has(self.password)) || (!has(self.credentialsSecretRef)
                                 && has(self.username) && has(self.password))
-                          bearer:
-                            properties:
-                              token:
-                                type: string
-                              tokenSecretRef:
-                                description: SecretKeySelector selects a key of a
-                                  Secret.
-                                properties:
-                                  key:
-                                    description: The key of the secret to select from.  Must
-                                      be a valid secret key.
-                                    type: string
-                                  name:
-                                    default: ""
-                                    description: |-
-                                      Name of the referent.
-                                      This field is effectively required, but due to backwards compatibility is
-                                      allowed to be empty. Instances of this type with an empty value here are
-                                      almost certainly wrong.
-                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                                    type: string
-                                  optional:
-                                    description: Specify whether the Secret or its
-                                      key must be defined
-                                    type: boolean
-                                required:
-                                - key
-                                type: object
-                                x-kubernetes-map-type: atomic
-                            type: object
-                            x-kubernetes-validations:
-                            - message: exactly one of the fields in [token tokenSecretRef]
-                                must be set
-                              rule: '[has(self.token),has(self.tokenSecretRef)].filter(x,x==true).size()
-                                == 1'
                           jwt:
                             properties:
                               algorithm:
@@ -241,9 +206,9 @@ spec:
                               rule: has(self.token) != has(self.tokenSecretRef)
                         type: object
                         x-kubernetes-validations:
-                        - message: exactly one of the fields in [basic bearer jwt
-                            token] must be set
-                          rule: '[has(self.basic),has(self.bearer),has(self.jwt),has(self.token)].filter(x,x==true).size()
+                        - message: exactly one of the fields in [basic jwt token]
+                            must be set
+                          rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
                         type: string
@@ -266,16 +231,12 @@ spec:
                         type: object
                       pagination:
                         properties:
-                          enabled:
-                            type: boolean
                           itemsField:
                             description: 'Example: "results"'
                             type: string
                           nextField:
                             description: 'Example: "next"'
                             type: string
-                        required:
-                        - enabled
                         type: object
                       timeout:
                         type: string
@@ -309,11 +270,13 @@ spec:
                             type: boolean
                         type: object
                       url:
-                        minLength: 1
                         type: string
-                    required:
-                    - url
                     type: object
+                    x-kubernetes-validations:
+                    - message: at least one of the fields in [url acceptPush] must
+                        be set
+                      rule: '[has(self.url),has(self.acceptPush)].filter(x,x==true).size()
+                        >= 1'
                 type: object
                 x-kubernetes-validations:
                 - message: exactly one of the fields in [http] must be set
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 646fb0a..66bec04 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -13,7 +13,7 @@ func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
+		cfg.AcceptPush = *spec.Provider.HTTP.AcceptPush
 		return http.New(cfg, *spec.Provider.HTTP), cfg, nil
 	default:
 		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
diff --git a/internal/controller/discovery/loaders/http/const.go b/internal/controller/discovery/loaders/http/const.go
new file mode 100644
index 0000000..4b18f6c
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/const.go
@@ -0,0 +1,9 @@
+package http
+
+import "time"
+
+const (
+	DefaultPollInterval = 1 * time.Hour
+	DefaultTimeout      = 30 * time.Second
+	DefaultAcceptPush   = false
+)
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index a009b76..b55bcef 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -16,20 +16,15 @@ import (
 	loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils"
 )
 
-const (
-	defaultPollInterval   = 30 * time.Second
-	defaultTimeoutSeconds = 30
-)
-
 // Loader implements the HTTP pull discovery mechanism
 type Loader struct {
-	commonCfg core.CommonLoaderConfig
+	loaderCfg core.CommonLoaderConfig
 	spec      *gnmicv1alpha1.HTTPConfig
 }
 
 // New instantiates the http loader with the provided config
 func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader {
-	return &Loader{commonCfg: cfg, spec: &httpConfig}
+	return &Loader{loaderCfg: cfg, spec: &httpConfig}
 }
 
 func (l *Loader) Name() string {
@@ -37,16 +32,20 @@ func (l *Loader) Name() string {
 }
 
 func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error {
+	if l.spec.URL == "" {
+		return nil
+	}
+
 	logger := log.FromContext(ctx).WithValues(
 		"component", "loader",
 		"name", l.Name(),
-		"targetsource", l.commonCfg.TargetsourceNN,
+		"targetsource", l.loaderCfg.TargetsourceNN,
 	)
 
 	logger.Info(
 		"HTTP loader started",
-		"targetsource", l.commonCfg.TargetsourceNN.Name,
-		"namespace", l.commonCfg.TargetsourceNN.Namespace,
+		"targetsource", l.loaderCfg.TargetsourceNN.Name,
+		"namespace", l.loaderCfg.TargetsourceNN.Namespace,
 	)
 
 	logger.Info("HTTP loader started")
@@ -57,9 +56,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	// }
 
 	client := &http.Client{
-		Timeout: defaultTimeoutSeconds * time.Second,
+		Timeout: l.spec.Timeout.Duration,
 	}
-	interval := defaultPollInterval
+	interval := l.spec.PollInterval.Duration
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
@@ -87,8 +86,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			return
 		}
 
-		snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString())
-		if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil {
+		snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString())
+		if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.loaderCfg.ChunkSize); err != nil {
 			logger.Error(
 				err,
 				"Failed to send discovery snapshot",
diff --git a/internal/webhook/v1alpha1/targetsource_webhook.go b/internal/webhook/v1alpha1/targetsource_webhook.go
index b3eb960..992d67b 100644
--- a/internal/webhook/v1alpha1/targetsource_webhook.go
+++ b/internal/webhook/v1alpha1/targetsource_webhook.go
@@ -21,12 +21,14 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	httpDefaults "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
 
 // nolint:unused
@@ -65,7 +67,20 @@ func (d *TargetSourceCustomDefaulter) Default(_ context.Context, obj runtime.Obj
 	}
 	targetsourcelog.Info("Defaulting for TargetSource", "name", targetsource.GetName())
 
-	// TODO(user): fill in your defaulting logic.
+	// HTTP Config Defaulting
+	if targetsource.Spec.Provider.HTTP != nil {
+		http := targetsource.Spec.Provider.HTTP
+
+		if http.PollInterval == nil {
+			http.PollInterval.Duration = httpDefaults.DefaultPollInterval
+		}
+		if http.Timeout == nil {
+			http.Timeout.Duration = httpDefaults.DefaultTimeout
+		}
+		if http.AcceptPush == nil {
+			http.AcceptPush = pointer.Bool(httpDefaults.DefaultAcceptPush)
+		}
+	}
 
 	return nil
 }

From 9208766b0cc3a0eeeaf0e82270a818bd724e4cc5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 07:43:33 +0000
Subject: [PATCH 094/165] remove closeChannel and fix cleanup logic

---
 internal/controller/targetsource_controller.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index c65e254..ccdf430 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -177,7 +177,6 @@ func (r *TargetSourceReconciler) startDiscovery(
 	cleanup := func() {
 		cancel()
 		r.DiscoveryRegistry.Unregister(key)
-		close(targetChannel)
 	}
 
 	messageProcessor := discovery.NewMessageProcessor(
@@ -225,9 +224,6 @@ func (r *TargetSourceReconciler) startDiscovery(
 		} else {
 			logger.Error(nil, "Target loader exited unexpectedly without error")
 		}
-
-		// Any exit is considered a bug that should stop the discovery runtime
-		cleanup()
 	}()
 
 	return nil

From 209948e2e1c2cebb1bbc78d4183d6be6bcdd3805 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 07:54:23 +0000
Subject: [PATCH 095/165] fix: resolved pointer and returns smells

---
 internal/controller/discovery/loaders.go       | 8 ++++----
 internal/controller/targetsource_controller.go | 4 +---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 7f2c656..f882aa2 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -9,16 +9,16 @@ import (
 )
 
 // NewLoader creates a loader by name
-func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, core.CommonLoaderConfig, error) {
+func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
 
 	switch {
 	case spec.Provider.HTTP != nil:
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
-		return http.New(cfg), cfg, nil
+		return http.New(*cfg), nil
 	case spec.Provider.Consul != nil:
-		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
-		return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 
 }
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index ccdf430..522aabd 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -185,9 +185,7 @@ func (r *TargetSourceReconciler) startDiscovery(
 		targetSource,
 		targetChannel,
 	)
-	loader, loaderConfig, err := discovery.NewLoader(loaderConfig,
-		&targetSource.Spec,
-	)
+	loader, err := discovery.NewLoader(&loaderConfig, targetSource.Spec)
 	if err != nil {
 		logger.Error(err, "Target loader could not be created")
 		cleanup()

From 1a0f4476b6149efef34bd38f599b29c0faedd6e4 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:04:50 +0000
Subject: [PATCH 096/165] improved logging message

---
 internal/controller/discovery/loaders.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index f882aa2..c888c27 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -16,9 +16,9 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
 		return http.New(*cfg), nil
 	case spec.Provider.Consul != nil:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, fmt.Errorf("Unimplemented targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
-		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN)
+		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 
 }

From d94c23fb5f20c061e95ff148d0e8e4bfae6e98f5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:23:48 +0000
Subject: [PATCH 097/165] improved error handling

---
 .../controller/discovery/message_processor.go | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index ed66940..b16603e 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -127,8 +127,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc
 // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly
 func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
 	if m.activeSnapshot == nil {
-		m.startNewSnapshot(chunk, logger)
-		return nil
+		return m.startNewSnapshot(ctx, chunk, logger)
 	}
 
 	snapshot := m.activeSnapshot
@@ -149,14 +148,13 @@ func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.Disco
 		}
 
 		// Start collecting the new snapshot
-		m.startNewSnapshot(chunk, logger)
-		return nil
+		return m.startNewSnapshot(ctx, chunk, logger)
 	}
 
-	return m.collectSnapshot(chunk, logger)
+	return m.collectSnapshot(ctx, chunk, logger)
 }
 
-func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) {
+func (m *MessageProcessor) startNewSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
 	m.activeSnapshot = &snapshotBuffer{
 		snapshotID:  chunk.SnapshotID,
 		totalChunks: chunk.TotalChunks,
@@ -166,10 +164,10 @@ func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger
 	// Delete buffered events that will be current with new snapshot
 	m.deferredEvents = nil
 
-	m.collectSnapshot(chunk, logger)
+	return m.collectSnapshot(ctx, chunk, logger)
 }
 
-func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error {
+func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error {
 	snapshot := m.activeSnapshot
 
 	if chunk.TotalChunks != snapshot.totalChunks {
@@ -178,6 +176,7 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger
 			"Snapshot totalChunks mismatch",
 			"snapshotID", snapshot.snapshotID,
 		)
+		return fmt.Errorf("snapshot totalChunks mismatch")
 	}
 	if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks {
 		logger.Error(
@@ -186,7 +185,7 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger
 			"chunkIndex", chunk.ChunkIndex,
 		)
 		m.activeSnapshot = nil
-		return nil
+		return fmt.Errorf("invalid chunk index")
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
 		logger.Error(
@@ -195,13 +194,14 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger
 			"chunkIndex", chunk.ChunkIndex,
 		)
 		m.activeSnapshot = nil
-		return nil
+		return fmt.Errorf("duplicate snapshot chunk")
 	}
 
 	snapshot.received[chunk.ChunkIndex] = chunk.Targets
 
 	if len(snapshot.received) == snapshot.totalChunks {
 		snapshot.complete = true
+		return m.applySnapshot(ctx, snapshot, logger)
 	}
 
 	return nil
@@ -232,7 +232,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 				"chunkIndex", i,
 			)
 			m.activeSnapshot = nil
-			return nil
+			return fmt.Errorf("missing snapshot chunk %d", i)
 		}
 		allTargets = append(allTargets, chunk...)
 	}
@@ -243,7 +243,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 		"targets", len(allTargets),
 	)
 
-	// apply all targets
+	// todo: apply all targets
 	// a.applyTargets
 
 	// Replay deferred events

From e3f18d8a213be2afd81f5c617eaae1b7ad068652 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:26:05 +0000
Subject: [PATCH 098/165] refactor: ctx should flow not be stored

---
 internal/controller/discovery/message_processor.go | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index b16603e..a95a208 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -22,7 +22,6 @@ type snapshotBuffer struct {
 
 // MessageProcessor consumes discovery messages and applies them to Kubernetes
 type MessageProcessor struct {
-	ctx            context.Context
 	client         client.Client
 	scheme         *runtime.Scheme
 	targetSource   *gnmicv1alpha1.TargetSource
@@ -46,8 +45,6 @@ func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T
 // Run is a long‑running loop that receives target snapshots
 // and reconciles Target CRs accordingly
 func (m *MessageProcessor) Run(ctx context.Context) error {
-	m.ctx = ctx
-
 	logger := log.FromContext(ctx).WithValues(
 		"component", "message-processor",
 		"targetsource", m.targetSource.Name,
@@ -56,7 +53,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 
 	logger.Info("Message processor started")
 
-	for m.ctx.Err() == nil {
+	for ctx.Err() == nil {
 		select {
 		case batch, ok := <-m.in:
 			if !ok {
@@ -79,7 +76,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 			msg := m.queue[0]
 			m.queue = m.queue[1:]
 
-			if err := m.processMessage(m.ctx, msg, logger); err != nil {
+			if err := m.processMessage(ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
 				// Q: when to return an error vs just log and continue?

From bc1b3508e7e34f2f5c199f448bf7baceb2966b6a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:30:48 +0000
Subject: [PATCH 099/165] refactor: resetSnapshot

---
 .../controller/discovery/message_processor.go    | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index a95a208..bbefd24 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -181,7 +181,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco
 			"Snapshot chunk index out of range",
 			"chunkIndex", chunk.ChunkIndex,
 		)
-		m.activeSnapshot = nil
+		m.resetSnapshot()
 		return fmt.Errorf("invalid chunk index")
 	}
 	if _, exists := snapshot.received[chunk.ChunkIndex]; exists {
@@ -190,7 +190,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco
 			"Duplicate snapshot chunk received",
 			"chunkIndex", chunk.ChunkIndex,
 		)
-		m.activeSnapshot = nil
+		m.resetSnapshot()
 		return fmt.Errorf("duplicate snapshot chunk")
 	}
 
@@ -207,7 +207,7 @@ func (m *MessageProcessor) collectSnapshot(ctx context.Context, chunk core.Disco
 func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error {
 	select {
 	case <-ctx.Done():
-		m.activeSnapshot = nil
+		m.resetSnapshot()
 		return nil
 	default:
 	}
@@ -216,7 +216,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 	for i := 0; i < snapshot.totalChunks; i++ {
 		select {
 		case <-ctx.Done():
-			m.activeSnapshot = nil
+			m.resetSnapshot()
 			return nil
 		default:
 		}
@@ -228,7 +228,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 				"Missing snapshot chunk",
 				"chunkIndex", i,
 			)
-			m.activeSnapshot = nil
+			m.resetSnapshot()
 			return fmt.Errorf("missing snapshot chunk %d", i)
 		}
 		allTargets = append(allTargets, chunk...)
@@ -255,7 +255,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot
 		}
 	}
 
-	m.activeSnapshot = nil
+	m.resetSnapshot()
 	m.deferredEvents = nil
 	return nil
 }
@@ -290,3 +290,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE
 	}
 	return nil
 }
+
+func (m *MessageProcessor) resetSnapshot() {
+	m.activeSnapshot = nil
+}

From 020be5ae1d507b05051b37edc59bb5d6fc123dc0 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:31:51 +0000
Subject: [PATCH 100/165] refactor: context cancellation

---
 internal/controller/discovery/message_processor.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index bbefd24..b1d893e 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -53,7 +53,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 
 	logger.Info("Message processor started")
 
-	for ctx.Err() == nil {
+	for {
 		select {
 		case batch, ok := <-m.in:
 			if !ok {

From 3280229ed3374583925b4c3600dea5f2115d219e Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 08:48:30 +0000
Subject: [PATCH 101/165] refactor: default error handling now logs errors
 instead of terminating the message processor

---
 internal/controller/discovery/message_processor.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index b1d893e..f7aafb1 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -79,15 +79,15 @@ func (m *MessageProcessor) Run(ctx context.Context) error {
 			if err := m.processMessage(ctx, msg, logger); err != nil {
 				// Returning error lets the supervisor (controller)
 				// tear down and restart the pipeline via reconciliation
-				// Q: when to return an error vs just log and continue?
-				return err
+				logger.Info(
+					"Could not process the message",
+					"error", err,
+				)
+				return nil
 			}
 
 		}
 	}
-
-	logger.Info("Message processor stopped")
-	return nil
 }
 
 func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error {

From 39f16502bd1e399cf30567818ee5b22a1104e24d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 09:21:34 +0000
Subject: [PATCH 102/165] refactor: pointer missuse

---
 internal/controller/discovery/loaders/http/loader.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index b55bcef..e791af9 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -19,12 +19,12 @@ import (
 // Loader implements the HTTP pull discovery mechanism
 type Loader struct {
 	loaderCfg core.CommonLoaderConfig
-	spec      *gnmicv1alpha1.HTTPConfig
+	spec      gnmicv1alpha1.HTTPConfig
 }
 
 // New instantiates the http loader with the provided config
 func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader {
-	return &Loader{loaderCfg: cfg, spec: &httpConfig}
+	return &Loader{loaderCfg: cfg, spec: httpConfig}
 }
 
 func (l *Loader) Name() string {
@@ -74,8 +74,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			ctx,
 			client,
 			l.spec.URL,
-			l.spec.Authorization.Token.Scheme,
-			l.spec.Authorization.Token.Token,
+			*l.spec.Authorization.Token.Scheme,
+			*l.spec.Authorization.Token.Token,
 		)
 		if err != nil {
 			logger.Error(

From f1d8c3165c9687107fa02a47f47ea92bff83c3be Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 09:35:05 +0000
Subject: [PATCH 103/165] move defaulting logic to kubebuilder:default

---
 api/v1alpha1/targetsource_types.go            | 32 +++++----
 api/v1alpha1/zz_generated.deepcopy.go         | 72 +------------------
 .../operator.gnmic.dev_targetsources.yaml     |  3 +
 .../discovery/loaders/http/loader.go          |  4 +-
 .../webhook/v1alpha1/targetsource_webhook.go  | 17 +----
 5 files changed, 25 insertions(+), 103 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 78a2f74..0ca98ab 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -44,8 +44,11 @@ type HTTPConfig struct {
 	URL string `json:"url,omitempty"`
 	// +kubebuilder:validation:Optional
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
+	// TODO: increase default value
+	// +kubebuilder:default="30s"
 	// +kubebuilder:validation:Optional
 	PollInterval *metav1.Duration `json:"interval,omitempty"`
+	// +kubebuilder:default="10s"
 	// +kubebuilder:validation:Optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 	// +kubebuilder:validation:Optional
@@ -54,8 +57,9 @@ type HTTPConfig struct {
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
 	// +kubebuilder:validation:Optional
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
+	// +kubebuilder:default=false
 	// +kubebuilder:validation:Optional
-	AcceptPush *bool `json:"acceptPush,omitempty"`
+	AcceptPush bool `json:"acceptPush,omitempty"`
 }
 
 type ClientTLSConfig struct {
@@ -73,16 +77,16 @@ type AuthorizationSpec struct {
 // Enforce EITHER inline creds OR secret ref
 // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
-	Username             *string                   `json:"username,omitempty"`
-	Password             *string                   `json:"password,omitempty"`
+	Username             string                    `json:"username,omitempty"`
+	Password             string                    `json:"password,omitempty"`
 	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
 // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
 type TokenAuthSpec struct {
 	// +kubebuilder:validation:MinLength=1
-	Scheme         *string                   `json:"scheme"`
-	Token          *string                   `json:"token,omitempty"`
+	Scheme         string                    `json:"scheme"`
+	Token          string                    `json:"token,omitempty"`
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
@@ -91,35 +95,35 @@ type TokenAuthSpec struct {
 // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
 type JWTAuthSpec struct {
 	// Static pre-generated JWT
-	Token          *string                   `json:"token,omitempty"`
+	Token          string                    `json:"token,omitempty"`
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 	// Optional: generate JWT dynamically
 	Claims              map[string]string         `json:"claims,omitempty"`
 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
 	// HS256, RS256, ES256, etc.
-	Algorithm *string          `json:"algorithm,omitempty"`
+	Algorithm string           `json:"algorithm,omitempty"`
 	TTL       *metav1.Duration `json:"ttl,omitempty"`
 }
 
 type PaginationSpec struct {
 	// Example: "results"
-	ItemsField *string `json:"itemsField,omitempty"`
+	ItemsField string `json:"itemsField,omitempty"`
 	// Example: "next"
-	NextField *string `json:"nextField,omitempty"`
+	NextField string `json:"nextField,omitempty"`
 }
 
 // JSONPath-style expressions
 type ResponseMappingSpec struct {
-	Name    *string           `json:"name"`
-	Address *string           `json:"address"`
-	Port    *string           `json:"port,omitempty"`
+	Name    string            `json:"name"`
+	Address string            `json:"address"`
+	Port    string            `json:"port,omitempty"`
 	Labels  map[string]string `json:"labels,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
 type TargetSourceStatus struct {
-	Status       *string     `json:"status"`
-	TargetsCount *int32      `json:"targetsCount"`
+	Status       string      `json:"status"`
+	TargetsCount int32       `json:"targetsCount"`
 	LastSync     metav1.Time `json:"lastSync"`
 }
 
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 453dcc9..d87d54d 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -79,16 +79,6 @@ func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) {
 	*out = *in
-	if in.Username != nil {
-		in, out := &in.Username, &out.Username
-		*out = new(string)
-		**out = **in
-	}
-	if in.Password != nil {
-		in, out := &in.Password, &out.Password
-		*out = new(string)
-		**out = **in
-	}
 	if in.CredentialsSecretRef != nil {
 		in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef
 		*out = new(v1.SecretKeySelector)
@@ -366,18 +356,13 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	if in.Pagination != nil {
 		in, out := &in.Pagination, &out.Pagination
 		*out = new(PaginationSpec)
-		(*in).DeepCopyInto(*out)
+		**out = **in
 	}
 	if in.ResponseMapping != nil {
 		in, out := &in.ResponseMapping, &out.ResponseMapping
 		*out = new(ResponseMappingSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.AcceptPush != nil {
-		in, out := &in.AcceptPush, &out.AcceptPush
-		*out = new(bool)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig.
@@ -522,11 +507,6 @@ func (in *InputStatus) DeepCopy() *InputStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
 	*out = *in
-	if in.Token != nil {
-		in, out := &in.Token, &out.Token
-		*out = new(string)
-		**out = **in
-	}
 	if in.TokenSecretRef != nil {
 		in, out := &in.TokenSecretRef, &out.TokenSecretRef
 		*out = new(v1.SecretKeySelector)
@@ -544,11 +524,6 @@ func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
 		*out = new(v1.SecretKeySelector)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.Algorithm != nil {
-		in, out := &in.Algorithm, &out.Algorithm
-		*out = new(string)
-		**out = **in
-	}
 	if in.TTL != nil {
 		in, out := &in.TTL, &out.TTL
 		*out = new(metav1.Duration)
@@ -742,16 +717,6 @@ func (in *OutputStatus) DeepCopy() *OutputStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) {
 	*out = *in
-	if in.ItemsField != nil {
-		in, out := &in.ItemsField, &out.ItemsField
-		*out = new(string)
-		**out = **in
-	}
-	if in.NextField != nil {
-		in, out := &in.NextField, &out.NextField
-		*out = new(string)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec.
@@ -1018,21 +983,6 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
 	*out = *in
-	if in.Name != nil {
-		in, out := &in.Name, &out.Name
-		*out = new(string)
-		**out = **in
-	}
-	if in.Address != nil {
-		in, out := &in.Address, &out.Address
-		*out = new(string)
-		**out = **in
-	}
-	if in.Port != nil {
-		in, out := &in.Port, &out.Port
-		*out = new(string)
-		**out = **in
-	}
 	if in.Labels != nil {
 		in, out := &in.Labels, &out.Labels
 		*out = make(map[string]string, len(*in))
@@ -1523,16 +1473,6 @@ func (in *TargetSourceSpec) DeepCopy() *TargetSourceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TargetSourceStatus) DeepCopyInto(out *TargetSourceStatus) {
 	*out = *in
-	if in.Status != nil {
-		in, out := &in.Status, &out.Status
-		*out = new(string)
-		**out = **in
-	}
-	if in.TargetsCount != nil {
-		in, out := &in.TargetsCount, &out.TargetsCount
-		*out = new(int32)
-		**out = **in
-	}
 	in.LastSync.DeepCopyInto(&out.LastSync)
 }
 
@@ -1606,16 +1546,6 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) {
 	*out = *in
-	if in.Scheme != nil {
-		in, out := &in.Scheme, &out.Scheme
-		*out = new(string)
-		**out = **in
-	}
-	if in.Token != nil {
-		in, out := &in.Token, &out.Token
-		*out = new(string)
-		**out = **in
-	}
 	if in.TokenSecretRef != nil {
 		in, out := &in.TokenSecretRef, &out.TokenSecretRef
 		*out = new(v1.SecretKeySelector)
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 588bccf..2e6872a 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -44,6 +44,7 @@ spec:
                   http:
                     properties:
                       acceptPush:
+                        default: false
                         type: boolean
                       authorization:
                         properties:
@@ -211,6 +212,7 @@ spec:
                           rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
+                        default: 30s
                         type: string
                       mapping:
                         description: JSONPath-style expressions
@@ -239,6 +241,7 @@ spec:
                             type: string
                         type: object
                       timeout:
+                        default: 10s
                         type: string
                       tls:
                         properties:
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index e791af9..bb8a025 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -74,8 +74,8 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			ctx,
 			client,
 			l.spec.URL,
-			*l.spec.Authorization.Token.Scheme,
-			*l.spec.Authorization.Token.Token,
+			l.spec.Authorization.Token.Scheme,
+			l.spec.Authorization.Token.Token,
 		)
 		if err != nil {
 			logger.Error(
diff --git a/internal/webhook/v1alpha1/targetsource_webhook.go b/internal/webhook/v1alpha1/targetsource_webhook.go
index 992d67b..b3eb960 100644
--- a/internal/webhook/v1alpha1/targetsource_webhook.go
+++ b/internal/webhook/v1alpha1/targetsource_webhook.go
@@ -21,14 +21,12 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-	httpDefaults "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
 
 // nolint:unused
@@ -67,20 +65,7 @@ func (d *TargetSourceCustomDefaulter) Default(_ context.Context, obj runtime.Obj
 	}
 	targetsourcelog.Info("Defaulting for TargetSource", "name", targetsource.GetName())
 
-	// HTTP Config Defaulting
-	if targetsource.Spec.Provider.HTTP != nil {
-		http := targetsource.Spec.Provider.HTTP
-
-		if http.PollInterval == nil {
-			http.PollInterval.Duration = httpDefaults.DefaultPollInterval
-		}
-		if http.Timeout == nil {
-			http.Timeout.Duration = httpDefaults.DefaultTimeout
-		}
-		if http.AcceptPush == nil {
-			http.AcceptPush = pointer.Bool(httpDefaults.DefaultAcceptPush)
-		}
-	}
+	// TODO(user): fill in your defaulting logic.
 
 	return nil
 }

From b0c63ff27f437524233e565ea27c262fbe721bd9 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 09:36:02 +0000
Subject: [PATCH 104/165] remove pointer from bool

---
 api/v1alpha1/targetsource_types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 0ca98ab..7d0ca67 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -63,7 +63,7 @@ type HTTPConfig struct {
 }
 
 type ClientTLSConfig struct {
-	InsecureSkipVerify *bool                     `json:"insecureSkipVerify,omitempty"`
+	InsecureSkipVerify bool                     `json:"insecureSkipVerify,omitempty"`
 	CASecretRef        *corev1.SecretKeySelector `json:"caSecretRef,omitempty"`
 }
 

From c422dff6a4a1c0b5affea49db7db00734a267479 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 09:38:55 +0000
Subject: [PATCH 105/165] update deepcopy

---
 api/v1alpha1/zz_generated.deepcopy.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index d87d54d..f115201 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -99,11 +99,6 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
-	if in.InsecureSkipVerify != nil {
-		in, out := &in.InsecureSkipVerify, &out.InsecureSkipVerify
-		*out = new(bool)
-		**out = **in
-	}
 	if in.CASecretRef != nil {
 		in, out := &in.CASecretRef, &out.CASecretRef
 		*out = new(v1.SecretKeySelector)

From abb718089d5e4b066df1eda0ba666c65ff375ba4 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Wed, 13 May 2026 09:41:34 +0000
Subject: [PATCH 106/165] fix: pointer issue

---
 internal/controller/discovery/loaders.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 57588f2..9143ae4 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -13,7 +13,7 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		cfg.AcceptPush = *spec.Provider.HTTP.AcceptPush
+		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
 		return http.New(*cfg, *spec.Provider.HTTP), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)

From 36cf9fddf31b5fbdbe704992971d920fbb43025b Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 08:32:51 +0000
Subject: [PATCH 107/165] add helper to read secrets

---
 internal/controller/discovery/client.go | 43 +++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
index cb02161..9ccbb68 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/client.go
@@ -2,7 +2,9 @@ package discovery
 
 import (
 	"context"
+	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
@@ -30,3 +32,44 @@ func fetchExistingTargets(
 
 	return targetList.Items, nil
 }
+
+// Helper: GetSecretValues returns values from a secret
+// If keys are provided -> returns only those keys
+// If keys is empty -> returns entire secret data
+func GetSecretValues(
+	ctx context.Context,
+	c client.Client,
+	namespace string,
+	secretRef string,
+	keys ...string,
+) (map[string]string, error) {
+	var secret corev1.Secret
+	if err := c.Get(ctx,
+		client.ObjectKey{
+			Name:      secretRef,
+			Namespace: namespace,
+		}, &secret); err != nil {
+		return nil, fmt.Errorf("failed to get secret %s/%s: %w", namespace, secretRef, err)
+	}
+
+	result := make(map[string]string)
+
+	// Return full secret
+	if len(keys) == 0 {
+		for k, v := range secret.Data {
+			result[k] = string(v)
+		}
+		return result, nil
+	}
+
+	// Return specific keys
+	for _, key := range keys {
+		val, ok := secret.Data[key]
+		if !ok {
+			return nil, fmt.Errorf("key %s missing in secret %s/%s", key, namespace, secretRef)
+		}
+		result[key] = string(val)
+	}
+
+	return result, nil
+}

From 4f70c437aea340a8fd5d2c2a66bb839927e01489 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 08:33:04 +0000
Subject: [PATCH 108/165] implement basic and token authentication

---
 api/v1alpha1/targetsource_types.go            |  39 ++++---
 api/v1alpha1/zz_generated.deepcopy.go         |  42 -------
 .../operator.gnmic.dev_targetsources.yaml     |  95 +++-------------
 internal/controller/discovery/loaders.go      | 104 +++++++++++++++++-
 .../discovery/loaders/http/const.go           |   9 --
 .../discovery/loaders/http/loader.go          |  48 +++++---
 .../controller/targetsource_controller.go     |   5 +-
 7 files changed, 175 insertions(+), 167 deletions(-)
 delete mode 100644 internal/controller/discovery/loaders/http/const.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 7d0ca67..ee3838c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -63,7 +63,8 @@ type HTTPConfig struct {
 }
 
 type ClientTLSConfig struct {
-	InsecureSkipVerify bool                     `json:"insecureSkipVerify,omitempty"`
+	// +kubebuilder:default:=false
+	InsecureSkipVerify bool                      `json:"insecureSkipVerify,omitempty"`
 	CASecretRef        *corev1.SecretKeySelector `json:"caSecretRef,omitempty"`
 }
 
@@ -71,14 +72,18 @@ type ClientTLSConfig struct {
 type AuthorizationSpec struct {
 	Basic *BasicAuthSpec `json:"basic,omitempty"`
 	Token *TokenAuthSpec `json:"token,omitempty"`
-	JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
+	// JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
 }
 
 // Enforce EITHER inline creds OR secret ref
 // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
-	Username             string                    `json:"username,omitempty"`
-	Password             string                    `json:"password,omitempty"`
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	// CredentialsSecretRef references a secret containing:
+	// - username
+	// - password
+	// NOTE: key field is ignored; fixed keys are used instead
 	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
@@ -90,20 +95,20 @@ type TokenAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
 // +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
-type JWTAuthSpec struct {
-	// Static pre-generated JWT
-	Token          string                    `json:"token,omitempty"`
-	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
-	// Optional: generate JWT dynamically
-	Claims              map[string]string         `json:"claims,omitempty"`
-	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
-	// HS256, RS256, ES256, etc.
-	Algorithm string           `json:"algorithm,omitempty"`
-	TTL       *metav1.Duration `json:"ttl,omitempty"`
-}
+// type JWTAuthSpec struct {
+// 	// Static pre-generated JWT
+// 	Token          string                    `json:"token,omitempty"`
+// 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+// 	// Optional: generate JWT dynamically
+// 	Claims              map[string]string         `json:"claims,omitempty"`
+// 	Key                 string                    `json:"key,omitempty"`
+// 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
+// 	// HS256, RS256, ES256, etc.
+// 	Algorithm string           `json:"algorithm,omitempty"`
+// 	TTL       *metav1.Duration `json:"ttl,omitempty"`
+// }
 
 type PaginationSpec struct {
 	// Example: "results"
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index f115201..3844789 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -59,11 +59,6 @@ func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) {
 		*out = new(TokenAuthSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.JWT != nil {
-		in, out := &in.JWT, &out.JWT
-		*out = new(JWTAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationSpec.
@@ -499,43 +494,6 @@ func (in *InputStatus) DeepCopy() *InputStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *JWTAuthSpec) DeepCopyInto(out *JWTAuthSpec) {
-	*out = *in
-	if in.TokenSecretRef != nil {
-		in, out := &in.TokenSecretRef, &out.TokenSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.Claims != nil {
-		in, out := &in.Claims, &out.Claims
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-	if in.SigningKeySecretRef != nil {
-		in, out := &in.SigningKeySecretRef, &out.SigningKeySecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.TTL != nil {
-		in, out := &in.TTL, &out.TTL
-		*out = new(metav1.Duration)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthSpec.
-func (in *JWTAuthSpec) DeepCopy() *JWTAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(JWTAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Output) DeepCopyInto(out *Output) {
 	*out = *in
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 2e6872a..aab3917 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -52,8 +52,11 @@ spec:
                             description: Enforce EITHER inline creds OR secret ref
                             properties:
                               credentialsSecretRef:
-                                description: SecretKeySelector selects a key of a
-                                  Secret.
+                                description: |-
+                                  CredentialsSecretRef references a secret containing:
+                                  - username
+                                  - password
+                                  NOTE: key field is ignored; fixed keys are used instead
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -87,85 +90,6 @@ spec:
                               rule: (has(self.credentialsSecretRef) && !has(self.username)
                                 && !has(self.password)) || (!has(self.credentialsSecretRef)
                                 && has(self.username) && has(self.password))
-                          jwt:
-                            properties:
-                              algorithm:
-                                description: HS256, RS256, ES256, etc.
-                                type: string
-                              claims:
-                                additionalProperties:
-                                  type: string
-                                description: 'Optional: generate JWT dynamically'
-                                type: object
-                              signingKeySecretRef:
-                                description: SecretKeySelector selects a key of a
-                                  Secret.
-                                properties:
-                                  key:
-                                    description: The key of the secret to select from.  Must
-                                      be a valid secret key.
-                                    type: string
-                                  name:
-                                    default: ""
-                                    description: |-
-                                      Name of the referent.
-                                      This field is effectively required, but due to backwards compatibility is
-                                      allowed to be empty. Instances of this type with an empty value here are
-                                      almost certainly wrong.
-                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                                    type: string
-                                  optional:
-                                    description: Specify whether the Secret or its
-                                      key must be defined
-                                    type: boolean
-                                required:
-                                - key
-                                type: object
-                                x-kubernetes-map-type: atomic
-                              token:
-                                description: Static pre-generated JWT
-                                type: string
-                              tokenSecretRef:
-                                description: SecretKeySelector selects a key of a
-                                  Secret.
-                                properties:
-                                  key:
-                                    description: The key of the secret to select from.  Must
-                                      be a valid secret key.
-                                    type: string
-                                  name:
-                                    default: ""
-                                    description: |-
-                                      Name of the referent.
-                                      This field is effectively required, but due to backwards compatibility is
-                                      allowed to be empty. Instances of this type with an empty value here are
-                                      almost certainly wrong.
-                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                                    type: string
-                                  optional:
-                                    description: Specify whether the Secret or its
-                                      key must be defined
-                                    type: boolean
-                                required:
-                                - key
-                                type: object
-                                x-kubernetes-map-type: atomic
-                              ttl:
-                                type: string
-                            type: object
-                            x-kubernetes-validations:
-                            - message: static JWT token and generated JWT configuration
-                                cannot be combined
-                              rule: '!((has(self.token) || has(self.tokenSecretRef))
-                                && (has(self.signingKeySecretRef) || has(self.claims)))'
-                            - message: static JWT token and generated JWT configuration
-                                cannot be combined
-                              rule: '!((has(self.token) || has(self.tokenSecretRef))
-                                && (has(self.signingKeySecretRef) || has(self.claims)))'
-                            - message: algorithm must be specified when generating
-                                a JWT
-                              rule: '!has(self.signingKeySecretRef) || self.algorithm
-                                != ""'
                           token:
                             properties:
                               scheme:
@@ -240,6 +164,14 @@ spec:
                             description: 'Example: "next"'
                             type: string
                         type: object
+                        x-kubernetes-validations:
+                        - message: static JWT token and generated JWT configuration
+                            cannot be combined
+                          rule: '!((has(self.token) || has(self.tokenSecretRef)) &&
+                            ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))'
+                        - message: algorithm must be specified when generating a JWT
+                          rule: '!has(self.signingKeySecretRef) || self.algorithm
+                            != ""'
                       timeout:
                         default: 10s
                         type: string
@@ -270,6 +202,7 @@ spec:
                             type: object
                             x-kubernetes-map-type: atomic
                           insecureSkipVerify:
+                            default: false
                             type: boolean
                         type: object
                       url:
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 9143ae4..65888b8 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -1,22 +1,120 @@
 package discovery
 
 import (
+	"context"
 	"fmt"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	http "github.com/gnmic/operator/internal/controller/discovery/loaders/http"
 )
 
 // NewLoader creates a loader by name
-func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
+func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
-		return http.New(*cfg, *spec.Provider.HTTP), nil
+		httpSpec := *spec.Provider.HTTP
+		cfg.AcceptPush = httpSpec.AcceptPush
+
+		// TODO: watch secrets -> if secret changes reconcile has to be executed
+		if httpSpec.Authorization != nil {
+			if err := resolveAuthorizationIntoSpec(
+				ctx,
+				c,
+				cfg.TargetsourceNN.Namespace,
+				httpSpec.Authorization,
+			); err != nil {
+				return nil, err
+			}
+		}
+
+		return http.New(*cfg, httpSpec), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 
 }
+
+func resolveAuthorizationIntoSpec(
+	ctx context.Context,
+	c client.Client,
+	namespace string,
+	authSpec *gnmicv1alpha1.AuthorizationSpec,
+) error {
+	if authSpec == nil {
+		return nil
+	}
+	auth := authSpec
+
+	switch {
+	case auth.Basic != nil:
+		b := auth.Basic
+
+		if b.CredentialsSecretRef != nil {
+			values, err := GetSecretValues(
+				ctx,
+				c,
+				namespace,
+				b.CredentialsSecretRef.Name,
+				"username",
+				"password",
+			)
+			if err != nil {
+				return err
+			}
+			b.Username = values["username"]
+			b.Password = values["password"]
+		}
+
+	case auth.Token != nil:
+		t := auth.Token
+		if t.TokenSecretRef != nil {
+			values, err := GetSecretValues(
+				ctx,
+				c,
+				namespace,
+				t.TokenSecretRef.Name,
+				"token",
+			)
+			if err != nil {
+				return err
+			}
+			t.Token = values["token"]
+		}
+
+		// case auth.JWT != nil:
+		// 	jwt := auth.JWT
+		// 	if jwt.TokenSecretRef != nil {
+		// 		values, err := GetSecretValues(
+		// 			ctx,
+		// 			c,
+		// 			namespaceName,
+		// 			jwt.TokenSecretRef.Name,
+		// 			"token",
+		// 		)
+		// 		if err != nil {
+		// 			return err
+		// 		}
+		// 		jwt.Token = values[jwt.TokenSecretRef.Key]
+		// 	}
+		// 	if jwt.SigningKeySecretRef != nil {
+		// 		values, err := GetSecretValues(
+		// 			ctx,
+		// 			c,
+		// 			namespaceName,
+		// 			jwt.SigningKeySecretRef.Name,
+		// 			"key",
+		// 		)
+		// 		if err != nil {
+		// 			return err
+		// 		}
+		// 		jwt.Key = values[jwt.SigningKeySecretRef.Key]
+
+		// 	}
+	}
+
+	return nil
+}
diff --git a/internal/controller/discovery/loaders/http/const.go b/internal/controller/discovery/loaders/http/const.go
deleted file mode 100644
index 4b18f6c..0000000
--- a/internal/controller/discovery/loaders/http/const.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package http
-
-import "time"
-
-const (
-	DefaultPollInterval = 1 * time.Hour
-	DefaultTimeout      = 30 * time.Second
-	DefaultAcceptPush   = false
-)
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index bb8a025..356a911 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -70,13 +70,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 
 	// helper function to fetch targets and emit discovery messages
 	fetchAndEmit := func() {
-		targets, err := l.fetchTargetsFromHTTPEndpoint(
-			ctx,
-			client,
-			l.spec.URL,
-			l.spec.Authorization.Token.Scheme,
-			l.spec.Authorization.Token.Token,
-		)
+		targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client)
 		if err != nil {
 			logger.Error(
 				err,
@@ -123,18 +117,14 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,
-	url string,
-	scheme string,
-	token string,
 ) ([]core.DiscoveredTarget, error) {
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.spec.URL, nil)
 	if err != nil {
 		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
 	}
 
 	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Authorization", fmt.Sprintf("%s %s", scheme, token))
+	l.applyAuthorization(req)
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -153,3 +143,35 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 
 	return targets, nil
 }
+
+func (l *Loader) applyAuthorization(req *http.Request) {
+	auth := l.spec.Authorization
+	if auth == nil {
+		return
+	}
+
+	switch {
+	case auth.Basic != nil:
+		req.SetBasicAuth(
+			auth.Basic.Username,
+			auth.Basic.Password,
+		)
+
+	case auth.Token != nil:
+		req.Header.Set(
+			"Authorization",
+			fmt.Sprintf("%s %s",
+				auth.Token.Scheme,
+				auth.Token.Token,
+			),
+		)
+
+		// case auth.JWT != nil:
+		// 	if auth.JWT.Token != "" {
+		// 		req.Header.Set(
+		// 			"Authorization",
+		// 			fmt.Sprintf("Bearer %s", auth.JWT.Token),
+		// 		)
+		// 	}
+	}
+}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 522aabd..7cf135e 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -94,7 +94,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, nil
 	}
 
-	if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil {
+	if err := r.startDiscovery(ctx, req.NamespacedName, targetSource, logger); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -162,6 +162,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour
 // - MessageProcessor and Loader must run for the lifetime of the TargetSource
 // - Any unexpected exit is treated as a bug and triggers full shutdown
 func (r *TargetSourceReconciler) startDiscovery(
+	reconcileCtx context.Context,
 	key types.NamespacedName,
 	targetSource *gnmicv1alpha1.TargetSource,
 	logger logr.Logger,
@@ -185,7 +186,7 @@ func (r *TargetSourceReconciler) startDiscovery(
 		targetSource,
 		targetChannel,
 	)
-	loader, err := discovery.NewLoader(&loaderConfig, targetSource.Spec)
+	loader, err := discovery.NewLoader(reconcileCtx, r.Client, &loaderConfig, targetSource.Spec)
 	if err != nil {
 		logger.Error(err, "Target loader could not be created")
 		cleanup()

From 966cd59f6e9d73c65a1c0b7b7f6341f6aa4db370 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 08:43:18 +0000
Subject: [PATCH 109/165] support .Key for TokenSecretRef

---
 internal/controller/discovery/loaders.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 65888b8..98e63d2 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -72,17 +72,21 @@ func resolveAuthorizationIntoSpec(
 	case auth.Token != nil:
 		t := auth.Token
 		if t.TokenSecretRef != nil {
+			key := "token"
+			if t.TokenSecretRef.Key != "" {
+				key = t.TokenSecretRef.Key
+			}
 			values, err := GetSecretValues(
 				ctx,
 				c,
 				namespace,
 				t.TokenSecretRef.Name,
-				"token",
+				key,
 			)
 			if err != nil {
 				return err
 			}
-			t.Token = values["token"]
+			t.Token = values[key]
 		}
 
 		// case auth.JWT != nil:

From 862e28d6f7d0139ddd4ff163d11c3d4e0b828dc0 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 09:42:23 +0000
Subject: [PATCH 110/165] update targetsource

---
 api/v1alpha1/targetsource_types.go            | 113 +++++++++++++++---
 api/v1alpha1/zz_generated.deepcopy.go         |   9 +-
 .../operator.gnmic.dev_targetsources.yaml     |  94 +++++++++++++--
 3 files changed, 185 insertions(+), 31 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index ee3838c..143da3c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -24,74 +24,135 @@ import (
 // TargetSourceSpec defines the desired state of TargetSource
 // +kubebuilder:validation:Required
 type TargetSourceSpec struct {
+	// Provider defines the source of targets for this TargetSource
+	// Only one provider can be specified per TargetSource
+	// +kubebuilder:validation:Required
 	Provider *ProviderSpec `json:"provider"`
 
+	// TODO: implement in message processor
+	// Optional port to use for discovered targets if not specified by the provider
+	// +kubebuilder:validation:Optional
+	TargetPort int32 `json:"targetPort,omitempty"`
+
+	// Optional labels to apply to all targets discovered by this TargetSource
 	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
+	// The TargetProfile to use for targets discovered by this TargetSource
+	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinLength=1
 	TargetProfile string `json:"targetProfile"`
 }
 
+// ProviderSpec defines the source of targets for a TargetSource
+// Only one provider can be specified per TargetSource
 // +kubebuilder:validation:ExactlyOneOf=http
 type ProviderSpec struct {
+	// HTTP defines the configuration for a HTTP provider
 	HTTP *HTTPConfig `json:"http,omitempty"`
 }
 
+// HTTPConfig defines the configuration for the HTTP provider
 // +kubebuilder:validation:AtLeastOneOf=url;acceptPush
 type HTTPConfig struct {
+	// URL of the HTTP endpoint to pull targets from
+	// If defined, the loader will periodically poll this endpoint for targets
 	// +kubebuilder:validation:Optional
 	URL string `json:"url,omitempty"`
+
+	// If true, the loader will accept pushed target updates to the controller endpoint
+	// The endpoint will be /{namespace}/{targetsource}/<todo>
+	// +kubebuilder:default=false
+	// +kubebuilder:validation:Optional
+	AcceptPush bool `json:"acceptPush,omitempty"`
+
+	// Optional authorization configuration for accessing the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
+
+	// Optional interval for polling the HTTP endpoint for targets
 	// TODO: increase default value
 	// +kubebuilder:default="30s"
 	// +kubebuilder:validation:Optional
 	PollInterval *metav1.Duration `json:"interval,omitempty"`
+
+	// Optional timeout for HTTP requests to the endpoint
 	// +kubebuilder:default="10s"
 	// +kubebuilder:validation:Optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
+
+	// Optional TLS configuration for connecting to the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	TLS *ClientTLSConfig `json:"tls,omitempty"`
+
+	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
+
+	// Optional mapping configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
-	// +kubebuilder:default=false
-	// +kubebuilder:validation:Optional
-	AcceptPush bool `json:"acceptPush,omitempty"`
 }
 
+// +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive"
 type ClientTLSConfig struct {
+	// Skip TLS verification of the Provider's certificate.
 	// +kubebuilder:default:=false
-	InsecureSkipVerify bool                      `json:"insecureSkipVerify,omitempty"`
-	CASecretRef        *corev1.SecretKeySelector `json:"caSecretRef,omitempty"`
+	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
+
+	// Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+	// chain presented by the Provider. Only used if using HTTPS to connect to Provider and
+	// ignored for HTTP connections.
+	// Mutually exclusive with CABundleSecretRef.
+	// +optional
+	CABundle []byte `json:"caBundle,omitempty"`
+
+	// Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+	// verifying the certificate chain presented by the Provider when using HTTPS.
+	// Mutually exclusive with CABundle.
+	CABundleSecretRef *corev1.SecretKeySelector `json:"caBundleSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=basic;jwt;token
+// AuthorizationSpec defines the configuration for authentication
+// +kubebuilder:validation:ExactlyOneOf=basic;token
 type AuthorizationSpec struct {
+	// Basic authentication configuration
 	Basic *BasicAuthSpec `json:"basic,omitempty"`
+	// Token-based authentication configuration
 	Token *TokenAuthSpec `json:"token,omitempty"`
 	// JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
+	// MTLS
 }
 
+// BasicAuthSpec defines the configuration for basic authentication
 // Enforce EITHER inline creds OR secret ref
 // +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
+	// Username for basic auth
+	// Mutually exclusive with CredentialsSecretRef.
 	Username string `json:"username,omitempty"`
+	// Password for basic auth
+	// Mutually exclusive with CredentialsSecretRef.
 	Password string `json:"password,omitempty"`
-	// CredentialsSecretRef references a secret containing:
-	// - username
-	// - password
-	// NOTE: key field is ignored; fixed keys are used instead
+
+	// Reference to a Secret containing "username" and "password" keys to use for
+	// basic authentication when connecting to the Provider.
+	// Mutually exclusive with Username and Password.
 	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
+// TokenAuthSpec defines the configuration for token-based authentication
 // +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
 type TokenAuthSpec struct {
+	// Scheme for the token, e.g. "Bearer"
 	// +kubebuilder:validation:MinLength=1
-	Scheme         string                    `json:"scheme"`
-	Token          string                    `json:"token,omitempty"`
+	Scheme string `json:"scheme"`
+	// Token value for authentication
+	// Mutually exclusive with TokenSecretRef.
+	Token string `json:"token,omitempty"`
+	// Reference to a Secret containing a key with the token value to use for
+	// authentication when connecting to the Provider.
+	// Mutually exclusive with Token.
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
@@ -110,19 +171,37 @@ type TokenAuthSpec struct {
 // 	TTL       *metav1.Duration `json:"ttl,omitempty"`
 // }
 
+// PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
+	// JSONPath-style expression to extract the list of targets from the response
 	// Example: "results"
 	ItemsField string `json:"itemsField,omitempty"`
+
+	// JSONPath-style expression to extract the next page token or URL from the response for pagination
 	// Example: "next"
 	NextField string `json:"nextField,omitempty"`
 }
 
-// JSONPath-style expressions
+// JSONPath-style expressions to extract target fields from the response
+// and map them to the corresponding Target fields.
 type ResponseMappingSpec struct {
-	Name    string            `json:"name"`
-	Address string            `json:"address"`
-	Port    string            `json:"port,omitempty"`
-	Labels  map[string]string `json:"labels,omitempty"`
+	// JSONPath expression to extract the target name from the response
+	// +kubebuilder:validation:Required
+	Name string `json:"name"`
+
+	// JSONPath expression to extract the target address from the response
+	// +kubebuilder:validation:Required
+	Address string `json:"address"`
+
+	// JSONPath expression to extract the target port from the response
+	// +kubebuilder:validation:Optional
+	Port string `json:"port,omitempty"`
+
+	// JSONPath expression to extract the target labels from the response
+	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+	// with values from the response taking precedence in case of conflicts.
+	// +kubebuilder:validation:Optional
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 3844789..dc4b784 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -94,8 +94,13 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
-	if in.CASecretRef != nil {
-		in, out := &in.CASecretRef, &out.CASecretRef
+	if in.CABundle != nil {
+		in, out := &in.CABundle, &out.CABundle
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
+	if in.CABundleSecretRef != nil {
+		in, out := &in.CABundleSecretRef, &out.CABundleSecretRef
 		*out = new(v1.SecretKeySelector)
 		(*in).DeepCopyInto(*out)
 	}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index aab3917..a6f1e1d 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -40,23 +40,31 @@ spec:
             description: TargetSourceSpec defines the desired state of TargetSource
             properties:
               provider:
+                description: |-
+                  Provider defines the source of targets for this TargetSource
+                  Only one provider can be specified per TargetSource
                 properties:
                   http:
+                    description: HTTP defines the configuration for a HTTP provider
                     properties:
                       acceptPush:
                         default: false
+                        description: |-
+                          If true, the loader will accept pushed target updates to the controller endpoint
+                          The endpoint will be /{namespace}/{targetsource}/<todo>
                         type: boolean
                       authorization:
+                        description: Optional authorization configuration for accessing
+                          the HTTP endpoint
                         properties:
                           basic:
-                            description: Enforce EITHER inline creds OR secret ref
+                            description: Basic authentication configuration
                             properties:
                               credentialsSecretRef:
                                 description: |-
-                                  CredentialsSecretRef references a secret containing:
-                                  - username
-                                  - password
-                                  NOTE: key field is ignored; fixed keys are used instead
+                                  Reference to a Secret containing "username" and "password" keys to use for
+                                  basic authentication when connecting to the Provider.
+                                  Mutually exclusive with Username and Password.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -80,8 +88,14 @@ spec:
                                 type: object
                                 x-kubernetes-map-type: atomic
                               password:
+                                description: |-
+                                  Password for basic auth
+                                  Mutually exclusive with CredentialsSecretRef.
                                 type: string
                               username:
+                                description: |-
+                                  Username for basic auth
+                                  Mutually exclusive with CredentialsSecretRef.
                                 type: string
                             type: object
                             x-kubernetes-validations:
@@ -91,15 +105,22 @@ spec:
                                 && !has(self.password)) || (!has(self.credentialsSecretRef)
                                 && has(self.username) && has(self.password))
                           token:
+                            description: Token-based authentication configuration
                             properties:
                               scheme:
+                                description: Scheme for the token, e.g. "Bearer"
                                 minLength: 1
                                 type: string
                               token:
+                                description: |-
+                                  Token value for authentication
+                                  Mutually exclusive with TokenSecretRef.
                                 type: string
                               tokenSecretRef:
-                                description: SecretKeySelector selects a key of a
-                                  Secret.
+                                description: |-
+                                  Reference to a Secret containing a key with the token value to use for
+                                  authentication when connecting to the Provider.
+                                  Mutually exclusive with Token.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -137,31 +158,50 @@ spec:
                             == 1'
                       interval:
                         default: 30s
+                        description: Optional interval for polling the HTTP endpoint
+                          for targets
                         type: string
                       mapping:
-                        description: JSONPath-style expressions
+                        description: Optional mapping configuration for parsing responses
+                          from the HTTP endpoint
                         properties:
                           address:
+                            description: JSONPath expression to extract the target
+                              address from the response
                             type: string
                           labels:
                             additionalProperties:
                               type: string
+                            description: |-
+                              JSONPath expression to extract the target labels from the response
+                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+                              with values from the response taking precedence in case of conflicts.
                             type: object
                           name:
+                            description: JSONPath expression to extract the target
+                              name from the response
                             type: string
                           port:
+                            description: JSONPath expression to extract the target
+                              port from the response
                             type: string
                         required:
                         - address
                         - name
                         type: object
                       pagination:
+                        description: Optional pagination configuration for parsing
+                          responses from the HTTP endpoint
                         properties:
                           itemsField:
-                            description: 'Example: "results"'
+                            description: |-
+                              JSONPath-style expression to extract the list of targets from the response
+                              Example: "results"
                             type: string
                           nextField:
-                            description: 'Example: "next"'
+                            description: |-
+                              JSONPath-style expression to extract the next page token or URL from the response for pagination
+                              Example: "next"
                             type: string
                         type: object
                         x-kubernetes-validations:
@@ -174,11 +214,25 @@ spec:
                             != ""'
                       timeout:
                         default: 10s
+                        description: Optional timeout for HTTP requests to the endpoint
                         type: string
                       tls:
+                        description: Optional TLS configuration for connecting to
+                          the HTTP endpoint
                         properties:
-                          caSecretRef:
-                            description: SecretKeySelector selects a key of a Secret.
+                          caBundle:
+                            description: |-
+                              Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+                              chain presented by the Provider. Only used if using HTTPS to connect to Provider and
+                              ignored for HTTP connections.
+                              Mutually exclusive with CABundleSecretRef.
+                            format: byte
+                            type: string
+                          caBundleSecretRef:
+                            description: |-
+                              Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+                              verifying the certificate chain presented by the Provider when using HTTPS.
+                              Mutually exclusive with CABundle.
                             properties:
                               key:
                                 description: The key of the secret to select from.  Must
@@ -203,9 +257,16 @@ spec:
                             x-kubernetes-map-type: atomic
                           insecureSkipVerify:
                             default: false
+                            description: Skip TLS verification of the Provider's certificate.
                             type: boolean
                         type: object
+                        x-kubernetes-validations:
+                        - message: caBundle and caBundleSecretRef are mutually exclusive
+                          rule: '!(has(self.caBundle) && has(self.caBundleSecretRef))'
                       url:
+                        description: |-
+                          URL of the HTTP endpoint to pull targets from
+                          If defined, the loader will periodically poll this endpoint for targets
                         type: string
                     type: object
                     x-kubernetes-validations:
@@ -220,8 +281,17 @@ spec:
               targetLabels:
                 additionalProperties:
                   type: string
+                description: Optional labels to apply to all targets discovered by
+                  this TargetSource
                 type: object
+              targetPort:
+                description: Optional port to use for discovered targets if not specified
+                  by the provider
+                format: int32
+                type: integer
               targetProfile:
+                description: The TargetProfile to use for targets discovered by this
+                  TargetSource
                 minLength: 1
                 type: string
             required:

From e6e9439be84116136e6c2c9e0230056a92071866 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 10:04:38 +0000
Subject: [PATCH 111/165] support TLS verification

---
 internal/controller/discovery/loaders.go      | 44 ++++++++++++++++++-
 .../discovery/loaders/http/loader.go          | 28 +++++++++++-
 2 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 98e63d2..1e5ea46 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -30,12 +30,21 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi
 				return nil, err
 			}
 		}
+		if httpSpec.TLS != nil {
+			if err := resolveTLSIntoSpec(
+				ctx,
+				c,
+				cfg.TargetsourceNN.Namespace,
+				httpSpec.TLS,
+			); err != nil {
+				return nil, err
+			}
+		}
 
 		return http.New(*cfg, httpSpec), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
-
 }
 
 func resolveAuthorizationIntoSpec(
@@ -122,3 +131,36 @@ func resolveAuthorizationIntoSpec(
 
 	return nil
 }
+
+func resolveTLSIntoSpec(
+	ctx context.Context,
+	c client.Client,
+	namespace string,
+	tlsSpec *gnmicv1alpha1.ClientTLSConfig,
+) error {
+	if tlsSpec == nil {
+		return nil
+	}
+	tls := tlsSpec
+
+	if tls.CABundleSecretRef != nil {
+		key := "ca.crt"
+		if tls.CABundleSecretRef.Key != "" {
+			key = tls.CABundleSecretRef.Key
+		}
+		values, err := GetSecretValues(
+			ctx,
+			c,
+			namespace,
+			tls.CABundleSecretRef.Name,
+			key,
+		)
+		if err != nil {
+			return err
+		}
+		// convert string to []byte
+		tls.CABundle = []byte(values[key])
+	}
+
+	return nil
+}
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 356a911..d956799 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -2,6 +2,8 @@ package http
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -55,8 +57,9 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	// 	return errors.New("HTTP loader requires spec.provider.http to be set")
 	// }
 
-	client := &http.Client{
-		Timeout: l.spec.Timeout.Duration,
+	client, err := l.buildHTTPClient()
+	if err != nil {
+		return fmt.Errorf("failed to build HTTP client: %w", err)
 	}
 	interval := l.spec.PollInterval.Duration
 	ticker := time.NewTicker(interval)
@@ -114,6 +117,27 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	}
 }
 
+func (l *Loader) buildHTTPClient() (*http.Client, error) {
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: l.spec.TLS != nil && l.spec.TLS.InsecureSkipVerify,
+	}
+
+	if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 {
+		certPool := x509.NewCertPool()
+		if ok := certPool.AppendCertsFromPEM(l.spec.TLS.CABundle); !ok {
+			return nil, fmt.Errorf("Failed to parse CA bundle for TargetSource %s/%s\n", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
+		}
+		tlsConfig.RootCAs = certPool
+	}
+
+	return &http.Client{
+		Timeout: l.spec.Timeout.Duration,
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}, nil
+}
+
 func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,

From 055bfb05e96b204456f65d4f47180342bbd03270 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 14:11:14 +0000
Subject: [PATCH 112/165] make manifest and generate

---
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index a6f1e1d..d603546 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -152,9 +152,9 @@ spec:
                               rule: has(self.token) != has(self.tokenSecretRef)
                         type: object
                         x-kubernetes-validations:
-                        - message: exactly one of the fields in [basic jwt token]
-                            must be set
-                          rule: '[has(self.basic),has(self.jwt),has(self.token)].filter(x,x==true).size()
+                        - message: exactly one of the fields in [basic token] must
+                            be set
+                          rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
                         default: 30s

From 1deb8ccbaac61ef238579a11a4bfcec89fd1ba3e Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 14:21:08 +0000
Subject: [PATCH 113/165] fix CRD issues

---
 api/v1alpha1/targetsource_types.go                     | 4 ++--
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 8 --------
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 143da3c..aa5b8ce 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -156,8 +156,8 @@ type TokenAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
+// +kubebuilder(disabled):validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder(disabled):validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
 // type JWTAuthSpec struct {
 // 	// Static pre-generated JWT
 // 	Token          string                    `json:"token,omitempty"`
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d603546..6851ad7 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -204,14 +204,6 @@ spec:
                               Example: "next"
                             type: string
                         type: object
-                        x-kubernetes-validations:
-                        - message: static JWT token and generated JWT configuration
-                            cannot be combined
-                          rule: '!((has(self.token) || has(self.tokenSecretRef)) &&
-                            ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))'
-                        - message: algorithm must be specified when generating a JWT
-                          rule: '!has(self.signingKeySecretRef) || self.algorithm
-                            != ""'
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint

From 816b04f50aa77b5cab0b945ea08c4f9337840dca Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 08:34:40 +0000
Subject: [PATCH 114/165] add support for pagination

---
 api/v1alpha1/targetsource_types.go            |  12 +-
 .../operator.gnmic.dev_targetsources.yaml     |  12 +-
 .../discovery/loaders/http/loader.go          | 134 +++++++++++++++---
 3 files changed, 134 insertions(+), 24 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index aa5b8ce..c4d3382 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -173,12 +173,18 @@ type TokenAuthSpec struct {
 
 // PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
-	// JSONPath-style expression to extract the list of targets from the response
+	// Field name in the JSON response that contains the list of items (targets).
+	// Must refer to a top-level key in the response object.
 	// Example: "results"
 	ItemsField string `json:"itemsField,omitempty"`
 
-	// JSONPath-style expression to extract the next page token or URL from the response for pagination
-	// Example: "next"
+	// Field name in the JSON response that contains the next page reference.
+	// The value can be either:
+	// - a full URL (used directly for the next request), or
+	// - a pagination token (appended as a query parameter using this field name as the key).
+	//
+	// Must refer to a top-level key in the response object.
+	// Example: "next" or "nextToken"
 	NextField string `json:"nextField,omitempty"`
 }
 
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 6851ad7..3e7f89f 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -195,13 +195,19 @@ spec:
                         properties:
                           itemsField:
                             description: |-
-                              JSONPath-style expression to extract the list of targets from the response
+                              Field name in the JSON response that contains the list of items (targets).
+                              Must refer to a top-level key in the response object.
                               Example: "results"
                             type: string
                           nextField:
                             description: |-
-                              JSONPath-style expression to extract the next page token or URL from the response for pagination
-                              Example: "next"
+                              Field name in the JSON response that contains the next page reference.
+                              The value can be either:
+                              - a full URL (used directly for the next request), or
+                              - a pagination token (appended as a query parameter using this field name as the key).
+
+                              Must refer to a top-level key in the response object.
+                              Example: "next" or "nextToken"
                             type: string
                         type: object
                       timeout:
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index d956799..c064a7b 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/google/uuid"
@@ -142,30 +143,56 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,
 ) ([]core.DiscoveredTarget, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, l.spec.URL, nil)
-	if err != nil {
-		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
-	}
+	var allTargets []core.DiscoveredTarget
+	currentUrl := l.spec.URL
 
-	req.Header.Set("Accept", "application/json")
-	l.applyAuthorization(req)
+	for {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentUrl, nil)
+		if err != nil {
+			return nil, fmt.Errorf("creating HTTP request failed: %w", err)
+		}
 
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("HTTP request failed: %w", err)
-	}
-	defer resp.Body.Close()
+		req.Header.Set("Accept", "application/json")
+		l.applyAuthorization(req)
 
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
-	}
+		resp, err := client.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("HTTP request failed: %w", err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
+		}
 
-	var targets []core.DiscoveredTarget
-	if err := json.NewDecoder(resp.Body).Decode(&targets); err != nil {
-		return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
+		// Decode response into raw map for pagination support
+		var raw map[string]interface{}
+		if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
+			return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
+		}
+
+		// Extract targets from response
+		targets, err := l.extractTargetsFromResponse(raw)
+		if err != nil {
+			return nil, err
+		}
+		allTargets = append(allTargets, targets...)
+
+		// Check for pagination
+		nextPageInfo, err := l.extractNextPageInfo(raw)
+		if err != nil {
+			return nil, err
+		}
+		if nextPageInfo == "" {
+			break
+		}
+		nextURL, err := l.buildNextURL(currentUrl, nextPageInfo)
+		if err != nil {
+			return nil, err
+		}
+		currentUrl = nextURL
 	}
 
-	return targets, nil
+	return allTargets, nil
 }
 
 func (l *Loader) applyAuthorization(req *http.Request) {
@@ -199,3 +226,74 @@ func (l *Loader) applyAuthorization(req *http.Request) {
 		// 	}
 	}
 }
+
+func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) {
+	var targets []core.DiscoveredTarget
+
+	if l.spec.Pagination == nil || l.spec.Pagination.ItemsField == "" {
+		// No pagination config, assume entire response is the target list
+		data, err := json.Marshal(raw)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshal response: %w", err)
+		}
+
+		if err := json.Unmarshal(data, &targets); err != nil {
+			return nil, fmt.Errorf("failed to decode targets: %w", err)
+		}
+
+		return targets, nil
+	}
+
+	// Extract from field
+	items, ok := raw[l.spec.Pagination.ItemsField]
+	if !ok {
+		return nil, fmt.Errorf("itemsField '%s' not found in response", l.spec.Pagination.ItemsField)
+	}
+
+	data, err := json.Marshal(items)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := json.Unmarshal(data, &targets); err != nil {
+		return nil, fmt.Errorf("failed to decode targets from itemsField: %w", err)
+	}
+
+	return targets, nil
+}
+
+func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) {
+	if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" {
+		return "", nil
+	}
+
+	val, ok := raw[l.spec.Pagination.NextField]
+	if !ok {
+		return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField)
+	}
+
+	next, ok := val.(string)
+	if !ok {
+		return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField)
+	}
+
+	return next, nil
+}
+
+func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) {
+	// nextVal is a full URL -> return as is
+	if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" {
+		return nextVal, nil
+	}
+
+	// nextVal is a token -> append as query parameter
+	parsedURL, err := url.Parse(currentURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err)
+	}
+	q := parsedURL.Query()
+	q.Set(l.spec.Pagination.NextField, nextVal)
+	parsedURL.RawQuery = q.Encode()
+
+	return parsedURL.String(), nil
+}

From c01b199d8bb9605956b4ec17ec4f2e900dbd19f5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 08:37:33 +0000
Subject: [PATCH 115/165] refactor

---
 .../controller/discovery/loaders/http/auth.go | 38 ++++++++++
 .../discovery/loaders/http/loader.go          | 69 -------------------
 .../discovery/loaders/http/pagination.go      | 42 +++++++++++
 3 files changed, 80 insertions(+), 69 deletions(-)
 create mode 100644 internal/controller/discovery/loaders/http/auth.go
 create mode 100644 internal/controller/discovery/loaders/http/pagination.go

diff --git a/internal/controller/discovery/loaders/http/auth.go b/internal/controller/discovery/loaders/http/auth.go
new file mode 100644
index 0000000..0af0556
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/auth.go
@@ -0,0 +1,38 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+)
+
+func (l *Loader) applyAuthorization(req *http.Request) {
+	auth := l.spec.Authorization
+	if auth == nil {
+		return
+	}
+
+	switch {
+	case auth.Basic != nil:
+		req.SetBasicAuth(
+			auth.Basic.Username,
+			auth.Basic.Password,
+		)
+
+	case auth.Token != nil:
+		req.Header.Set(
+			"Authorization",
+			fmt.Sprintf("%s %s",
+				auth.Token.Scheme,
+				auth.Token.Token,
+			),
+		)
+
+		// case auth.JWT != nil:
+		// 	if auth.JWT.Token != "" {
+		// 		req.Header.Set(
+		// 			"Authorization",
+		// 			fmt.Sprintf("Bearer %s", auth.JWT.Token),
+		// 		)
+		// 	}
+	}
+}
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index c064a7b..55689df 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/google/uuid"
@@ -195,38 +194,6 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	return allTargets, nil
 }
 
-func (l *Loader) applyAuthorization(req *http.Request) {
-	auth := l.spec.Authorization
-	if auth == nil {
-		return
-	}
-
-	switch {
-	case auth.Basic != nil:
-		req.SetBasicAuth(
-			auth.Basic.Username,
-			auth.Basic.Password,
-		)
-
-	case auth.Token != nil:
-		req.Header.Set(
-			"Authorization",
-			fmt.Sprintf("%s %s",
-				auth.Token.Scheme,
-				auth.Token.Token,
-			),
-		)
-
-		// case auth.JWT != nil:
-		// 	if auth.JWT.Token != "" {
-		// 		req.Header.Set(
-		// 			"Authorization",
-		// 			fmt.Sprintf("Bearer %s", auth.JWT.Token),
-		// 		)
-		// 	}
-	}
-}
-
 func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) {
 	var targets []core.DiscoveredTarget
 
@@ -261,39 +228,3 @@ func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.
 
 	return targets, nil
 }
-
-func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) {
-	if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" {
-		return "", nil
-	}
-
-	val, ok := raw[l.spec.Pagination.NextField]
-	if !ok {
-		return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField)
-	}
-
-	next, ok := val.(string)
-	if !ok {
-		return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField)
-	}
-
-	return next, nil
-}
-
-func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) {
-	// nextVal is a full URL -> return as is
-	if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" {
-		return nextVal, nil
-	}
-
-	// nextVal is a token -> append as query parameter
-	parsedURL, err := url.Parse(currentURL)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err)
-	}
-	q := parsedURL.Query()
-	q.Set(l.spec.Pagination.NextField, nextVal)
-	parsedURL.RawQuery = q.Encode()
-
-	return parsedURL.String(), nil
-}
diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go
new file mode 100644
index 0000000..6a4a3ec
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/pagination.go
@@ -0,0 +1,42 @@
+package http
+
+import (
+	"fmt"
+	"net/url"
+)
+
+func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) {
+	if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" {
+		return "", nil
+	}
+
+	val, ok := raw[l.spec.Pagination.NextField]
+	if !ok {
+		return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField)
+	}
+
+	next, ok := val.(string)
+	if !ok {
+		return "", fmt.Errorf("nextField '%s' is not a string in response", l.spec.Pagination.NextField)
+	}
+
+	return next, nil
+}
+
+func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) {
+	// nextVal is a full URL -> return as is
+	if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" {
+		return nextVal, nil
+	}
+
+	// nextVal is a token -> append as query parameter
+	parsedURL, err := url.Parse(currentURL)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse current URL in order to build next URL: %w", err)
+	}
+	q := parsedURL.Query()
+	q.Set(l.spec.Pagination.NextField, nextVal)
+	parsedURL.RawQuery = q.Encode()
+
+	return parsedURL.String(), nil
+}

From 5d95c9028fb63d0a600f37a4e5ecc3f573092b3d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 12:22:36 +0000
Subject: [PATCH 116/165] add support for JSONPath mapping

---
 api/v1alpha1/targetsource_types.go            |  4 +-
 .../operator.gnmic.dev_targetsources.yaml     |  6 +-
 go.mod                                        |  2 +
 go.sum                                        |  5 ++
 internal/controller/discovery/core/types.go   |  7 +-
 .../discovery/loaders/http/loader.go          | 50 ++++++-----
 .../discovery/loaders/http/mapper.go          | 78 +++++++++++++++++
 .../discovery/loaders/http/mapper_direct.go   | 70 +++++++++++++++
 .../discovery/loaders/http/mapper_jsonpath.go | 87 +++++++++++++++++++
 .../controller/discovery/message_processor.go |  3 +-
 10 files changed, 283 insertions(+), 29 deletions(-)
 create mode 100644 internal/controller/discovery/loaders/http/mapper.go
 create mode 100644 internal/controller/discovery/loaders/http/mapper_direct.go
 create mode 100644 internal/controller/discovery/loaders/http/mapper_jsonpath.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index c4d3382..fae55cf 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -195,9 +195,9 @@ type ResponseMappingSpec struct {
 	// +kubebuilder:validation:Required
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target address from the response
+	// JSONPath expression to extract the target IP from the response
 	// +kubebuilder:validation:Required
-	Address string `json:"address"`
+	IP string `json:"ip"`
 
 	// JSONPath expression to extract the target port from the response
 	// +kubebuilder:validation:Optional
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 3e7f89f..adc55ee 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -165,9 +165,9 @@ spec:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
                         properties:
-                          address:
+                          ip:
                             description: JSONPath expression to extract the target
-                              address from the response
+                              IP from the response
                             type: string
                           labels:
                             additionalProperties:
@@ -186,7 +186,7 @@ spec:
                               port from the response
                             type: string
                         required:
-                        - address
+                        - ip
                         - name
                         type: object
                       pagination:
diff --git a/go.mod b/go.mod
index 827da2a..782f99d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,8 @@ require (
 require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/PaesslerAG/gval v1.0.0 // indirect
+	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
diff --git a/go.sum b/go.sum
index 8a613b4..f38b648 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,11 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs=
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 99605b9..51a3477 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -37,9 +37,10 @@ const (
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
-	Name    string
-	Address string
-	Labels  map[string]string
+	Name   string
+	IP     string
+	Port   int32
+	Labels map[string]string
 }
 
 type DiscoveryEvent struct {
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 55689df..27eb4c5 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -195,35 +195,45 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 }
 
 func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) {
-	var targets []core.DiscoveredTarget
+	var items []interface{}
 
-	if l.spec.Pagination == nil || l.spec.Pagination.ItemsField == "" {
-		// No pagination config, assume entire response is the target list
-		data, err := json.Marshal(raw)
-		if err != nil {
-			return nil, fmt.Errorf("failed to marshal response: %w", err)
+	if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" {
+		// Extract items array from response using itemsField
+		val, ok := raw[l.spec.Pagination.ItemsField]
+		if !ok {
+			return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField)
 		}
 
-		if err := json.Unmarshal(data, &targets); err != nil {
-			return nil, fmt.Errorf("failed to decode targets: %w", err)
+		arr, ok := val.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField)
 		}
 
-		return targets, nil
+		items = arr
+	} else {
+		// fallback: whole response is array
+		data, _ := json.Marshal(raw)
+		var out []interface{}
+		if err := json.Unmarshal(data, &out); err != nil {
+			return nil, fmt.Errorf("failed to interpret response as list")
+		}
+		items = out
 	}
 
-	// Extract from field
-	items, ok := raw[l.spec.Pagination.ItemsField]
-	if !ok {
-		return nil, fmt.Errorf("itemsField '%s' not found in response", l.spec.Pagination.ItemsField)
-	}
+	// Map items to targets
+	var targets []core.DiscoveredTarget
+	for _, item := range items {
+		obj, ok := item.(map[string]interface{})
+		if !ok {
+			continue
+		}
 
-	data, err := json.Marshal(items)
-	if err != nil {
-		return nil, err
-	}
+		target, err := l.mapItem(obj)
+		if err != nil {
+			return nil, err
+		}
 
-	if err := json.Unmarshal(data, &targets); err != nil {
-		return nil, fmt.Errorf("failed to decode targets from itemsField: %w", err)
+		targets = append(targets, target)
 	}
 
 	return targets, nil
diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
new file mode 100644
index 0000000..bb36113
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -0,0 +1,78 @@
+package http
+
+import (
+	"strconv"
+
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+)
+
+// valueGetter defines the contract for extracting values from a response item
+type valueGetter interface {
+	GetName() (string, error)
+	GetIP() (string, error)
+	GetPort() int32
+	GetLabels() map[string]string
+}
+
+// getGetter selects the extraction strategy based on the spec
+// If no ResponseMapping is defined -> use direct mapping
+func (l *Loader) getGetter(item map[string]interface{}) valueGetter {
+	if l.spec.ResponseMapping == nil {
+		return &directGetter{
+			item: item,
+		}
+	}
+
+	return &jsonPathGetter{
+		item: item,
+		spec: l.spec.ResponseMapping,
+	}
+}
+
+// mapItem is the mapping entrypoint used by the loader
+// It uses the selected valueGetter and produces a DiscoveredTarget
+func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, error) {
+	getter := l.getGetter(item)
+
+	name, err := getter.GetName()
+	if err != nil {
+		return core.DiscoveredTarget{}, err
+	}
+
+	ip, err := getter.GetIP()
+	if err != nil {
+		return core.DiscoveredTarget{}, err
+	}
+
+	port := getter.GetPort()
+	labels := getter.GetLabels()
+
+	return core.DiscoveredTarget{
+		Name:   name,
+		IP:     ip,
+		Port:   port,
+		Labels: labels,
+	}, nil
+}
+
+// extractPort attempts to normalize different JSON types into int32
+//
+// Supports:
+// - float64 (default JSON number type)
+// - string ("1234")
+//
+// Returns 0 if conversion fails (treated as "no port specified").
+func extractPort(val interface{}) int32 {
+	switch v := val.(type) {
+	case float64:
+		return int32(v)
+	case string:
+		p, err := strconv.Atoi(v)
+		if err != nil {
+			return 0
+		}
+		return int32(p)
+	default:
+		return 0
+	}
+}
diff --git a/internal/controller/discovery/loaders/http/mapper_direct.go b/internal/controller/discovery/loaders/http/mapper_direct.go
new file mode 100644
index 0000000..1d135f8
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/mapper_direct.go
@@ -0,0 +1,70 @@
+package http
+
+import (
+	"fmt"
+)
+
+// directGetter extracts values via direct map access
+// Example input:
+//
+//	{
+//	  "name": "router1",
+//	  "ip": "10.0.0.1",
+//	  "port": 57400,
+//	  "labels": { ... }
+//	}
+type directGetter struct {
+	item map[string]interface{}
+}
+
+// GetName extracts the "name" field directly
+func (g *directGetter) GetName() (string, error) {
+	val, ok := g.item["name"].(string)
+	if !ok || val == "" {
+		return "", fmt.Errorf("name must be a non-empty string")
+	}
+	return val, nil
+}
+
+// GetIP extracts the "ip" field directly.
+func (g *directGetter) GetIP() (string, error) {
+	val, ok := g.item["ip"].(string)
+	if !ok || val == "" {
+		return "", fmt.Errorf("ip must be a non-empty string")
+	}
+	return val, nil
+}
+
+// GetPort extracts and normalizes the "port" field
+//
+// Behavior:
+// - supports int, float64, string
+// - returns 0 if value is missing or invalid
+func (g *directGetter) GetPort() int32 {
+	if val, ok := g.item["port"]; ok {
+		return extractPort(val)
+	}
+	return 0
+}
+
+// GetLabels extracts labels from the "labels" field
+// Expected format:
+//
+//	"labels": {
+//	  "key": "value"
+//	}
+//
+// Non-string values are converted to string
+func (g *directGetter) GetLabels() map[string]string {
+	labels := make(map[string]string)
+
+	if val, ok := g.item["labels"]; ok {
+		if m, ok := val.(map[string]interface{}); ok {
+			for k, v := range m {
+				labels[k] = fmt.Sprintf("%v", v)
+			}
+		}
+	}
+
+	return labels
+}
diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
new file mode 100644
index 0000000..194c79b
--- /dev/null
+++ b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
@@ -0,0 +1,87 @@
+package http
+
+import (
+	"fmt"
+
+	"github.com/PaesslerAG/jsonpath"
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+)
+
+// jsonPathGetter extracts values using JSONPath expressions defined in the CR
+// Example mapping:
+//
+//	name: "$.hostname"
+//	ip: "$.ip"
+//	port: "$.port"
+//	labels:
+//	  rack: "$.meta.rack"
+type jsonPathGetter struct {
+	item map[string]interface{}
+	spec *gnmicv1alpha1.ResponseMappingSpec
+}
+
+// helper function to execute JSONPath queries
+func (g *jsonPathGetter) get(expr string) (interface{}, error) {
+	return jsonpath.Get(expr, g.item)
+}
+
+// GetName extracts the target name using JSONPath
+func (g *jsonPathGetter) GetName() (string, error) {
+	val, err := g.get(g.spec.Name)
+	if err != nil {
+		return "", fmt.Errorf("name mapping failed: %w", err)
+	}
+
+	str, ok := val.(string)
+	if !ok || str == "" {
+		return "", fmt.Errorf("name must be a non-empty string")
+	}
+
+	return str, nil
+}
+
+// GetIP extracts the IP using JSONPath
+func (g *jsonPathGetter) GetIP() (string, error) {
+	val, err := g.get(g.spec.IP)
+	if err != nil {
+		return "", fmt.Errorf("IP mapping failed: %w", err)
+	}
+
+	str, ok := val.(string)
+	if !ok || str == "" {
+		return "", fmt.Errorf("IP must be a non-empty string")
+	}
+
+	return str, nil
+}
+
+// GetPort extracts the port using JSONPath
+//
+// Behavior:
+// - returns 0 if no port mapping defined
+// - returns 0 if extraction fails or value invalid
+func (g *jsonPathGetter) GetPort() int32 {
+	if g.spec.Port == "" {
+		return 0
+	}
+
+	val, err := g.get(g.spec.Port)
+	if err != nil {
+		return 0
+	}
+
+	return extractPort(val)
+}
+
+// GetLabels extracts labels using JSONPath expressions defined per label key
+func (g *jsonPathGetter) GetLabels() map[string]string {
+	labels := make(map[string]string)
+
+	for key, expr := range g.spec.Labels {
+		if val, err := jsonpath.Get(expr, g.item); err == nil {
+			labels[key] = fmt.Sprintf("%v", val)
+		}
+	}
+
+	return labels
+}
diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index f7aafb1..cb1e068 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -283,7 +283,8 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE
 		logger.Info(
 			"Applying Target",
 			"target", event.Target.Name,
-			"address", event.Target.Address,
+			"port", event.Target.Port,
+			"ip", event.Target.IP,
 			"labels", event.Target.Labels,
 			"targetsource", m.targetSource.Name,
 		)

From 6a83f49beee0905a996dae1164b9553f044c80e7 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 12:35:33 +0000
Subject: [PATCH 117/165] add support for TargetProfile supplied by provider

---
 api/v1alpha1/targetsource_types.go            |  4 ++++
 .../operator.gnmic.dev_targetsources.yaml     |  4 ++++
 internal/controller/discovery/core/types.go   |  9 ++++----
 .../discovery/loaders/http/mapper.go          | 11 ++++++----
 .../discovery/loaders/http/mapper_direct.go   | 14 ++++++++++++-
 .../discovery/loaders/http/mapper_jsonpath.go | 21 ++++++++++++++++++-
 6 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index fae55cf..c83e5e2 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -208,6 +208,10 @@ type ResponseMappingSpec struct {
 	// with values from the response taking precedence in case of conflicts.
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
+
+	// JSONPath expression to extract the target profile from the response
+	// +kubebuilder:validation:Optional
+	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index adc55ee..b4ecb44 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -185,6 +185,10 @@ spec:
                             description: JSONPath expression to extract the target
                               port from the response
                             type: string
+                          targetProfile:
+                            description: JSONPath expression to extract the target
+                              profile from the response
+                            type: string
                         required:
                         - ip
                         - name
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 51a3477..1b40897 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -37,10 +37,11 @@ const (
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
-	Name   string
-	IP     string
-	Port   int32
-	Labels map[string]string
+	Name          string
+	IP            string
+	Port          int32
+	Labels        map[string]string
+	TargetProfile string
 }
 
 type DiscoveryEvent struct {
diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index bb36113..95aa557 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -12,6 +12,7 @@ type valueGetter interface {
 	GetIP() (string, error)
 	GetPort() int32
 	GetLabels() map[string]string
+	GetTargetProfile() string
 }
 
 // getGetter selects the extraction strategy based on the spec
@@ -46,12 +47,14 @@ func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, er
 
 	port := getter.GetPort()
 	labels := getter.GetLabels()
+	targetProfile := getter.GetTargetProfile()
 
 	return core.DiscoveredTarget{
-		Name:   name,
-		IP:     ip,
-		Port:   port,
-		Labels: labels,
+		Name:          name,
+		IP:            ip,
+		Port:          port,
+		Labels:        labels,
+		TargetProfile: targetProfile,
 	}, nil
 }
 
diff --git a/internal/controller/discovery/loaders/http/mapper_direct.go b/internal/controller/discovery/loaders/http/mapper_direct.go
index 1d135f8..185e1cb 100644
--- a/internal/controller/discovery/loaders/http/mapper_direct.go
+++ b/internal/controller/discovery/loaders/http/mapper_direct.go
@@ -11,7 +11,8 @@ import (
 //	  "name": "router1",
 //	  "ip": "10.0.0.1",
 //	  "port": 57400,
-//	  "labels": { ... }
+//	  "labels": { ... },
+//	  "targetProfile": "profile1"
 //	}
 type directGetter struct {
 	item map[string]interface{}
@@ -68,3 +69,14 @@ func (g *directGetter) GetLabels() map[string]string {
 
 	return labels
 }
+
+// GetTargetProfile extracts the "targetProfile" field directly
+//
+// Behavior:
+// - returns "" if value is missing or invalid
+func (g *directGetter) GetTargetProfile() string {
+	if val, ok := g.item["targetProfile"].(string); ok {
+		return val
+	}
+	return ""
+}
diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
index 194c79b..85bf00a 100644
--- a/internal/controller/discovery/loaders/http/mapper_jsonpath.go
+++ b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
@@ -78,10 +78,29 @@ func (g *jsonPathGetter) GetLabels() map[string]string {
 	labels := make(map[string]string)
 
 	for key, expr := range g.spec.Labels {
-		if val, err := jsonpath.Get(expr, g.item); err == nil {
+		if val, err := g.get(expr); err == nil {
 			labels[key] = fmt.Sprintf("%v", val)
 		}
 	}
 
 	return labels
 }
+
+// GetTargetProfile extracts the target profile using JSONPath
+//
+// Behavior:
+// - returns "" if no target profile mapping defined
+// - returns "" if extraction fails or value invalid
+func (g *jsonPathGetter) GetTargetProfile() string {
+	val, err := g.get(g.spec.TargetProfile)
+	if err != nil {
+		return ""
+	}
+
+	str, ok := val.(string)
+	if !ok {
+		return ""
+	}
+
+	return str
+}

From 1e9feb6f92fe6feb2266562aee93d39e3793fe35 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 12:44:19 +0000
Subject: [PATCH 118/165] refactor

---
 internal/controller/discovery/loaders.go      |  4 ++++
 .../discovery/loaders/http/loader.go          | 24 +++++++++++++------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 1e5ea46..42ab588 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -47,6 +47,8 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi
 	}
 }
 
+// resolveAuthorizationIntoSpec fetches credentials from Kubernetes Secrets
+// and populates the AuthorizationSpec accordingly
 func resolveAuthorizationIntoSpec(
 	ctx context.Context,
 	c client.Client,
@@ -132,6 +134,8 @@ func resolveAuthorizationIntoSpec(
 	return nil
 }
 
+// resolveTLSIntoSpec fetches TLS credentials from Kubernetes Secrets
+// and populates the ClientTLSConfig accordingly
 func resolveTLSIntoSpec(
 	ctx context.Context,
 	c client.Client,
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 27eb4c5..5742092 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -19,20 +19,26 @@ import (
 )
 
 // Loader implements the HTTP pull discovery mechanism
+// It periodically polls an HTTP endpoint, extracts targets from the response,
+// and emits discovery snapshots downstream
 type Loader struct {
 	loaderCfg core.CommonLoaderConfig
 	spec      gnmicv1alpha1.HTTPConfig
 }
 
-// New instantiates the http loader with the provided config
+// New creates a new HTTP loader instance with the provided configuration.
+// The loader is stateless apart from its config and spec
 func New(cfg core.CommonLoaderConfig, httpConfig gnmicv1alpha1.HTTPConfig) core.Loader {
 	return &Loader{loaderCfg: cfg, spec: httpConfig}
 }
 
+// Name returns the loader's name, used for logging and metrics
 func (l *Loader) Name() string {
 	return "http"
 }
 
+// Run starts the HTTP discovery loop
+// It performs an immediate fetch and then continues polling at a fixed interval
 func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error {
 	if l.spec.URL == "" {
 		return nil
@@ -52,11 +58,6 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 
 	logger.Info("HTTP loader started")
 
-	// Input Validation of spec
-	// if l.spec.URL == "nil" {
-	// 	return errors.New("HTTP loader requires spec.provider.http to be set")
-	// }
-
 	client, err := l.buildHTTPClient()
 	if err != nil {
 		return fmt.Errorf("failed to build HTTP client: %w", err)
@@ -73,6 +74,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 
 	// helper function to fetch targets and emit discovery messages
 	fetchAndEmit := func() {
+		// Fetch targets from HTTP endpoint
 		targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client)
 		if err != nil {
 			logger.Error(
@@ -83,6 +85,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			return
 		}
 
+		// Emit discovery snapshot downstream
 		snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString())
 		if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.loaderCfg.ChunkSize); err != nil {
 			logger.Error(
@@ -117,11 +120,13 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	}
 }
 
+// buildHTTPClient constructs an HTTP client with optional configuration
 func (l *Loader) buildHTTPClient() (*http.Client, error) {
 	tlsConfig := &tls.Config{
 		InsecureSkipVerify: l.spec.TLS != nil && l.spec.TLS.InsecureSkipVerify,
 	}
 
+	// If a CA bundle is provided, add it to the TLS config
 	if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 {
 		certPool := x509.NewCertPool()
 		if ok := certPool.AppendCertsFromPEM(l.spec.TLS.CABundle); !ok {
@@ -130,6 +135,7 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) {
 		tlsConfig.RootCAs = certPool
 	}
 
+	// Build the HTTP client with the specified timeout and TLS config
 	return &http.Client{
 		Timeout: l.spec.Timeout.Duration,
 		Transport: &http.Transport{
@@ -138,6 +144,7 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) {
 	}, nil
 }
 
+// fetchTargetsFromHTTPEndpoint retrieves targets from the configured HTTP endpoint
 func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,
@@ -146,14 +153,15 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	currentUrl := l.spec.URL
 
 	for {
+		// Create HTTP request with context
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentUrl, nil)
 		if err != nil {
 			return nil, fmt.Errorf("creating HTTP request failed: %w", err)
 		}
-
 		req.Header.Set("Accept", "application/json")
 		l.applyAuthorization(req)
 
+		// Execute HTTP request
 		resp, err := client.Do(req)
 		if err != nil {
 			return nil, fmt.Errorf("HTTP request failed: %w", err)
@@ -194,6 +202,8 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	return allTargets, nil
 }
 
+// extractTargetsFromResponse extracts items from the response
+// and maps each item into a DiscoveredTarget
 func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
 

From 0e8ea1a9cea0bb801598ea87fe6d2378ddf0f947 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 13:13:21 +0000
Subject: [PATCH 119/165] fix interfaces

---
 .../discovery/loaders/http/loader.go          | 49 ++++++++++---------
 .../discovery/loaders/http/pagination.go      | 13 ++++-
 2 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 5742092..eef202b 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -172,7 +172,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 		}
 
 		// Decode response into raw map for pagination support
-		var raw map[string]interface{}
+		var raw interface{}
 		if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
 			return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
 		}
@@ -204,30 +204,33 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 
 // extractTargetsFromResponse extracts items from the response
 // and maps each item into a DiscoveredTarget
-func (l *Loader) extractTargetsFromResponse(raw map[string]interface{}) ([]core.DiscoveredTarget, error) {
+func (l *Loader) extractTargetsFromResponse(raw interface{}) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
 
-	if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" {
-		// Extract items array from response using itemsField
-		val, ok := raw[l.spec.Pagination.ItemsField]
-		if !ok {
-			return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField)
-		}
-
-		arr, ok := val.([]interface{})
-		if !ok {
-			return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField)
-		}
-
-		items = arr
-	} else {
-		// fallback: whole response is array
-		data, _ := json.Marshal(raw)
-		var out []interface{}
-		if err := json.Unmarshal(data, &out); err != nil {
-			return nil, fmt.Errorf("failed to interpret response as list")
-		}
-		items = out
+	switch v := raw.(type) {
+	// Top-level array response
+	case []interface{}:
+		items = v
+	// Object with itemsField containing the array
+	case map[string]interface{}:
+		if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" {
+			// Extract items array from response using itemsField
+			val, ok := v[l.spec.Pagination.ItemsField]
+			if !ok {
+				return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField)
+			}
+
+			arr, ok := val.([]interface{})
+			if !ok {
+				return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField)
+			}
+
+			items = arr
+		} else {
+			return nil, fmt.Errorf("response is an object but no itemsField specified for TargetSource %s/%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
+		}
+	default:
+		return nil, fmt.Errorf("unexpected response format")
 	}
 
 	// Map items to targets
diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go
index 6a4a3ec..9fef778 100644
--- a/internal/controller/discovery/loaders/http/pagination.go
+++ b/internal/controller/discovery/loaders/http/pagination.go
@@ -5,12 +5,20 @@ import (
 	"net/url"
 )
 
-func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error) {
+// extractNextPageInfo extracts pagination information from a response
+func (l *Loader) extractNextPageInfo(raw interface{}) (string, error) {
 	if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" {
 		return "", nil
 	}
 
-	val, ok := raw[l.spec.Pagination.NextField]
+	// Only objects can have "next" fields
+	obj, ok := raw.(map[string]interface{})
+	if !ok {
+		// array case -> no pagination
+		return "", nil
+	}
+
+	val, ok := obj[l.spec.Pagination.NextField]
 	if !ok {
 		return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField)
 	}
@@ -23,6 +31,7 @@ func (l *Loader) extractNextPageInfo(raw map[string]interface{}) (string, error)
 	return next, nil
 }
 
+// buildNextURL constructs the URL for the next page based on the current URL and pagination info
 func (l *Loader) buildNextURL(currentURL, nextVal string) (string, error) {
 	// nextVal is a full URL -> return as is
 	if parsed, err := url.Parse(nextVal); err == nil && parsed.Scheme != "" {

From 262760c43a78cd62a6bd01681b1b575e11756531 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 15 May 2026 13:57:56 +0000
Subject: [PATCH 120/165] fix incorrect conversion between integer types

---
 internal/controller/discovery/loaders/http/mapper.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index 95aa557..de618fa 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -1,6 +1,7 @@
 package http
 
 import (
+	"math"
 	"strconv"
 
 	"github.com/gnmic/operator/internal/controller/discovery/core"
@@ -68,9 +69,12 @@ func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, er
 func extractPort(val interface{}) int32 {
 	switch v := val.(type) {
 	case float64:
+		if v < 0 || v > math.MaxInt32 {
+			return 0
+		}
 		return int32(v)
 	case string:
-		p, err := strconv.Atoi(v)
+		p, err := strconv.ParseInt(v, 10, 32)
 		if err != nil {
 			return 0
 		}

From 9d356081cecb26b33140ded75d2870e782634d8f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 14 May 2026 09:42:23 +0000
Subject: [PATCH 121/165] update targetsource

---
 api/v1alpha1/targetsource_types.go            | 168 +++++++++++-
 api/v1alpha1/zz_generated.deepcopy.go         | 179 +++++++++++--
 .../operator.gnmic.dev_targetsources.yaml     | 249 +++++++++++++++++-
 3 files changed, 556 insertions(+), 40 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 3d69743..143da3c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -17,37 +17,191 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // TargetSourceSpec defines the desired state of TargetSource
 // +kubebuilder:validation:Required
 type TargetSourceSpec struct {
+	// Provider defines the source of targets for this TargetSource
+	// Only one provider can be specified per TargetSource
+	// +kubebuilder:validation:Required
 	Provider *ProviderSpec `json:"provider"`
 
+	// TODO: implement in message processor
+	// Optional port to use for discovered targets if not specified by the provider
+	// +kubebuilder:validation:Optional
+	TargetPort int32 `json:"targetPort,omitempty"`
+
+	// Optional labels to apply to all targets discovered by this TargetSource
 	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
+	// The TargetProfile to use for targets discovered by this TargetSource
+	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinLength=1
 	TargetProfile string `json:"targetProfile"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=http;consul
+// ProviderSpec defines the source of targets for a TargetSource
+// Only one provider can be specified per TargetSource
+// +kubebuilder:validation:ExactlyOneOf=http
 type ProviderSpec struct {
-	HTTP   *HTTPConfig   `json:"http,omitempty"`
-	Consul *ConsulConfig `json:"consul,omitempty"`
+	// HTTP defines the configuration for a HTTP provider
+	HTTP *HTTPConfig `json:"http,omitempty"`
 }
 
+// HTTPConfig defines the configuration for the HTTP provider
+// +kubebuilder:validation:AtLeastOneOf=url;acceptPush
 type HTTPConfig struct {
-	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url"`
+	// URL of the HTTP endpoint to pull targets from
+	// If defined, the loader will periodically poll this endpoint for targets
+	// +kubebuilder:validation:Optional
+	URL string `json:"url,omitempty"`
+
+	// If true, the loader will accept pushed target updates to the controller endpoint
+	// The endpoint will be /{namespace}/{targetsource}/<todo>
+	// +kubebuilder:default=false
 	// +kubebuilder:validation:Optional
 	AcceptPush bool `json:"acceptPush,omitempty"`
+
+	// Optional authorization configuration for accessing the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
+
+	// Optional interval for polling the HTTP endpoint for targets
+	// TODO: increase default value
+	// +kubebuilder:default="30s"
+	// +kubebuilder:validation:Optional
+	PollInterval *metav1.Duration `json:"interval,omitempty"`
+
+	// Optional timeout for HTTP requests to the endpoint
+	// +kubebuilder:default="10s"
+	// +kubebuilder:validation:Optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
+
+	// Optional TLS configuration for connecting to the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	TLS *ClientTLSConfig `json:"tls,omitempty"`
+
+	// Optional pagination configuration for parsing responses from the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	Pagination *PaginationSpec `json:"pagination,omitempty"`
+
+	// Optional mapping configuration for parsing responses from the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
+}
+
+// +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive"
+type ClientTLSConfig struct {
+	// Skip TLS verification of the Provider's certificate.
+	// +kubebuilder:default:=false
+	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
+
+	// Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+	// chain presented by the Provider. Only used if using HTTPS to connect to Provider and
+	// ignored for HTTP connections.
+	// Mutually exclusive with CABundleSecretRef.
+	// +optional
+	CABundle []byte `json:"caBundle,omitempty"`
+
+	// Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+	// verifying the certificate chain presented by the Provider when using HTTPS.
+	// Mutually exclusive with CABundle.
+	CABundleSecretRef *corev1.SecretKeySelector `json:"caBundleSecretRef,omitempty"`
+}
+
+// AuthorizationSpec defines the configuration for authentication
+// +kubebuilder:validation:ExactlyOneOf=basic;token
+type AuthorizationSpec struct {
+	// Basic authentication configuration
+	Basic *BasicAuthSpec `json:"basic,omitempty"`
+	// Token-based authentication configuration
+	Token *TokenAuthSpec `json:"token,omitempty"`
+	// JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
+	// MTLS
 }
 
-type ConsulConfig struct {
+// BasicAuthSpec defines the configuration for basic authentication
+// Enforce EITHER inline creds OR secret ref
+// +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
+type BasicAuthSpec struct {
+	// Username for basic auth
+	// Mutually exclusive with CredentialsSecretRef.
+	Username string `json:"username,omitempty"`
+	// Password for basic auth
+	// Mutually exclusive with CredentialsSecretRef.
+	Password string `json:"password,omitempty"`
+
+	// Reference to a Secret containing "username" and "password" keys to use for
+	// basic authentication when connecting to the Provider.
+	// Mutually exclusive with Username and Password.
+	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
+}
+
+// TokenAuthSpec defines the configuration for token-based authentication
+// +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
+type TokenAuthSpec struct {
+	// Scheme for the token, e.g. "Bearer"
 	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url,omitempty"`
+	Scheme string `json:"scheme"`
+	// Token value for authentication
+	// Mutually exclusive with TokenSecretRef.
+	Token string `json:"token,omitempty"`
+	// Reference to a Secret containing a key with the token value to use for
+	// authentication when connecting to the Provider.
+	// Mutually exclusive with Token.
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
+// type JWTAuthSpec struct {
+// 	// Static pre-generated JWT
+// 	Token          string                    `json:"token,omitempty"`
+// 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+// 	// Optional: generate JWT dynamically
+// 	Claims              map[string]string         `json:"claims,omitempty"`
+// 	Key                 string                    `json:"key,omitempty"`
+// 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
+// 	// HS256, RS256, ES256, etc.
+// 	Algorithm string           `json:"algorithm,omitempty"`
+// 	TTL       *metav1.Duration `json:"ttl,omitempty"`
+// }
+
+// PaginationSpec defines the configuration for paginating through responses from providers
+type PaginationSpec struct {
+	// JSONPath-style expression to extract the list of targets from the response
+	// Example: "results"
+	ItemsField string `json:"itemsField,omitempty"`
+
+	// JSONPath-style expression to extract the next page token or URL from the response for pagination
+	// Example: "next"
+	NextField string `json:"nextField,omitempty"`
+}
+
+// JSONPath-style expressions to extract target fields from the response
+// and map them to the corresponding Target fields.
+type ResponseMappingSpec struct {
+	// JSONPath expression to extract the target name from the response
+	// +kubebuilder:validation:Required
+	Name string `json:"name"`
+
+	// JSONPath expression to extract the target address from the response
+	// +kubebuilder:validation:Required
+	Address string `json:"address"`
+
+	// JSONPath expression to extract the target port from the response
+	// +kubebuilder:validation:Optional
+	Port string `json:"port,omitempty"`
+
+	// JSONPath expression to extract the target labels from the response
+	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+	// with values from the response taking precedence in case of conflicts.
+	// +kubebuilder:validation:Optional
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 61e81fd..dc4b784 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -46,6 +46,76 @@ func (in *APIConfig) DeepCopy() *APIConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizationSpec) DeepCopyInto(out *AuthorizationSpec) {
+	*out = *in
+	if in.Basic != nil {
+		in, out := &in.Basic, &out.Basic
+		*out = new(BasicAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Token != nil {
+		in, out := &in.Token, &out.Token
+		*out = new(TokenAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationSpec.
+func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizationSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) {
+	*out = *in
+	if in.CredentialsSecretRef != nil {
+		in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuthSpec.
+func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(BasicAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
+	*out = *in
+	if in.CABundle != nil {
+		in, out := &in.CABundle, &out.CABundle
+		*out = make([]byte, len(*in))
+		copy(*out, *in)
+	}
+	if in.CABundleSecretRef != nil {
+		in, out := &in.CABundleSecretRef, &out.CABundleSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientTLSConfig.
+func (in *ClientTLSConfig) DeepCopy() *ClientTLSConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(ClientTLSConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Cluster) DeepCopyInto(out *Cluster) {
 	*out = *in
@@ -213,21 +283,6 @@ func (in *ClusterTargetState) DeepCopy() *ClusterTargetState {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ConsulConfig) DeepCopyInto(out *ConsulConfig) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsulConfig.
-func (in *ConsulConfig) DeepCopy() *ConsulConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(ConsulConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GRPCKeepAliveConfig) DeepCopyInto(out *GRPCKeepAliveConfig) {
 	*out = *in
@@ -273,6 +328,36 @@ func (in *GRPCTunnelConfig) DeepCopy() *GRPCTunnelConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	*out = *in
+	if in.Authorization != nil {
+		in, out := &in.Authorization, &out.Authorization
+		*out = new(AuthorizationSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PollInterval != nil {
+		in, out := &in.PollInterval, &out.PollInterval
+		*out = new(metav1.Duration)
+		**out = **in
+	}
+	if in.Timeout != nil {
+		in, out := &in.Timeout, &out.Timeout
+		*out = new(metav1.Duration)
+		**out = **in
+	}
+	if in.TLS != nil {
+		in, out := &in.TLS, &out.TLS
+		*out = new(ClientTLSConfig)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Pagination != nil {
+		in, out := &in.Pagination, &out.Pagination
+		*out = new(PaginationSpec)
+		**out = **in
+	}
+	if in.ResponseMapping != nil {
+		in, out := &in.ResponseMapping, &out.ResponseMapping
+		*out = new(ResponseMappingSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig.
@@ -587,6 +672,21 @@ func (in *OutputStatus) DeepCopy() *OutputStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PaginationSpec) DeepCopyInto(out *PaginationSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PaginationSpec.
+func (in *PaginationSpec) DeepCopy() *PaginationSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PaginationSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pipeline) DeepCopyInto(out *Pipeline) {
 	*out = *in
@@ -824,12 +924,7 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) {
 	if in.HTTP != nil {
 		in, out := &in.HTTP, &out.HTTP
 		*out = new(HTTPConfig)
-		**out = **in
-	}
-	if in.Consul != nil {
-		in, out := &in.Consul, &out.Consul
-		*out = new(ConsulConfig)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 }
 
@@ -843,6 +938,28 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResponseMappingSpec.
+func (in *ResponseMappingSpec) DeepCopy() *ResponseMappingSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ResponseMappingSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) {
 	*out = *in
@@ -1384,6 +1501,26 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenAuthSpec.
+func (in *TokenAuthSpec) DeepCopy() *TokenAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(TokenAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TunnelTargetPolicy) DeepCopyInto(out *TunnelTargetPolicy) {
 	*out = *in
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 37d6919..d603546 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -40,33 +40,258 @@ spec:
             description: TargetSourceSpec defines the desired state of TargetSource
             properties:
               provider:
+                description: |-
+                  Provider defines the source of targets for this TargetSource
+                  Only one provider can be specified per TargetSource
                 properties:
-                  consul:
-                    properties:
-                      url:
-                        minLength: 1
-                        type: string
-                    type: object
                   http:
+                    description: HTTP defines the configuration for a HTTP provider
                     properties:
                       acceptPush:
+                        default: false
+                        description: |-
+                          If true, the loader will accept pushed target updates to the controller endpoint
+                          The endpoint will be /{namespace}/{targetsource}/<todo>
                         type: boolean
+                      authorization:
+                        description: Optional authorization configuration for accessing
+                          the HTTP endpoint
+                        properties:
+                          basic:
+                            description: Basic authentication configuration
+                            properties:
+                              credentialsSecretRef:
+                                description: |-
+                                  Reference to a Secret containing "username" and "password" keys to use for
+                                  basic authentication when connecting to the Provider.
+                                  Mutually exclusive with Username and Password.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                              password:
+                                description: |-
+                                  Password for basic auth
+                                  Mutually exclusive with CredentialsSecretRef.
+                                type: string
+                              username:
+                                description: |-
+                                  Username for basic auth
+                                  Mutually exclusive with CredentialsSecretRef.
+                                type: string
+                            type: object
+                            x-kubernetes-validations:
+                            - message: either credentialsSecretRef OR both username
+                                and password must be set, but not a mix
+                              rule: (has(self.credentialsSecretRef) && !has(self.username)
+                                && !has(self.password)) || (!has(self.credentialsSecretRef)
+                                && has(self.username) && has(self.password))
+                          token:
+                            description: Token-based authentication configuration
+                            properties:
+                              scheme:
+                                description: Scheme for the token, e.g. "Bearer"
+                                minLength: 1
+                                type: string
+                              token:
+                                description: |-
+                                  Token value for authentication
+                                  Mutually exclusive with TokenSecretRef.
+                                type: string
+                              tokenSecretRef:
+                                description: |-
+                                  Reference to a Secret containing a key with the token value to use for
+                                  authentication when connecting to the Provider.
+                                  Mutually exclusive with Token.
+                                properties:
+                                  key:
+                                    description: The key of the secret to select from.  Must
+                                      be a valid secret key.
+                                    type: string
+                                  name:
+                                    default: ""
+                                    description: |-
+                                      Name of the referent.
+                                      This field is effectively required, but due to backwards compatibility is
+                                      allowed to be empty. Instances of this type with an empty value here are
+                                      almost certainly wrong.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  optional:
+                                    description: Specify whether the Secret or its
+                                      key must be defined
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                                x-kubernetes-map-type: atomic
+                            required:
+                            - scheme
+                            type: object
+                            x-kubernetes-validations:
+                            - message: either token or tokenSecretRef must be set,
+                                but not both
+                              rule: has(self.token) != has(self.tokenSecretRef)
+                        type: object
+                        x-kubernetes-validations:
+                        - message: exactly one of the fields in [basic token] must
+                            be set
+                          rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
+                            == 1'
+                      interval:
+                        default: 30s
+                        description: Optional interval for polling the HTTP endpoint
+                          for targets
+                        type: string
+                      mapping:
+                        description: Optional mapping configuration for parsing responses
+                          from the HTTP endpoint
+                        properties:
+                          address:
+                            description: JSONPath expression to extract the target
+                              address from the response
+                            type: string
+                          labels:
+                            additionalProperties:
+                              type: string
+                            description: |-
+                              JSONPath expression to extract the target labels from the response
+                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+                              with values from the response taking precedence in case of conflicts.
+                            type: object
+                          name:
+                            description: JSONPath expression to extract the target
+                              name from the response
+                            type: string
+                          port:
+                            description: JSONPath expression to extract the target
+                              port from the response
+                            type: string
+                        required:
+                        - address
+                        - name
+                        type: object
+                      pagination:
+                        description: Optional pagination configuration for parsing
+                          responses from the HTTP endpoint
+                        properties:
+                          itemsField:
+                            description: |-
+                              JSONPath-style expression to extract the list of targets from the response
+                              Example: "results"
+                            type: string
+                          nextField:
+                            description: |-
+                              JSONPath-style expression to extract the next page token or URL from the response for pagination
+                              Example: "next"
+                            type: string
+                        type: object
+                        x-kubernetes-validations:
+                        - message: static JWT token and generated JWT configuration
+                            cannot be combined
+                          rule: '!((has(self.token) || has(self.tokenSecretRef)) &&
+                            ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))'
+                        - message: algorithm must be specified when generating a JWT
+                          rule: '!has(self.signingKeySecretRef) || self.algorithm
+                            != ""'
+                      timeout:
+                        default: 10s
+                        description: Optional timeout for HTTP requests to the endpoint
+                        type: string
+                      tls:
+                        description: Optional TLS configuration for connecting to
+                          the HTTP endpoint
+                        properties:
+                          caBundle:
+                            description: |-
+                              Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+                              chain presented by the Provider. Only used if using HTTPS to connect to Provider and
+                              ignored for HTTP connections.
+                              Mutually exclusive with CABundleSecretRef.
+                            format: byte
+                            type: string
+                          caBundleSecretRef:
+                            description: |-
+                              Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+                              verifying the certificate chain presented by the Provider when using HTTPS.
+                              Mutually exclusive with CABundle.
+                            properties:
+                              key:
+                                description: The key of the secret to select from.  Must
+                                  be a valid secret key.
+                                type: string
+                              name:
+                                default: ""
+                                description: |-
+                                  Name of the referent.
+                                  This field is effectively required, but due to backwards compatibility is
+                                  allowed to be empty. Instances of this type with an empty value here are
+                                  almost certainly wrong.
+                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                type: string
+                              optional:
+                                description: Specify whether the Secret or its key
+                                  must be defined
+                                type: boolean
+                            required:
+                            - key
+                            type: object
+                            x-kubernetes-map-type: atomic
+                          insecureSkipVerify:
+                            default: false
+                            description: Skip TLS verification of the Provider's certificate.
+                            type: boolean
+                        type: object
+                        x-kubernetes-validations:
+                        - message: caBundle and caBundleSecretRef are mutually exclusive
+                          rule: '!(has(self.caBundle) && has(self.caBundleSecretRef))'
                       url:
-                        minLength: 1
+                        description: |-
+                          URL of the HTTP endpoint to pull targets from
+                          If defined, the loader will periodically poll this endpoint for targets
                         type: string
-                    required:
-                    - url
                     type: object
+                    x-kubernetes-validations:
+                    - message: at least one of the fields in [url acceptPush] must
+                        be set
+                      rule: '[has(self.url),has(self.acceptPush)].filter(x,x==true).size()
+                        >= 1'
                 type: object
                 x-kubernetes-validations:
-                - message: exactly one of the fields in [http consul] must be set
-                  rule: '[has(self.http),has(self.consul)].filter(x,x==true).size()
-                    == 1'
+                - message: exactly one of the fields in [http] must be set
+                  rule: '[has(self.http)].filter(x,x==true).size() == 1'
               targetLabels:
                 additionalProperties:
                   type: string
+                description: Optional labels to apply to all targets discovered by
+                  this TargetSource
                 type: object
+              targetPort:
+                description: Optional port to use for discovered targets if not specified
+                  by the provider
+                format: int32
+                type: integer
               targetProfile:
+                description: The TargetProfile to use for targets discovered by this
+                  TargetSource
                 minLength: 1
                 type: string
             required:

From bcc0b4f993526db5ffd7b9f45dc6fd47c05c4d00 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 20 May 2026 13:30:54 -0600
Subject: [PATCH 122/165] cherry-pick 5d95c90: DiscoveredTarget type changes

---
 api/v1alpha1/targetsource_types.go                 |  4 ++--
 .../bases/operator.gnmic.dev_targetsources.yaml    |  6 +++---
 go.mod                                             |  2 ++
 go.sum                                             |  5 +++++
 internal/controller/discovery/core/types.go        |  7 ++++---
 internal/controller/discovery/loaders.go           |  2 --
 .../controller/discovery/loaders/http/loader.go    | 14 ++++++++------
 internal/controller/discovery/message_processor.go |  3 ++-
 8 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 143da3c..666805d 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -189,9 +189,9 @@ type ResponseMappingSpec struct {
 	// +kubebuilder:validation:Required
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target address from the response
+	// JSONPath expression to extract the target IP from the response
 	// +kubebuilder:validation:Required
-	Address string `json:"address"`
+	IP string `json:"ip"`
 
 	// JSONPath expression to extract the target port from the response
 	// +kubebuilder:validation:Optional
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d603546..1b71922 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -165,9 +165,9 @@ spec:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
                         properties:
-                          address:
+                          ip:
                             description: JSONPath expression to extract the target
-                              address from the response
+                              IP from the response
                             type: string
                           labels:
                             additionalProperties:
@@ -186,7 +186,7 @@ spec:
                               port from the response
                             type: string
                         required:
-                        - address
+                        - ip
                         - name
                         type: object
                       pagination:
diff --git a/go.mod b/go.mod
index 9dc2b78..c877a7b 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,8 @@ require (
 require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/PaesslerAG/gval v1.0.0 // indirect
+	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
diff --git a/go.sum b/go.sum
index 45485f1..d900003 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,11 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs=
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 99605b9..51a3477 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -37,9 +37,10 @@ const (
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
-	Name    string
-	Address string
-	Labels  map[string]string
+	Name   string
+	IP     string
+	Port   int32
+	Labels map[string]string
 }
 
 type DiscoveryEvent struct {
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index c888c27..c4ebe78 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -15,8 +15,6 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec
 	case spec.Provider.HTTP != nil:
 		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
 		return http.New(*cfg), nil
-	case spec.Provider.Consul != nil:
-		return nil, fmt.Errorf("Unimplemented targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 3325adb..a2bfa0e 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -53,14 +53,16 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString())
 			targets := []core.DiscoveredTarget{
 				{
-					Name:    "ceos1",
-					Address: "clab-3-nodes-ceos1:6030",
-					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
+					Name:   "ceos1",
+					IP:     "clab-3-nodes-ceos1",
+					Port:   57400,
+					Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 				{
-					Name:    "leaf1",
-					Address: "clab-3-nodes-leaf1:57400",
-					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
+					Name:   "leaf1",
+					IP:     "clab-3-nodes-leaf1",
+					Port:   57400,
+					Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 			}
 
diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index f7aafb1..cb1e068 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -283,7 +283,8 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE
 		logger.Info(
 			"Applying Target",
 			"target", event.Target.Name,
-			"address", event.Target.Address,
+			"port", event.Target.Port,
+			"ip", event.Target.IP,
 			"labels", event.Target.Labels,
 			"targetsource", m.targetSource.Name,
 		)

From d523adb2653a9fc6abaa174b90a6589a04007b58 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 20 May 2026 13:33:18 -0600
Subject: [PATCH 123/165] renamed IP to Address

---
 api/v1alpha1/targetsource_types.go               |  4 ++--
 .../bases/operator.gnmic.dev_targetsources.yaml  |  6 +++---
 internal/controller/discovery/core/types.go      |  8 ++++----
 .../controller/discovery/loaders/http/loader.go  | 16 ++++++++--------
 .../controller/discovery/message_processor.go    |  2 +-
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 666805d..143da3c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -189,9 +189,9 @@ type ResponseMappingSpec struct {
 	// +kubebuilder:validation:Required
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target IP from the response
+	// JSONPath expression to extract the target address from the response
 	// +kubebuilder:validation:Required
-	IP string `json:"ip"`
+	Address string `json:"address"`
 
 	// JSONPath expression to extract the target port from the response
 	// +kubebuilder:validation:Optional
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 1b71922..d603546 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -165,9 +165,9 @@ spec:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
                         properties:
-                          ip:
+                          address:
                             description: JSONPath expression to extract the target
-                              IP from the response
+                              address from the response
                             type: string
                           labels:
                             additionalProperties:
@@ -186,7 +186,7 @@ spec:
                               port from the response
                             type: string
                         required:
-                        - ip
+                        - address
                         - name
                         type: object
                       pagination:
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 51a3477..66bbe50 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -37,10 +37,10 @@ const (
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
-	Name   string
-	IP     string
-	Port   int32
-	Labels map[string]string
+	Name    string
+	Address string
+	Port    int32
+	Labels  map[string]string
 }
 
 type DiscoveryEvent struct {
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index a2bfa0e..aace9cd 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -53,16 +53,16 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString())
 			targets := []core.DiscoveredTarget{
 				{
-					Name:   "ceos1",
-					IP:     "clab-3-nodes-ceos1",
-					Port:   57400,
-					Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
+					Name:    "ceos1",
+					Address: "clab-3-nodes-ceos1",
+					Port:    57400,
+					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 				{
-					Name:   "leaf1",
-					IP:     "clab-3-nodes-leaf1",
-					Port:   57400,
-					Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
+					Name:    "leaf1",
+					Address: "clab-3-nodes-leaf1",
+					Port:    57400,
+					Labels:  map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()},
 				},
 			}
 
diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/message_processor.go
index cb1e068..930635d 100644
--- a/internal/controller/discovery/message_processor.go
+++ b/internal/controller/discovery/message_processor.go
@@ -284,7 +284,7 @@ func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryE
 			"Applying Target",
 			"target", event.Target.Name,
 			"port", event.Target.Port,
-			"ip", event.Target.IP,
+			"ip", event.Target.Address,
 			"labels", event.Target.Labels,
 			"targetsource", m.targetSource.Name,
 		)

From 2c0d7cb2d69c7852ea0fbab9262013931bf3a8e2 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 20 May 2026 14:24:27 -0600
Subject: [PATCH 124/165] disabled JWTAuthSpec validations

---
 api/v1alpha1/targetsource_types.go                     | 4 ++--
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 8 --------
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 143da3c..5bbfcec 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -156,8 +156,8 @@ type TokenAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
+// disabled: +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
+// disabled: +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
 // type JWTAuthSpec struct {
 // 	// Static pre-generated JWT
 // 	Token          string                    `json:"token,omitempty"`
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d603546..6851ad7 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -204,14 +204,6 @@ spec:
                               Example: "next"
                             type: string
                         type: object
-                        x-kubernetes-validations:
-                        - message: static JWT token and generated JWT configuration
-                            cannot be combined
-                          rule: '!((has(self.token) || has(self.tokenSecretRef)) &&
-                            ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))'
-                        - message: algorithm must be specified when generating a JWT
-                          rule: '!has(self.signingKeySecretRef) || self.algorithm
-                            != ""'
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint

From 77a5bd4277eba07fb623ef7b4b21c6e2a2fc1bf7 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Wed, 20 May 2026 19:04:22 -0600
Subject: [PATCH 125/165] cherry-pick 6a83f49: added TargetProfile variable to
 DiscoveredTarget

---
 api/v1alpha1/targetsource_types.go                     | 4 ++++
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 4 ++++
 internal/controller/discovery/core/types.go            | 9 +++++----
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 5bbfcec..5b0f8e5 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -202,6 +202,10 @@ type ResponseMappingSpec struct {
 	// with values from the response taking precedence in case of conflicts.
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
+
+	// JSONPath expression to extract the target profile from the response
+	// +kubebuilder:validation:Optional
+	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 6851ad7..5816ecd 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -185,6 +185,10 @@ spec:
                             description: JSONPath expression to extract the target
                               port from the response
                             type: string
+                          targetProfile:
+                            description: JSONPath expression to extract the target
+                              profile from the response
+                            type: string
                         required:
                         - address
                         - name
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 66bbe50..27c4774 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -37,10 +37,11 @@ const (
 // DiscoveredTarget represents a target discovered from an external source
 // before it is materialized as a Kubernetes Target CR
 type DiscoveredTarget struct {
-	Name    string
-	Address string
-	Port    int32
-	Labels  map[string]string
+	Name          string
+	Address       string
+	Port          int32
+	Labels        map[string]string
+	TargetProfile string
 }
 
 type DiscoveryEvent struct {

From 85571f47687ff99f1e7656c82f45eacba904c0fa Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:25:07 +0000
Subject: [PATCH 126/165] return if pagination field is empty

---
 internal/controller/discovery/loaders/http/pagination.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go
index 9fef778..49ddf9b 100644
--- a/internal/controller/discovery/loaders/http/pagination.go
+++ b/internal/controller/discovery/loaders/http/pagination.go
@@ -19,6 +19,10 @@ func (l *Loader) extractNextPageInfo(raw interface{}) (string, error) {
 	}
 
 	val, ok := obj[l.spec.Pagination.NextField]
+	if val == nil {
+		// No next page -> end of pagination
+		return "", nil
+	}
 	if !ok {
 		return "", fmt.Errorf("nextField '%s' not found in response", l.spec.Pagination.NextField)
 	}

From 8cdd529105240052fa533ca35d610f0cf3dfe55d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:25:28 +0000
Subject: [PATCH 127/165] refactor

---
 .../discovery/loaders/http/loader.go          | 138 ++++++++++++------
 1 file changed, 90 insertions(+), 48 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index eef202b..0aa00b3 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/go-logr/logr"
 	"github.com/google/uuid"
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -75,7 +76,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 	// helper function to fetch targets and emit discovery messages
 	fetchAndEmit := func() {
 		// Fetch targets from HTTP endpoint
-		targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client)
+		targets, err := l.fetchTargetsFromHTTPEndpoint(ctx, client, logger)
 		if err != nil {
 			logger.Error(
 				err,
@@ -84,6 +85,17 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			)
 			return
 		}
+		// TODO temporary log discovered targets
+		for _, t := range targets {
+			logger.Info(
+				"Discovered target",
+				"name", t.Name,
+				"ip", t.IP,
+				"port", t.Port,
+				"labels", t.Labels,
+				"profile", t.TargetProfile,
+			)
+		}
 
 		// Emit discovery snapshot downstream
 		snapshotID := fmt.Sprintf("%s-%s-%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name, uuid.NewString())
@@ -148,55 +160,66 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) {
 func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	ctx context.Context,
 	client *http.Client,
+	logger logr.Logger,
 ) ([]core.DiscoveredTarget, error) {
 	var allTargets []core.DiscoveredTarget
-	currentUrl := l.spec.URL
+	currentURL := l.spec.URL
 
 	for {
-		// Create HTTP request with context
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentUrl, nil)
-		if err != nil {
-			return nil, fmt.Errorf("creating HTTP request failed: %w", err)
+		// Build HTTP request
+		req, buildRequestErr := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil)
+		if buildRequestErr != nil {
+			return nil, fmt.Errorf("creating HTTP request failed: %w", buildRequestErr)
 		}
 		req.Header.Set("Accept", "application/json")
 		l.applyAuthorization(req)
 
 		// Execute HTTP request
-		resp, err := client.Do(req)
-		if err != nil {
-			return nil, fmt.Errorf("HTTP request failed: %w", err)
+		resp, requestErr := client.Do(req)
+		if requestErr != nil {
+			return nil, fmt.Errorf("HTTP request failed: %w", requestErr)
 		}
-		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
 			return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
 		}
 
-		// Decode response into raw map for pagination support
+		// Decode HTTP response
 		var raw interface{}
 		if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
+			resp.Body.Close()
 			return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
 		}
 
+		resp.Body.Close()
+
 		// Extract targets from response
-		targets, err := l.extractTargetsFromResponse(raw)
-		if err != nil {
-			return nil, err
+		targets, extractErr := l.extractTargetsFromResponse(raw, logger)
+		if extractErr != nil {
+			logger.Error(extractErr,
+				"Failed to extract targets from HTTP response",
+				"url", currentURL,
+			)
+		} else {
+			allTargets = append(allTargets, targets...)
 		}
-		allTargets = append(allTargets, targets...)
 
-		// Check for pagination
-		nextPageInfo, err := l.extractNextPageInfo(raw)
-		if err != nil {
-			return nil, err
+		// Extract pagination info
+		nextPageInfo, nextErr := l.extractNextPageInfo(raw)
+		if nextErr != nil {
+			logger.Error(nextErr, "Failed to extract next page info from HTTP response")
+			break
 		}
 		if nextPageInfo == "" {
 			break
 		}
-		nextURL, err := l.buildNextURL(currentUrl, nextPageInfo)
-		if err != nil {
-			return nil, err
+		// Build next page URL
+		nextURL, buildNextErr := l.buildNextURL(currentURL, nextPageInfo)
+		if buildNextErr != nil {
+			logger.Error(buildNextErr, "Failed to build next URL")
+			break
 		}
-		currentUrl = nextURL
+		currentURL = nextURL
 	}
 
 	return allTargets, nil
@@ -204,33 +227,44 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 
 // extractTargetsFromResponse extracts items from the response
 // and maps each item into a DiscoveredTarget
-func (l *Loader) extractTargetsFromResponse(raw interface{}) ([]core.DiscoveredTarget, error) {
+func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
 
-	switch v := raw.(type) {
-	// Top-level array response
-	case []interface{}:
-		items = v
-	// Object with itemsField containing the array
-	case map[string]interface{}:
-		if l.spec.Pagination != nil && l.spec.Pagination.ItemsField != "" {
-			// Extract items array from response using itemsField
-			val, ok := v[l.spec.Pagination.ItemsField]
-			if !ok {
-				return nil, fmt.Errorf("itemsField '%s' not found", l.spec.Pagination.ItemsField)
-			}
-
-			arr, ok := val.([]interface{})
-			if !ok {
-				return nil, fmt.Errorf("itemsField '%s' is not an array", l.spec.Pagination.ItemsField)
-			}
-
-			items = arr
-		} else {
-			return nil, fmt.Errorf("response is an object but no itemsField specified for TargetSource %s/%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
+	if l.spec.ItemsField != "" {
+		obj, ok := raw.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf(
+				"invalid HTTP response: expected JSON object when itemsField '%s' is configured (e.g. {\"%s\": [...]})",
+				l.spec.ItemsField,
+				l.spec.ItemsField,
+			)
 		}
-	default:
-		return nil, fmt.Errorf("unexpected response format")
+
+		results, ok := obj[l.spec.ItemsField]
+		if !ok {
+			return nil, fmt.Errorf(
+				"invalid HTTP response: itemsField '%s' not found. ensure the API response contains this field (e.g. {\"%s\": [...]})",
+				l.spec.ItemsField,
+				l.spec.ItemsField,
+			)
+		}
+
+		array, ok := results.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf(
+				"invalid HTTP response: itemsField '%s' must be an array of objects (e.g. {\"%s\": [...]})",
+				l.spec.ItemsField,
+				l.spec.ItemsField,
+			)
+		}
+
+		items = array
+	} else {
+		array, ok := raw.([]interface{})
+		if !ok {
+			return nil, fmt.Errorf("invalid HTTP response: expected a JSON array because itemsField is not set (e.g. [{...}, {...}])")
+		}
+		items = array
 	}
 
 	// Map items to targets
@@ -238,12 +272,20 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}) ([]core.DiscoveredT
 	for _, item := range items {
 		obj, ok := item.(map[string]interface{})
 		if !ok {
+			logger.Error(fmt.Errorf("invalid target format"),
+				"Failed to convert target to map",
+				"item", item,
+			)
 			continue
 		}
 
 		target, err := l.mapItem(obj)
 		if err != nil {
-			return nil, err
+			logger.Error(err,
+				"Failed to map target",
+				"item", obj,
+			)
+			continue
 		}
 
 		targets = append(targets, target)

From f94e8d15290cfd05bd0ea395fd0233e2f0181309 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:25:46 +0000
Subject: [PATCH 128/165] use itemsField independent of pagination

---
 api/v1alpha1/targetsource_types.go             | 13 ++++++++-----
 .../operator.gnmic.dev_targetsources.yaml      | 18 ++++++++++--------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index c83e5e2..e3a5364 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -82,9 +82,17 @@ type HTTPConfig struct {
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
 	// Optional TLS configuration for connecting to the HTTP endpoint
+	// If it is an HTTP endpoint, this will be ignored
 	// +kubebuilder:validation:Optional
 	TLS *ClientTLSConfig `json:"tls,omitempty"`
 
+	// Field name in the JSON response that contains the list of items (targets).
+	// Must refer to a top-level key in the response object.
+	// If not specified, the entire response is expected to be a list of items.
+	// Example: "results"
+	// +kubebuilder:validation:Optional
+	ItemsField string `json:"itemsField,omitempty"`
+
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
@@ -173,11 +181,6 @@ type TokenAuthSpec struct {
 
 // PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
-	// Field name in the JSON response that contains the list of items (targets).
-	// Must refer to a top-level key in the response object.
-	// Example: "results"
-	ItemsField string `json:"itemsField,omitempty"`
-
 	// Field name in the JSON response that contains the next page reference.
 	// The value can be either:
 	// - a full URL (used directly for the next request), or
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index b4ecb44..83db155 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -161,6 +161,13 @@ spec:
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
+                      itemsField:
+                        description: |-
+                          Field name in the JSON response that contains the list of items (targets).
+                          Must refer to a top-level key in the response object.
+                          If not specified, the entire response is expected to be a list of items.
+                          Example: "results"
+                        type: string
                       mapping:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
@@ -197,12 +204,6 @@ spec:
                         description: Optional pagination configuration for parsing
                           responses from the HTTP endpoint
                         properties:
-                          itemsField:
-                            description: |-
-                              Field name in the JSON response that contains the list of items (targets).
-                              Must refer to a top-level key in the response object.
-                              Example: "results"
-                            type: string
                           nextField:
                             description: |-
                               Field name in the JSON response that contains the next page reference.
@@ -219,8 +220,9 @@ spec:
                         description: Optional timeout for HTTP requests to the endpoint
                         type: string
                       tls:
-                        description: Optional TLS configuration for connecting to
-                          the HTTP endpoint
+                        description: |-
+                          Optional TLS configuration for connecting to the HTTP endpoint
+                          If it is an HTTP endpoint, this will be ignored
                         properties:
                           caBundle:
                             description: |-

From 29f201fe664084b9801c4caeaf252e104a5984d5 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:27:00 +0000
Subject: [PATCH 129/165] rename ItemsField to TargetsField

---
 api/v1alpha1/targetsource_types.go               |  2 +-
 .../bases/operator.gnmic.dev_targetsources.yaml  | 14 +++++++-------
 .../controller/discovery/loaders/http/loader.go  | 16 ++++++++--------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index e3a5364..e315510 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -91,7 +91,7 @@ type HTTPConfig struct {
 	// If not specified, the entire response is expected to be a list of items.
 	// Example: "results"
 	// +kubebuilder:validation:Optional
-	ItemsField string `json:"itemsField,omitempty"`
+	TargetsField string `json:"targetsField,omitempty"`
 
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 83db155..2e218c7 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -161,13 +161,6 @@ spec:
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
-                      itemsField:
-                        description: |-
-                          Field name in the JSON response that contains the list of items (targets).
-                          Must refer to a top-level key in the response object.
-                          If not specified, the entire response is expected to be a list of items.
-                          Example: "results"
-                        type: string
                       mapping:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
@@ -215,6 +208,13 @@ spec:
                               Example: "next" or "nextToken"
                             type: string
                         type: object
+                      targetsField:
+                        description: |-
+                          Field name in the JSON response that contains the list of items (targets).
+                          Must refer to a top-level key in the response object.
+                          If not specified, the entire response is expected to be a list of items.
+                          Example: "results"
+                        type: string
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 0aa00b3..faaf7c5 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -230,22 +230,22 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
 
-	if l.spec.ItemsField != "" {
+	if l.spec.TargetsField != "" {
 		obj, ok := raw.(map[string]interface{})
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: expected JSON object when itemsField '%s' is configured (e.g. {\"%s\": [...]})",
-				l.spec.ItemsField,
-				l.spec.ItemsField,
+				l.spec.TargetsField,
+				l.spec.TargetsField,
 			)
 		}
 
-		results, ok := obj[l.spec.ItemsField]
+		results, ok := obj[l.spec.TargetsField]
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: itemsField '%s' not found. ensure the API response contains this field (e.g. {\"%s\": [...]})",
-				l.spec.ItemsField,
-				l.spec.ItemsField,
+				l.spec.TargetsField,
+				l.spec.TargetsField,
 			)
 		}
 
@@ -253,8 +253,8 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger)
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: itemsField '%s' must be an array of objects (e.g. {\"%s\": [...]})",
-				l.spec.ItemsField,
-				l.spec.ItemsField,
+				l.spec.TargetsField,
+				l.spec.TargetsField,
 			)
 		}
 

From 4d008c28ca2b07285f90e15b3eac8e79c314eabb Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:55:53 +0000
Subject: [PATCH 130/165] make tls caBundle more user friendly

---
 api/v1alpha1/targetsource_types.go                     | 4 ++--
 api/v1alpha1/zz_generated.deepcopy.go                  | 5 -----
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 3 +--
 internal/controller/discovery/loaders.go               | 3 +--
 internal/controller/discovery/loaders/http/loader.go   | 2 +-
 5 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index e315510..9baad20 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -108,12 +108,12 @@ type ClientTLSConfig struct {
 	// +kubebuilder:default:=false
 	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
 
-	// Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+	// Bundle of PEM CAs which will be used to validate the certificate
 	// chain presented by the Provider. Only used if using HTTPS to connect to Provider and
 	// ignored for HTTP connections.
 	// Mutually exclusive with CABundleSecretRef.
 	// +optional
-	CABundle []byte `json:"caBundle,omitempty"`
+	CABundle string `json:"caBundle,omitempty"`
 
 	// Reference to a Secret containing a bundle of PEM-encoded CAs to use when
 	// verifying the certificate chain presented by the Provider when using HTTPS.
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index dc4b784..f055e7d 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -94,11 +94,6 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
-	if in.CABundle != nil {
-		in, out := &in.CABundle, &out.CABundle
-		*out = make([]byte, len(*in))
-		copy(*out, *in)
-	}
 	if in.CABundleSecretRef != nil {
 		in, out := &in.CABundleSecretRef, &out.CABundleSecretRef
 		*out = new(v1.SecretKeySelector)
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 2e218c7..5e361cf 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -226,11 +226,10 @@ spec:
                         properties:
                           caBundle:
                             description: |-
-                              Base64-encoded bundle of PEM CAs which will be used to validate the certificate
+                              Bundle of PEM CAs which will be used to validate the certificate
                               chain presented by the Provider. Only used if using HTTPS to connect to Provider and
                               ignored for HTTP connections.
                               Mutually exclusive with CABundleSecretRef.
-                            format: byte
                             type: string
                           caBundleSecretRef:
                             description: |-
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 42ab588..f0c19cf 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -162,8 +162,7 @@ func resolveTLSIntoSpec(
 		if err != nil {
 			return err
 		}
-		// convert string to []byte
-		tls.CABundle = []byte(values[key])
+		tls.CABundle = (values[key])
 	}
 
 	return nil
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index faaf7c5..7d6caca 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -141,7 +141,7 @@ func (l *Loader) buildHTTPClient() (*http.Client, error) {
 	// If a CA bundle is provided, add it to the TLS config
 	if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 {
 		certPool := x509.NewCertPool()
-		if ok := certPool.AppendCertsFromPEM(l.spec.TLS.CABundle); !ok {
+		if ok := certPool.AppendCertsFromPEM([]byte(l.spec.TLS.CABundle)); !ok {
 			return nil, fmt.Errorf("Failed to parse CA bundle for TargetSource %s/%s\n", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
 		}
 		tlsConfig.RootCAs = certPool

From a69abddf55b2e6600a2ac630002fe6b52f5a7ea4 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 10:27:44 +0000
Subject: [PATCH 131/165] increase default poll interval

---
 api/v1alpha1/targetsource_types.go                     | 4 ++--
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 9baad20..a15ca70 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -71,8 +71,8 @@ type HTTPConfig struct {
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
 
 	// Optional interval for polling the HTTP endpoint for targets
-	// TODO: increase default value
-	// +kubebuilder:default="30s"
+	// TODO: add to docs
+	// +kubebuilder:default="6h"
 	// +kubebuilder:validation:Optional
 	PollInterval *metav1.Duration `json:"interval,omitempty"`
 
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 5e361cf..a6e0877 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -157,7 +157,7 @@ spec:
                           rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
-                        default: 30s
+                        default: 6h
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string

From 4342d66f4b5774c1f38a6dc837ac5877c6dec5ef Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 12:25:16 +0000
Subject: [PATCH 132/165] refactor fetchTargetsFromHTTPEndpoint

---
 .../discovery/loaders/http/loader.go          | 119 ++++++++++++------
 1 file changed, 78 insertions(+), 41 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 7d6caca..543b7d8 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -166,37 +166,18 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	currentURL := l.spec.URL
 
 	for {
-		// Build HTTP request
-		req, buildRequestErr := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil)
-		if buildRequestErr != nil {
-			return nil, fmt.Errorf("creating HTTP request failed: %w", buildRequestErr)
-		}
-		req.Header.Set("Accept", "application/json")
-		l.applyAuthorization(req)
-
-		// Execute HTTP request
-		resp, requestErr := client.Do(req)
-		if requestErr != nil {
-			return nil, fmt.Errorf("HTTP request failed: %w", requestErr)
-		}
-		if resp.StatusCode != http.StatusOK {
-			resp.Body.Close()
-			return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
-		}
-
-		// Decode HTTP response
-		var raw interface{}
-		if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
-			resp.Body.Close()
-			return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
+		raw, err := l.fetchPage(ctx, client, currentURL)
+		if err != nil {
+			logger.Error(err,
+				"Failed to fetch page from HTTP endpoint",
+				"url", currentURL,
+			)
+			break
 		}
 
-		resp.Body.Close()
-
 		// Extract targets from response
-		targets, extractErr := l.extractTargetsFromResponse(raw, logger)
-		if extractErr != nil {
-			logger.Error(extractErr,
+		if targets, err := l.extractTargetsFromResponse(raw, logger); err != nil {
+			logger.Error(err,
 				"Failed to extract targets from HTTP response",
 				"url", currentURL,
 			)
@@ -204,19 +185,9 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 			allTargets = append(allTargets, targets...)
 		}
 
-		// Extract pagination info
-		nextPageInfo, nextErr := l.extractNextPageInfo(raw)
-		if nextErr != nil {
-			logger.Error(nextErr, "Failed to extract next page info from HTTP response")
-			break
-		}
-		if nextPageInfo == "" {
-			break
-		}
-		// Build next page URL
-		nextURL, buildNextErr := l.buildNextURL(currentURL, nextPageInfo)
-		if buildNextErr != nil {
-			logger.Error(buildNextErr, "Failed to build next URL")
+		// Pagination
+		nextURL, stop := l.getNextURL(raw, currentURL, logger)
+		if stop {
 			break
 		}
 		currentURL = nextURL
@@ -225,6 +196,36 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	return allTargets, nil
 }
 
+// fetchPage performs an HTTP GET request to the specified URL and decodes the JSON response
+// and returns the raw response as an interface{}
+func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string) (interface{}, error) {
+	// Build HTTP request
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
+	}
+	req.Header.Set("Accept", "application/json")
+	l.applyAuthorization(req)
+
+	// Execute HTTP request
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("HTTP request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode)
+	}
+
+	// Decode HTTP response
+	var raw interface{}
+	if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
+		return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
+	}
+
+	return raw, nil
+}
+
 // extractTargetsFromResponse extracts items from the response
 // and maps each item into a DiscoveredTarget
 func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
@@ -293,3 +294,39 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger)
 
 	return targets, nil
 }
+
+// getNextURL determines the next page URL
+// Returns:
+// - nextURL: next request
+// - stop: whether to terminate loop
+func (l *Loader) getNextURL(
+	raw interface{},
+	currentURL string,
+	logger logr.Logger,
+) (string, bool) {
+	// Extract pagination info
+	nextPageInfo, err := l.extractNextPageInfo(raw)
+	if err != nil {
+		logger.Error(err,
+			"Failed to extract next page info from HTTP response",
+			"url", currentURL,
+		)
+		return "", true
+	}
+
+	if nextPageInfo == "" {
+		return "", true
+	}
+
+	// Build next page URL
+	nextURL, err := l.buildNextURL(currentURL, nextPageInfo)
+	if err != nil {
+		logger.Error(err,
+			"Failed to build next URL",
+			"url", currentURL,
+		)
+		return "", true
+	}
+
+	return nextURL, false
+}

From 0d744ba61e263f1b8ad08470ca3b4e42b1d17b33 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 13:47:23 -0600
Subject: [PATCH 133/165] added port + targetProfile handling to mapper

---
 internal/controller/discovery/mapper.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go
index bc42531..4690fd1 100644
--- a/internal/controller/discovery/mapper.go
+++ b/internal/controller/discovery/mapper.go
@@ -1,6 +1,7 @@
 package discovery
 
 import (
+	"fmt"
 	"maps"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,10 +21,19 @@ func generateTargetResource(d core.DiscoveredTarget, ts *gnmicv1alpha1.TargetSou
 		},
 	}
 
-	// Add Address from DiscoveredTarget
-	t.Spec.Address = d.Address
-	// Add default Target Profile from the TargetSource Spec TargetProfile
-	t.Spec.Profile = ts.Spec.TargetProfile
+	// Add Address + Port from DiscoveredTarget or use TargetSource.spec.targetPort
+	targetPort := ts.Spec.TargetPort
+	if d.Port != 0 {
+		targetPort = d.Port
+	}
+	t.Spec.Address = fmt.Sprintf("%s:%d", d.Address, targetPort)
+
+	// Add discovered Target Profile or use TargetSource.spec.targetProfile
+	targetProfile := ts.Spec.TargetProfile
+	if d.TargetProfile != "" {
+		targetProfile = d.TargetProfile
+	}
+	t.Spec.Profile = targetProfile
 
 	// Copy TargetLabels from TargetSource Spec & DiscoveredTarget. Discovered labels take precedence over TargetSource labels.
 	maps.Copy(t.Labels, ts.Spec.TargetLabels)

From 5b612f2d6ea32ebfd8b3c4567a23a48c47fe1c24 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 14:09:55 -0600
Subject: [PATCH 134/165] added webhook spec for authorization

---
 api/v1alpha1/targetsource_types.go            | 40 ++++++--
 api/v1alpha1/zz_generated.deepcopy.go         | 90 ++++++++++++++++++
 .../operator.gnmic.dev_targetsources.yaml     | 91 +++++++++++++++++--
 3 files changed, 209 insertions(+), 12 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index e8b4c4c..7d96c0d 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -60,12 +60,6 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	URL string `json:"url,omitempty"`
 
-	// If true, the loader will accept pushed target updates to the controller endpoint
-	// The endpoint will be /{namespace}/{targetsource}/<todo>
-	// +kubebuilder:default=false
-	// +kubebuilder:validation:Optional
-	AcceptPush bool `json:"acceptPush,omitempty"`
-
 	// Optional authorization configuration for accessing the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
@@ -92,6 +86,10 @@ type HTTPConfig struct {
 	// Optional mapping configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
+
+	// Optional configuration to enable webhooks
+	// +kubebuilder:validation:Optional
+	Webhook *WebhookSpec `json:"webhook,omitempty"`
 }
 
 // +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive"
@@ -208,6 +206,36 @@ type ResponseMappingSpec struct {
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
+// WebhookSpec defines the settings for event-based update mechanism (i.e. push-based)
+type WebhookSpec struct {
+	// +kubebuilder:default=false
+	Enabled bool `json:"enabled"`
+
+	// +kubebuilder:validation:Optional
+	Auth *WebhookAuthSpec `json:"auth,omitempty"`
+}
+
+// +kubebuilder:validation:ExactlyOneOf=bearer;signature
+type WebhookAuthSpec struct {
+	Bearer    *WebhookBearerAuthSpec    `json:"bearer,omitempty"`
+	Signature *WebhookSignatureAuthSpec `json:"signature,omitempty"`
+}
+
+type WebhookBearerAuthSpec struct {
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+type WebhookSignatureAuthSpec struct {
+	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
+
+	// Header containing the signature
+	Header string `json:"header,omitempty"`
+
+	// +kubebuilder:default="sha256"
+	// +kubebuilder:validation:Enum=sha1;sha256;sha512
+	Algorithm string `json:"algorithm,omitempty"`
+}
+
 // TargetSourceStatus defines the observed state of TargetSource
 type TargetSourceStatus struct {
 	Status             string      `json:"status,omitempty"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index dc4b784..9d59d3c 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -358,6 +358,11 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 		*out = new(ResponseMappingSpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Webhook != nil {
+		in, out := &in.Webhook, &out.Webhook
+		*out = new(WebhookSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConfig.
@@ -1614,3 +1619,88 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookAuthSpec) DeepCopyInto(out *WebhookAuthSpec) {
+	*out = *in
+	if in.Bearer != nil {
+		in, out := &in.Bearer, &out.Bearer
+		*out = new(WebhookBearerAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Signature != nil {
+		in, out := &in.Signature, &out.Signature
+		*out = new(WebhookSignatureAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookAuthSpec.
+func (in *WebhookAuthSpec) DeepCopy() *WebhookAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookBearerAuthSpec) DeepCopyInto(out *WebhookBearerAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookBearerAuthSpec.
+func (in *WebhookBearerAuthSpec) DeepCopy() *WebhookBearerAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookBearerAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookSignatureAuthSpec) DeepCopyInto(out *WebhookSignatureAuthSpec) {
+	*out = *in
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSignatureAuthSpec.
+func (in *WebhookSignatureAuthSpec) DeepCopy() *WebhookSignatureAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookSignatureAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
+	*out = *in
+	if in.Auth != nil {
+		in, out := &in.Auth, &out.Auth
+		*out = new(WebhookAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
+func (in *WebhookSpec) DeepCopy() *WebhookSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookSpec)
+	in.DeepCopyInto(out)
+	return out
+}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d6def2b..81c8e20 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -47,12 +47,6 @@ spec:
                   http:
                     description: HTTP defines the configuration for a HTTP provider
                     properties:
-                      acceptPush:
-                        default: false
-                        description: |-
-                          If true, the loader will accept pushed target updates to the controller endpoint
-                          The endpoint will be /{namespace}/{targetsource}/<todo>
-                        type: boolean
                       authorization:
                         description: Optional authorization configuration for accessing
                           the HTTP endpoint
@@ -264,6 +258,91 @@ spec:
                           URL of the HTTP endpoint to pull targets from
                           If defined, the loader will periodically poll this endpoint for targets
                         type: string
+                      webhook:
+                        description: Optional configuration to enable webhooks
+                        properties:
+                          auth:
+                            properties:
+                              bearer:
+                                properties:
+                                  tokenSecretRef:
+                                    description: SecretKeySelector selects a key of
+                                      a Secret.
+                                    properties:
+                                      key:
+                                        description: The key of the secret to select
+                                          from.  Must be a valid secret key.
+                                        type: string
+                                      name:
+                                        default: ""
+                                        description: |-
+                                          Name of the referent.
+                                          This field is effectively required, but due to backwards compatibility is
+                                          allowed to be empty. Instances of this type with an empty value here are
+                                          almost certainly wrong.
+                                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                        type: string
+                                      optional:
+                                        description: Specify whether the Secret or
+                                          its key must be defined
+                                        type: boolean
+                                    required:
+                                    - key
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                type: object
+                              signature:
+                                properties:
+                                  algorithm:
+                                    default: sha256
+                                    enum:
+                                    - sha1
+                                    - sha256
+                                    - sha512
+                                    type: string
+                                  header:
+                                    description: Header containing the signature
+                                    type: string
+                                  secretRef:
+                                    description: SecretKeySelector selects a key of
+                                      a Secret.
+                                    properties:
+                                      key:
+                                        description: The key of the secret to select
+                                          from.  Must be a valid secret key.
+                                        type: string
+                                      name:
+                                        default: ""
+                                        description: |-
+                                          Name of the referent.
+                                          This field is effectively required, but due to backwards compatibility is
+                                          allowed to be empty. Instances of this type with an empty value here are
+                                          almost certainly wrong.
+                                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                        type: string
+                                      optional:
+                                        description: Specify whether the Secret or
+                                          its key must be defined
+                                        type: boolean
+                                    required:
+                                    - key
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                required:
+                                - secretRef
+                                type: object
+                            type: object
+                            x-kubernetes-validations:
+                            - message: exactly one of the fields in [bearer signature]
+                                must be set
+                              rule: '[has(self.bearer),has(self.signature)].filter(x,x==true).size()
+                                == 1'
+                          enabled:
+                            default: false
+                            type: boolean
+                        required:
+                        - enabled
+                        type: object
                     type: object
                     x-kubernetes-validations:
                     - message: at least one of the fields in [url acceptPush] must

From 58b77d4929be55d594894c1f7fbaf648605cf1c2 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 14:11:23 -0600
Subject: [PATCH 135/165] removed inline credential fields

---
 api/v1alpha1/targetsource_types.go            | 15 -----------
 .../operator.gnmic.dev_targetsources.yaml     | 27 -------------------
 2 files changed, 42 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 7d96c0d..6e72439 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -123,34 +123,19 @@ type AuthorizationSpec struct {
 }
 
 // BasicAuthSpec defines the configuration for basic authentication
-// Enforce EITHER inline creds OR secret ref
-// +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
-	// Username for basic auth
-	// Mutually exclusive with CredentialsSecretRef.
-	Username string `json:"username,omitempty"`
-	// Password for basic auth
-	// Mutually exclusive with CredentialsSecretRef.
-	Password string `json:"password,omitempty"`
-
 	// Reference to a Secret containing "username" and "password" keys to use for
 	// basic authentication when connecting to the Provider.
-	// Mutually exclusive with Username and Password.
 	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
 // TokenAuthSpec defines the configuration for token-based authentication
-// +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
 type TokenAuthSpec struct {
 	// Scheme for the token, e.g. "Bearer"
 	// +kubebuilder:validation:MinLength=1
 	Scheme string `json:"scheme"`
-	// Token value for authentication
-	// Mutually exclusive with TokenSecretRef.
-	Token string `json:"token,omitempty"`
 	// Reference to a Secret containing a key with the token value to use for
 	// authentication when connecting to the Provider.
-	// Mutually exclusive with Token.
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 81c8e20..d333a9e 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -58,7 +58,6 @@ spec:
                                 description: |-
                                   Reference to a Secret containing "username" and "password" keys to use for
                                   basic authentication when connecting to the Provider.
-                                  Mutually exclusive with Username and Password.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -81,23 +80,7 @@ spec:
                                 - key
                                 type: object
                                 x-kubernetes-map-type: atomic
-                              password:
-                                description: |-
-                                  Password for basic auth
-                                  Mutually exclusive with CredentialsSecretRef.
-                                type: string
-                              username:
-                                description: |-
-                                  Username for basic auth
-                                  Mutually exclusive with CredentialsSecretRef.
-                                type: string
                             type: object
-                            x-kubernetes-validations:
-                            - message: either credentialsSecretRef OR both username
-                                and password must be set, but not a mix
-                              rule: (has(self.credentialsSecretRef) && !has(self.username)
-                                && !has(self.password)) || (!has(self.credentialsSecretRef)
-                                && has(self.username) && has(self.password))
                           token:
                             description: Token-based authentication configuration
                             properties:
@@ -105,16 +88,10 @@ spec:
                                 description: Scheme for the token, e.g. "Bearer"
                                 minLength: 1
                                 type: string
-                              token:
-                                description: |-
-                                  Token value for authentication
-                                  Mutually exclusive with TokenSecretRef.
-                                type: string
                               tokenSecretRef:
                                 description: |-
                                   Reference to a Secret containing a key with the token value to use for
                                   authentication when connecting to the Provider.
-                                  Mutually exclusive with Token.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -140,10 +117,6 @@ spec:
                             required:
                             - scheme
                             type: object
-                            x-kubernetes-validations:
-                            - message: either token or tokenSecretRef must be set,
-                                but not both
-                              rule: has(self.token) != has(self.tokenSecretRef)
                         type: object
                         x-kubernetes-validations:
                         - message: exactly one of the fields in [basic token] must

From aa023abdd11feb748b9d91c5627f116613fa2cb3 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:25:46 +0000
Subject: [PATCH 136/165] use itemsField independent of pagination

---
 api/v1alpha1/targetsource_types.go            | 21 +++++++++++++------
 .../operator.gnmic.dev_targetsources.yaml     | 17 ++++++++-------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 6e72439..63bd535 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -76,9 +76,17 @@ type HTTPConfig struct {
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
 	// Optional TLS configuration for connecting to the HTTP endpoint
+	// If it is an HTTP endpoint, this will be ignored
 	// +kubebuilder:validation:Optional
 	TLS *ClientTLSConfig `json:"tls,omitempty"`
 
+	// Field name in the JSON response that contains the list of items (targets).
+	// Must refer to a top-level key in the response object.
+	// If not specified, the entire response is expected to be a list of items.
+	// Example: "results"
+	// +kubebuilder:validation:Optional
+	ItemsField string `json:"itemsField,omitempty"`
+
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
@@ -156,12 +164,13 @@ type TokenAuthSpec struct {
 
 // PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
-	// JSONPath-style expression to extract the list of targets from the response
-	// Example: "results"
-	ItemsField string `json:"itemsField,omitempty"`
-
-	// JSONPath-style expression to extract the next page token or URL from the response for pagination
-	// Example: "next"
+	// Field name in the JSON response that contains the next page reference.
+	// The value can be either:
+	// - a full URL (used directly for the next request), or
+	// - a pagination token (appended as a query parameter using this field name as the key).
+	//
+	// Must refer to a top-level key in the response object.
+	// Example: "next" or "nextToken"
 	NextField string `json:"nextField,omitempty"`
 }
 
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d333a9e..f3a66f2 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -128,6 +128,13 @@ spec:
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
+                      itemsField:
+                        description: |-
+                          Field name in the JSON response that contains the list of items (targets).
+                          Must refer to a top-level key in the response object.
+                          If not specified, the entire response is expected to be a list of items.
+                          Example: "results"
+                        type: string
                       mapping:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
@@ -164,11 +171,6 @@ spec:
                         description: Optional pagination configuration for parsing
                           responses from the HTTP endpoint
                         properties:
-                          itemsField:
-                            description: |-
-                              JSONPath-style expression to extract the list of targets from the response
-                              Example: "results"
-                            type: string
                           nextField:
                             description: |-
                               JSONPath-style expression to extract the next page token or URL from the response for pagination
@@ -180,8 +182,9 @@ spec:
                         description: Optional timeout for HTTP requests to the endpoint
                         type: string
                       tls:
-                        description: Optional TLS configuration for connecting to
-                          the HTTP endpoint
+                        description: |-
+                          Optional TLS configuration for connecting to the HTTP endpoint
+                          If it is an HTTP endpoint, this will be ignored
                         properties:
                           caBundle:
                             description: |-

From fc6008da9ded3f1a5edff8150519bbaf557bf26d Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Thu, 21 May 2026 09:27:00 +0000
Subject: [PATCH 137/165] rename ItemsField to TargetsField

---
 api/v1alpha1/targetsource_types.go                 |  2 +-
 .../bases/operator.gnmic.dev_targetsources.yaml    | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 63bd535..d0045be 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -85,7 +85,7 @@ type HTTPConfig struct {
 	// If not specified, the entire response is expected to be a list of items.
 	// Example: "results"
 	// +kubebuilder:validation:Optional
-	ItemsField string `json:"itemsField,omitempty"`
+	TargetsField string `json:"targetsField,omitempty"`
 
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index f3a66f2..08615c8 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -128,13 +128,6 @@ spec:
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
-                      itemsField:
-                        description: |-
-                          Field name in the JSON response that contains the list of items (targets).
-                          Must refer to a top-level key in the response object.
-                          If not specified, the entire response is expected to be a list of items.
-                          Example: "results"
-                        type: string
                       mapping:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
@@ -177,6 +170,13 @@ spec:
                               Example: "next"
                             type: string
                         type: object
+                      targetsField:
+                        description: |-
+                          Field name in the JSON response that contains the list of items (targets).
+                          Must refer to a top-level key in the response object.
+                          If not specified, the entire response is expected to be a list of items.
+                          Example: "results"
+                        type: string
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint

From 0c1b414b609d5b2047b10fa125d0f2fd2658acd6 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 14:16:01 -0600
Subject: [PATCH 138/165] changed assertion to new webhook spec

---
 internal/controller/discovery/loaders.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index c4ebe78..dce3928 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -13,7 +13,7 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		cfg.AcceptPush = spec.Provider.HTTP.AcceptPush
+		cfg.AcceptPush = spec.Provider.HTTP.Webhook.Enabled
 		return http.New(*cfg), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)

From 7e06e146f98b266638da1a3f034bf2b41e4bb962 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 14:18:04 -0600
Subject: [PATCH 139/165] moved TargetsField to ResponseMappingSpec

---
 api/v1alpha1/targetsource_types.go            | 26 ++++++-------
 .../operator.gnmic.dev_targetsources.yaml     | 37 ++++++++++---------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index d0045be..0ca4181 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -80,13 +80,6 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	TLS *ClientTLSConfig `json:"tls,omitempty"`
 
-	// Field name in the JSON response that contains the list of items (targets).
-	// Must refer to a top-level key in the response object.
-	// If not specified, the entire response is expected to be a list of items.
-	// Example: "results"
-	// +kubebuilder:validation:Optional
-	TargetsField string `json:"targetsField,omitempty"`
-
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
@@ -177,25 +170,30 @@ type PaginationSpec struct {
 // JSONPath-style expressions to extract target fields from the response
 // and map them to the corresponding Target fields.
 type ResponseMappingSpec struct {
-	// JSONPath expression to extract the target name from the response
+	// Field name in the JSON response that contains the list of items (targets).
+	// If not specified, the entire response is expected to be a list of items.
+	// All subsequent fields are specified relative to this field
+	// Example: "results"
+	// +kubebuilder:validation:Optional
+	TargetsField string `json:"targetsField,omitempty"`
+
+	// JSONPath expression to extract the target name from the response list
 	// +kubebuilder:validation:Required
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target address from the response
+	// JSONPath expression to extract the target address from the response list
 	// +kubebuilder:validation:Required
 	Address string `json:"address"`
 
-	// JSONPath expression to extract the target port from the response
+	// JSONPath expression to extract the target port from the response list
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// JSONPath expression to extract the target labels from the response
-	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
-	// with values from the response taking precedence in case of conflicts.
+	// JSONPath expression to extract the target labels from the response list
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
 
-	// JSONPath expression to extract the target profile from the response
+	// JSONPath expression to extract the target profile from the response list
 	// +kubebuilder:validation:Optional
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 08615c8..e68385d 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -134,27 +134,32 @@ spec:
                         properties:
                           address:
                             description: JSONPath expression to extract the target
-                              address from the response
+                              address from the response list
                             type: string
                           labels:
                             additionalProperties:
                               type: string
-                            description: |-
-                              JSONPath expression to extract the target labels from the response
-                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
-                              with values from the response taking precedence in case of conflicts.
+                            description: JSONPath expression to extract the target
+                              labels from the response list
                             type: object
                           name:
                             description: JSONPath expression to extract the target
-                              name from the response
+                              name from the response list
                             type: string
                           port:
                             description: JSONPath expression to extract the target
-                              port from the response
+                              port from the response list
                             type: string
                           targetProfile:
                             description: JSONPath expression to extract the target
-                              profile from the response
+                              profile from the response list
+                            type: string
+                          targetsField:
+                            description: |-
+                              Field name in the JSON response that contains the list of items (targets).
+                              If not specified, the entire response is expected to be a list of items.
+                              All subsequent fields are specified relative to this field
+                              Example: "results"
                             type: string
                         required:
                         - address
@@ -166,17 +171,15 @@ spec:
                         properties:
                           nextField:
                             description: |-
-                              JSONPath-style expression to extract the next page token or URL from the response for pagination
-                              Example: "next"
+                              Field name in the JSON response that contains the next page reference.
+                              The value can be either:
+                              - a full URL (used directly for the next request), or
+                              - a pagination token (appended as a query parameter using this field name as the key).
+
+                              Must refer to a top-level key in the response object.
+                              Example: "next" or "nextToken"
                             type: string
                         type: object
-                      targetsField:
-                        description: |-
-                          Field name in the JSON response that contains the list of items (targets).
-                          Must refer to a top-level key in the response object.
-                          If not specified, the entire response is expected to be a list of items.
-                          Example: "results"
-                        type: string
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint

From 421d40ff3baf3c8c178000769beb968f76fcc03e Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 14:22:38 -0600
Subject: [PATCH 140/165] changed validation rule for url or webhook

---
 api/v1alpha1/targetsource_types.go                     | 2 +-
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 0ca4181..e5776ca 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -53,7 +53,7 @@ type ProviderSpec struct {
 }
 
 // HTTPConfig defines the configuration for the HTTP provider
-// +kubebuilder:validation:AtLeastOneOf=url;acceptPush
+// +kubebuilder:validation:AtLeastOneOf=url;webhook
 type HTTPConfig struct {
 	// URL of the HTTP endpoint to pull targets from
 	// If defined, the loader will periodically poll this endpoint for targets
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index e68385d..0aae289 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -324,9 +324,9 @@ spec:
                         type: object
                     type: object
                     x-kubernetes-validations:
-                    - message: at least one of the fields in [url acceptPush] must
-                        be set
-                      rule: '[has(self.url),has(self.acceptPush)].filter(x,x==true).size()
+                    - message: at least one of the fields in [url webhook] must be
+                        set
+                      rule: '[has(self.url),has(self.webhook)].filter(x,x==true).size()
                         >= 1'
                 type: object
                 x-kubernetes-validations:

From 85c46b5ec6e7ae599262968b58ade8795baca4c0 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 16:25:49 -0600
Subject: [PATCH 141/165] reworked kubebuilder validations

---
 api/v1alpha1/targetsource_types.go                  | 13 ++++++++-----
 .../crd/bases/operator.gnmic.dev_targetsources.yaml |  2 ++
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index e5776ca..9febc06 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -53,7 +53,7 @@ type ProviderSpec struct {
 }
 
 // HTTPConfig defines the configuration for the HTTP provider
-// +kubebuilder:validation:AtLeastOneOf=url;webhook
+// +kubebuilder:validation:AtLeastOneOf:=url;webhook
 type HTTPConfig struct {
 	// URL of the HTTP endpoint to pull targets from
 	// If defined, the loader will periodically poll this endpoint for targets
@@ -207,25 +207,28 @@ type WebhookSpec struct {
 	Auth *WebhookAuthSpec `json:"auth,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=bearer;signature
+// +kubebuilder:validation:ExactlyOneOf:=bearer;signature
 type WebhookAuthSpec struct {
 	Bearer    *WebhookBearerAuthSpec    `json:"bearer,omitempty"`
 	Signature *WebhookSignatureAuthSpec `json:"signature,omitempty"`
 }
 
+// +kubebuilder:validation:Required
 type WebhookBearerAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
+// +kubebuilder:validation:Required
 type WebhookSignatureAuthSpec struct {
 	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
 
 	// Header containing the signature
-	Header string `json:"header,omitempty"`
+	// +kubebuilder:validation:MinLength=1
+	Header string `json:"header"`
 
-	// +kubebuilder:default="sha256"
+	// +kubebuilder:default="sha512"
 	// +kubebuilder:validation:Enum=sha1;sha256;sha512
-	Algorithm string `json:"algorithm,omitempty"`
+	Algorithm string `json:"algorithm"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 0aae289..63ae646 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -281,6 +281,7 @@ spec:
                                     type: string
                                   header:
                                     description: Header containing the signature
+                                    minLength: 1
                                     type: string
                                   secretRef:
                                     description: SecretKeySelector selects a key of
@@ -308,6 +309,7 @@ spec:
                                     type: object
                                     x-kubernetes-map-type: atomic
                                 required:
+                                - header
                                 - secretRef
                                 type: object
                             type: object

From e7a62dff98fd03d420af63defa146e38540cc6a4 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Thu, 21 May 2026 16:33:20 -0600
Subject: [PATCH 142/165] generated manifests

---
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 63ae646..7816afc 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -273,7 +273,7 @@ spec:
                               signature:
                                 properties:
                                   algorithm:
-                                    default: sha256
+                                    default: sha512
                                     enum:
                                     - sha1
                                     - sha256
@@ -309,6 +309,7 @@ spec:
                                     type: object
                                     x-kubernetes-map-type: atomic
                                 required:
+                                - algorithm
                                 - header
                                 - secretRef
                                 type: object

From 513e53996d1aea19c4fd6e00cf94f4b73582c0a3 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 13:36:43 +0000
Subject: [PATCH 143/165] update targetsource CRD

---
 api/v1alpha1/targetsource_types.go            | 90 ++++++-------------
 api/v1alpha1/zz_generated.deepcopy.go         | 22 ++---
 .../operator.gnmic.dev_targetsources.yaml     | 90 ++++++-------------
 3 files changed, 65 insertions(+), 137 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index a15ca70..69bcf58 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -71,10 +71,10 @@ type HTTPConfig struct {
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
 
 	// Optional interval for polling the HTTP endpoint for targets
-	// TODO: add to docs
-	// +kubebuilder:default="6h"
+	// TODO: add to docs and increse to 6h
+	// +kubebuilder:default="30s"
 	// +kubebuilder:validation:Optional
-	PollInterval *metav1.Duration `json:"interval,omitempty"`
+	Interval *metav1.Duration `json:"interval,omitempty"`
 
 	// Optional timeout for HTTP requests to the endpoint
 	// +kubebuilder:default="10s"
@@ -86,13 +86,6 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	TLS *ClientTLSConfig `json:"tls,omitempty"`
 
-	// Field name in the JSON response that contains the list of items (targets).
-	// Must refer to a top-level key in the response object.
-	// If not specified, the entire response is expected to be a list of items.
-	// Example: "results"
-	// +kubebuilder:validation:Optional
-	TargetsField string `json:"targetsField,omitempty"`
-
 	// Optional pagination configuration for parsing responses from the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Pagination *PaginationSpec `json:"pagination,omitempty"`
@@ -102,23 +95,15 @@ type HTTPConfig struct {
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive"
 type ClientTLSConfig struct {
 	// Skip TLS verification of the Provider's certificate.
 	// +kubebuilder:default:=false
 	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
 
-	// Bundle of PEM CAs which will be used to validate the certificate
-	// chain presented by the Provider. Only used if using HTTPS to connect to Provider and
-	// ignored for HTTP connections.
-	// Mutually exclusive with CABundleSecretRef.
-	// +optional
-	CABundle string `json:"caBundle,omitempty"`
-
-	// Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+	// Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
 	// verifying the certificate chain presented by the Provider when using HTTPS.
 	// Mutually exclusive with CABundle.
-	CABundleSecretRef *corev1.SecretKeySelector `json:"caBundleSecretRef,omitempty"`
+	CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleSecretRef,omitempty"`
 }
 
 // AuthorizationSpec defines the configuration for authentication
@@ -128,57 +113,29 @@ type AuthorizationSpec struct {
 	Basic *BasicAuthSpec `json:"basic,omitempty"`
 	// Token-based authentication configuration
 	Token *TokenAuthSpec `json:"token,omitempty"`
-	// JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
-	// MTLS
 }
 
 // BasicAuthSpec defines the configuration for basic authentication
-// Enforce EITHER inline creds OR secret ref
-// +kubebuilder:validation:XValidation:rule="(has(self.credentialsSecretRef) && !has(self.username) && !has(self.password)) || (!has(self.credentialsSecretRef) && has(self.username) && has(self.password))",message="either credentialsSecretRef OR both username and password must be set, but not a mix"
 type BasicAuthSpec struct {
-	// Username for basic auth
-	// Mutually exclusive with CredentialsSecretRef.
-	Username string `json:"username,omitempty"`
-	// Password for basic auth
-	// Mutually exclusive with CredentialsSecretRef.
-	Password string `json:"password,omitempty"`
-
 	// Reference to a Secret containing "username" and "password" keys to use for
 	// basic authentication when connecting to the Provider.
 	// Mutually exclusive with Username and Password.
-	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
+	// +kubebuilder:validation:Required
+	CredentialsSecretRef corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
 }
 
 // TokenAuthSpec defines the configuration for token-based authentication
-// +kubebuilder:validation:XValidation:rule="has(self.token) != has(self.tokenSecretRef)",message="either token or tokenSecretRef must be set, but not both"
 type TokenAuthSpec struct {
 	// Scheme for the token, e.g. "Bearer"
 	// +kubebuilder:validation:MinLength=1
 	Scheme string `json:"scheme"`
-	// Token value for authentication
-	// Mutually exclusive with TokenSecretRef.
-	Token string `json:"token,omitempty"`
 	// Reference to a Secret containing a key with the token value to use for
 	// authentication when connecting to the Provider.
 	// Mutually exclusive with Token.
-	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+	// +kubebuilder:validation:Required
+	TokenSecretRef corev1.SecretKeySelector `json:"tokenSecretRef"`
 }
 
-// +kubebuilder(disabled):validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && (has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// +kubebuilder(disabled):validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
-// type JWTAuthSpec struct {
-// 	// Static pre-generated JWT
-// 	Token          string                    `json:"token,omitempty"`
-// 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
-// 	// Optional: generate JWT dynamically
-// 	Claims              map[string]string         `json:"claims,omitempty"`
-// 	Key                 string                    `json:"key,omitempty"`
-// 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
-// 	// HS256, RS256, ES256, etc.
-// 	Algorithm string           `json:"algorithm,omitempty"`
-// 	TTL       *metav1.Duration `json:"ttl,omitempty"`
-// }
-
 // PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
 	// Field name in the JSON response that contains the next page reference.
@@ -191,28 +148,39 @@ type PaginationSpec struct {
 	NextField string `json:"nextField,omitempty"`
 }
 
-// JSONPath-style expressions to extract target fields from the response
+// CEL expressions to extract target fields from the response
 // and map them to the corresponding Target fields.
 type ResponseMappingSpec struct {
-	// JSONPath expression to extract the target name from the response
-	// +kubebuilder:validation:Required
+	// Field name in the JSON response that contains the list of items (targets).
+	// If not specified, the entire response is expected to be a list of items.
+	// All subsequent fields are specified relative to this field
+	// Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
+	// +kubebuilder:validation:Optional
+	TargetsField string `json:"targetsField,omitempty"`
+
+	// CEL expression to extract the target name from the response
+	// If TargetsField is specified, this should be relative to TargetsField
+	// +kubebuilder:validation:Optional
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target IP from the response
-	// +kubebuilder:validation:Required
-	IP string `json:"ip"`
+	// CEL expression to extract the target Address from the response
+	// If TargetsField is specified, this should be relative to TargetsField
+	// +kubebuilder:validation:Optional
+	Address string `json:"address"`
 
-	// JSONPath expression to extract the target port from the response
+	// CEL expression to extract the target port from the response
+	// If TargetsField is specified, this should be relative to TargetsField
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// JSONPath expression to extract the target labels from the response
+	// CEL expression to extract the target labels from the response
 	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
 	// with values from the response taking precedence in case of conflicts.
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
 
-	// JSONPath expression to extract the target profile from the response
+	// CEL expression to extract the target profile from the response
+	// If TargetsField is specified, this should be relative to TargetsField
 	// +kubebuilder:validation:Optional
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index f055e7d..ac0119d 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -74,11 +74,7 @@ func (in *AuthorizationSpec) DeepCopy() *AuthorizationSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BasicAuthSpec) DeepCopyInto(out *BasicAuthSpec) {
 	*out = *in
-	if in.CredentialsSecretRef != nil {
-		in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
+	in.CredentialsSecretRef.DeepCopyInto(&out.CredentialsSecretRef)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BasicAuthSpec.
@@ -94,9 +90,9 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
-	if in.CABundleSecretRef != nil {
-		in, out := &in.CABundleSecretRef, &out.CABundleSecretRef
-		*out = new(v1.SecretKeySelector)
+	if in.CABundleRef != nil {
+		in, out := &in.CABundleRef, &out.CABundleRef
+		*out = new(v1.ConfigMapKeySelector)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -328,8 +324,8 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 		*out = new(AuthorizationSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.PollInterval != nil {
-		in, out := &in.PollInterval, &out.PollInterval
+	if in.Interval != nil {
+		in, out := &in.Interval, &out.Interval
 		*out = new(metav1.Duration)
 		**out = **in
 	}
@@ -1499,11 +1495,7 @@ func (in *TargetTLSConfig) DeepCopy() *TargetTLSConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TokenAuthSpec) DeepCopyInto(out *TokenAuthSpec) {
 	*out = *in
-	if in.TokenSecretRef != nil {
-		in, out := &in.TokenSecretRef, &out.TokenSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
+	in.TokenSecretRef.DeepCopyInto(&out.TokenSecretRef)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenAuthSpec.
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index a6e0877..36661dc 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -87,23 +87,9 @@ spec:
                                 - key
                                 type: object
                                 x-kubernetes-map-type: atomic
-                              password:
-                                description: |-
-                                  Password for basic auth
-                                  Mutually exclusive with CredentialsSecretRef.
-                                type: string
-                              username:
-                                description: |-
-                                  Username for basic auth
-                                  Mutually exclusive with CredentialsSecretRef.
-                                type: string
+                            required:
+                            - credentialsSecretRef
                             type: object
-                            x-kubernetes-validations:
-                            - message: either credentialsSecretRef OR both username
-                                and password must be set, but not a mix
-                              rule: (has(self.credentialsSecretRef) && !has(self.username)
-                                && !has(self.password)) || (!has(self.credentialsSecretRef)
-                                && has(self.username) && has(self.password))
                           token:
                             description: Token-based authentication configuration
                             properties:
@@ -111,11 +97,6 @@ spec:
                                 description: Scheme for the token, e.g. "Bearer"
                                 minLength: 1
                                 type: string
-                              token:
-                                description: |-
-                                  Token value for authentication
-                                  Mutually exclusive with TokenSecretRef.
-                                type: string
                               tokenSecretRef:
                                 description: |-
                                   Reference to a Secret containing a key with the token value to use for
@@ -145,11 +126,8 @@ spec:
                                 x-kubernetes-map-type: atomic
                             required:
                             - scheme
+                            - tokenSecretRef
                             type: object
-                            x-kubernetes-validations:
-                            - message: either token or tokenSecretRef must be set,
-                                but not both
-                              rule: has(self.token) != has(self.tokenSecretRef)
                         type: object
                         x-kubernetes-validations:
                         - message: exactly one of the fields in [basic token] must
@@ -157,7 +135,7 @@ spec:
                           rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
-                        default: 6h
+                        default: 30s
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
@@ -165,33 +143,41 @@ spec:
                         description: Optional mapping configuration for parsing responses
                           from the HTTP endpoint
                         properties:
-                          ip:
-                            description: JSONPath expression to extract the target
-                              IP from the response
+                          address:
+                            description: |-
+                              CEL expression to extract the target Address from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           labels:
                             additionalProperties:
                               type: string
                             description: |-
-                              JSONPath expression to extract the target labels from the response
+                              CEL expression to extract the target labels from the response
                               The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
                               with values from the response taking precedence in case of conflicts.
                             type: object
                           name:
-                            description: JSONPath expression to extract the target
-                              name from the response
+                            description: |-
+                              CEL expression to extract the target name from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           port:
-                            description: JSONPath expression to extract the target
-                              port from the response
+                            description: |-
+                              CEL expression to extract the target port from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           targetProfile:
-                            description: JSONPath expression to extract the target
-                              profile from the response
+                            description: |-
+                              CEL expression to extract the target profile from the response
+                              If TargetsField is specified, this should be relative to TargetsField
+                            type: string
+                          targetsField:
+                            description: |-
+                              Field name in the JSON response that contains the list of items (targets).
+                              If not specified, the entire response is expected to be a list of items.
+                              All subsequent fields are specified relative to this field
+                              Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
                             type: string
-                        required:
-                        - ip
-                        - name
                         type: object
                       pagination:
                         description: Optional pagination configuration for parsing
@@ -208,13 +194,6 @@ spec:
                               Example: "next" or "nextToken"
                             type: string
                         type: object
-                      targetsField:
-                        description: |-
-                          Field name in the JSON response that contains the list of items (targets).
-                          Must refer to a top-level key in the response object.
-                          If not specified, the entire response is expected to be a list of items.
-                          Example: "results"
-                        type: string
                       timeout:
                         default: 10s
                         description: Optional timeout for HTTP requests to the endpoint
@@ -224,22 +203,14 @@ spec:
                           Optional TLS configuration for connecting to the HTTP endpoint
                           If it is an HTTP endpoint, this will be ignored
                         properties:
-                          caBundle:
-                            description: |-
-                              Bundle of PEM CAs which will be used to validate the certificate
-                              chain presented by the Provider. Only used if using HTTPS to connect to Provider and
-                              ignored for HTTP connections.
-                              Mutually exclusive with CABundleSecretRef.
-                            type: string
                           caBundleSecretRef:
                             description: |-
-                              Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+                              Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
                               verifying the certificate chain presented by the Provider when using HTTPS.
                               Mutually exclusive with CABundle.
                             properties:
                               key:
-                                description: The key of the secret to select from.  Must
-                                  be a valid secret key.
+                                description: The key to select.
                                 type: string
                               name:
                                 default: ""
@@ -251,8 +222,8 @@ spec:
                                   More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
                                 type: string
                               optional:
-                                description: Specify whether the Secret or its key
-                                  must be defined
+                                description: Specify whether the ConfigMap or its
+                                  key must be defined
                                 type: boolean
                             required:
                             - key
@@ -263,9 +234,6 @@ spec:
                             description: Skip TLS verification of the Provider's certificate.
                             type: boolean
                         type: object
-                        x-kubernetes-validations:
-                        - message: caBundle and caBundleSecretRef are mutually exclusive
-                          rule: '!(has(self.caBundle) && has(self.caBundleSecretRef))'
                       url:
                         description: |-
                           URL of the HTTP endpoint to pull targets from

From bc2182fb05291bff316598f1e92026f7f319feaa Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 13:39:57 +0000
Subject: [PATCH 144/165] cleanup secret fetching

---
 internal/controller/discovery/loaders.go | 144 -----------------------
 1 file changed, 144 deletions(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index f0c19cf..f8d8c7d 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -18,152 +18,8 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi
 	case spec.Provider.HTTP != nil:
 		httpSpec := *spec.Provider.HTTP
 		cfg.AcceptPush = httpSpec.AcceptPush
-
-		// TODO: watch secrets -> if secret changes reconcile has to be executed
-		if httpSpec.Authorization != nil {
-			if err := resolveAuthorizationIntoSpec(
-				ctx,
-				c,
-				cfg.TargetsourceNN.Namespace,
-				httpSpec.Authorization,
-			); err != nil {
-				return nil, err
-			}
-		}
-		if httpSpec.TLS != nil {
-			if err := resolveTLSIntoSpec(
-				ctx,
-				c,
-				cfg.TargetsourceNN.Namespace,
-				httpSpec.TLS,
-			); err != nil {
-				return nil, err
-			}
-		}
-
 		return http.New(*cfg, httpSpec), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
 	}
 }
-
-// resolveAuthorizationIntoSpec fetches credentials from Kubernetes Secrets
-// and populates the AuthorizationSpec accordingly
-func resolveAuthorizationIntoSpec(
-	ctx context.Context,
-	c client.Client,
-	namespace string,
-	authSpec *gnmicv1alpha1.AuthorizationSpec,
-) error {
-	if authSpec == nil {
-		return nil
-	}
-	auth := authSpec
-
-	switch {
-	case auth.Basic != nil:
-		b := auth.Basic
-
-		if b.CredentialsSecretRef != nil {
-			values, err := GetSecretValues(
-				ctx,
-				c,
-				namespace,
-				b.CredentialsSecretRef.Name,
-				"username",
-				"password",
-			)
-			if err != nil {
-				return err
-			}
-			b.Username = values["username"]
-			b.Password = values["password"]
-		}
-
-	case auth.Token != nil:
-		t := auth.Token
-		if t.TokenSecretRef != nil {
-			key := "token"
-			if t.TokenSecretRef.Key != "" {
-				key = t.TokenSecretRef.Key
-			}
-			values, err := GetSecretValues(
-				ctx,
-				c,
-				namespace,
-				t.TokenSecretRef.Name,
-				key,
-			)
-			if err != nil {
-				return err
-			}
-			t.Token = values[key]
-		}
-
-		// case auth.JWT != nil:
-		// 	jwt := auth.JWT
-		// 	if jwt.TokenSecretRef != nil {
-		// 		values, err := GetSecretValues(
-		// 			ctx,
-		// 			c,
-		// 			namespaceName,
-		// 			jwt.TokenSecretRef.Name,
-		// 			"token",
-		// 		)
-		// 		if err != nil {
-		// 			return err
-		// 		}
-		// 		jwt.Token = values[jwt.TokenSecretRef.Key]
-		// 	}
-		// 	if jwt.SigningKeySecretRef != nil {
-		// 		values, err := GetSecretValues(
-		// 			ctx,
-		// 			c,
-		// 			namespaceName,
-		// 			jwt.SigningKeySecretRef.Name,
-		// 			"key",
-		// 		)
-		// 		if err != nil {
-		// 			return err
-		// 		}
-		// 		jwt.Key = values[jwt.SigningKeySecretRef.Key]
-
-		// 	}
-	}
-
-	return nil
-}
-
-// resolveTLSIntoSpec fetches TLS credentials from Kubernetes Secrets
-// and populates the ClientTLSConfig accordingly
-func resolveTLSIntoSpec(
-	ctx context.Context,
-	c client.Client,
-	namespace string,
-	tlsSpec *gnmicv1alpha1.ClientTLSConfig,
-) error {
-	if tlsSpec == nil {
-		return nil
-	}
-	tls := tlsSpec
-
-	if tls.CABundleSecretRef != nil {
-		key := "ca.crt"
-		if tls.CABundleSecretRef.Key != "" {
-			key = tls.CABundleSecretRef.Key
-		}
-		values, err := GetSecretValues(
-			ctx,
-			c,
-			namespace,
-			tls.CABundleSecretRef.Name,
-			key,
-		)
-		if err != nil {
-			return err
-		}
-		tls.CABundle = (values[key])
-	}
-
-	return nil
-}

From 022ed92641fb6298958db72dc3db213026e5067f Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 14:10:49 +0000
Subject: [PATCH 145/165] refactor

---
 api/v1alpha1/targetsource_types.go            |  66 ++++----
 api/v1alpha1/zz_generated.deepcopy.go         |  57 ++++---
 .../operator.gnmic.dev_targetsources.yaml     | 141 +++++++++---------
 internal/controller/discovery/loaders.go      |   2 +-
 4 files changed, 120 insertions(+), 146 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 9febc06..8cfb6ad 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -88,28 +88,20 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
 
-	// Optional configuration to enable webhooks
+	// Optional configuration to enable push
 	// +kubebuilder:validation:Optional
-	Webhook *WebhookSpec `json:"webhook,omitempty"`
+	Push *PushSpec `json:"push,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:rule="!(has(self.caBundle) && has(self.caBundleSecretRef))",message="caBundle and caBundleSecretRef are mutually exclusive"
 type ClientTLSConfig struct {
 	// Skip TLS verification of the Provider's certificate.
 	// +kubebuilder:default:=false
 	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
 
-	// Base64-encoded bundle of PEM CAs which will be used to validate the certificate
-	// chain presented by the Provider. Only used if using HTTPS to connect to Provider and
-	// ignored for HTTP connections.
-	// Mutually exclusive with CABundleSecretRef.
-	// +optional
-	CABundle []byte `json:"caBundle,omitempty"`
-
-	// Reference to a Secret containing a bundle of PEM-encoded CAs to use when
+	// Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
 	// verifying the certificate chain presented by the Provider when using HTTPS.
 	// Mutually exclusive with CABundle.
-	CABundleSecretRef *corev1.SecretKeySelector `json:"caBundleSecretRef,omitempty"`
+	CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleSecretRef,omitempty"`
 }
 
 // AuthorizationSpec defines the configuration for authentication
@@ -119,15 +111,14 @@ type AuthorizationSpec struct {
 	Basic *BasicAuthSpec `json:"basic,omitempty"`
 	// Token-based authentication configuration
 	Token *TokenAuthSpec `json:"token,omitempty"`
-	// JWT   *JWTAuthSpec   `json:"jwt,omitempty"`
-	// MTLS
 }
 
 // BasicAuthSpec defines the configuration for basic authentication
 type BasicAuthSpec struct {
 	// Reference to a Secret containing "username" and "password" keys to use for
 	// basic authentication when connecting to the Provider.
-	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef,omitempty"`
+	// +kubebuilder:validation:Required
+	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef"`
 }
 
 // TokenAuthSpec defines the configuration for token-based authentication
@@ -137,24 +128,11 @@ type TokenAuthSpec struct {
 	Scheme string `json:"scheme"`
 	// Reference to a Secret containing a key with the token value to use for
 	// authentication when connecting to the Provider.
+	// Mutually exclusive with Token.
+	// +kubebuilder:validation:Required
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
-// disabled: +kubebuilder:validation:XValidation:rule="!((has(self.token) || has(self.tokenSecretRef)) && ((has(self.key) || has(self.signingKeySecretRef) || has(self.claims)))",message="static JWT token and generated JWT configuration cannot be combined"
-// disabled: +kubebuilder:validation:XValidation:rule="!has(self.signingKeySecretRef) || self.algorithm != \"\"",message="algorithm must be specified when generating a JWT"
-// type JWTAuthSpec struct {
-// 	// Static pre-generated JWT
-// 	Token          string                    `json:"token,omitempty"`
-// 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
-// 	// Optional: generate JWT dynamically
-// 	Claims              map[string]string         `json:"claims,omitempty"`
-// 	Key                 string                    `json:"key,omitempty"`
-// 	SigningKeySecretRef *corev1.SecretKeySelector `json:"signingKeySecretRef,omitempty"`
-// 	// HS256, RS256, ES256, etc.
-// 	Algorithm string           `json:"algorithm,omitempty"`
-// 	TTL       *metav1.Duration `json:"ttl,omitempty"`
-// }
-
 // PaginationSpec defines the configuration for paginating through responses from providers
 type PaginationSpec struct {
 	// Field name in the JSON response that contains the next page reference.
@@ -167,39 +145,45 @@ type PaginationSpec struct {
 	NextField string `json:"nextField,omitempty"`
 }
 
-// JSONPath-style expressions to extract target fields from the response
+// CEL expressions to extract target fields from the response
 // and map them to the corresponding Target fields.
 type ResponseMappingSpec struct {
 	// Field name in the JSON response that contains the list of items (targets).
 	// If not specified, the entire response is expected to be a list of items.
 	// All subsequent fields are specified relative to this field
-	// Example: "results"
+	// Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
 	// +kubebuilder:validation:Optional
 	TargetsField string `json:"targetsField,omitempty"`
 
-	// JSONPath expression to extract the target name from the response list
-	// +kubebuilder:validation:Required
+	// CEL expression to extract the target name from the response
+	// If TargetsField is specified, this should be relative to TargetsField
+	// +kubebuilder:validation:Optional
 	Name string `json:"name"`
 
-	// JSONPath expression to extract the target address from the response list
-	// +kubebuilder:validation:Required
+	// CEL expression to extract the target Address from the response
+	// If TargetsField is specified, this should be relative to TargetsField
+	// +kubebuilder:validation:Optional
 	Address string `json:"address"`
 
-	// JSONPath expression to extract the target port from the response list
+	// CEL expression to extract the target port from the response
+	// If TargetsField is specified, this should be relative to TargetsField
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// JSONPath expression to extract the target labels from the response list
+	// CEL expression to extract the target labels from the response
+	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+	// with values from the response taking precedence in case of conflicts.
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
 
-	// JSONPath expression to extract the target profile from the response list
+	// CEL expression to extract the target profile from the response
+	// If TargetsField is specified, this should be relative to TargetsField
 	// +kubebuilder:validation:Optional
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
-// WebhookSpec defines the settings for event-based update mechanism (i.e. push-based)
-type WebhookSpec struct {
+// PushSpec defines the settings for event-based update mechanism (i.e. push-based)
+type PushSpec struct {
 	// +kubebuilder:default=false
 	Enabled bool `json:"enabled"`
 
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 9d59d3c..c9a7235 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -94,14 +94,9 @@ func (in *BasicAuthSpec) DeepCopy() *BasicAuthSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientTLSConfig) DeepCopyInto(out *ClientTLSConfig) {
 	*out = *in
-	if in.CABundle != nil {
-		in, out := &in.CABundle, &out.CABundle
-		*out = make([]byte, len(*in))
-		copy(*out, *in)
-	}
-	if in.CABundleSecretRef != nil {
-		in, out := &in.CABundleSecretRef, &out.CABundleSecretRef
-		*out = new(v1.SecretKeySelector)
+	if in.CABundleRef != nil {
+		in, out := &in.CABundleRef, &out.CABundleRef
+		*out = new(v1.ConfigMapKeySelector)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -358,9 +353,9 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 		*out = new(ResponseMappingSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.Webhook != nil {
-		in, out := &in.Webhook, &out.Webhook
-		*out = new(WebhookSpec)
+	if in.Push != nil {
+		in, out := &in.Push, &out.Push
+		*out = new(PushSpec)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -943,6 +938,26 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushSpec) DeepCopyInto(out *PushSpec) {
+	*out = *in
+	if in.Auth != nil {
+		in, out := &in.Auth, &out.Auth
+		*out = new(WebhookAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSpec.
+func (in *PushSpec) DeepCopy() *PushSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
 	*out = *in
@@ -1684,23 +1699,3 @@ func (in *WebhookSignatureAuthSpec) DeepCopy() *WebhookSignatureAuthSpec {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) {
-	*out = *in
-	if in.Auth != nil {
-		in, out := &in.Auth, &out.Auth
-		*out = new(WebhookAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec.
-func (in *WebhookSpec) DeepCopy() *WebhookSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookSpec)
-	in.DeepCopyInto(out)
-	return out
-}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 7816afc..b3eaa7d 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -80,6 +80,8 @@ spec:
                                 - key
                                 type: object
                                 x-kubernetes-map-type: atomic
+                            required:
+                            - credentialsSecretRef
                             type: object
                           token:
                             description: Token-based authentication configuration
@@ -92,6 +94,7 @@ spec:
                                 description: |-
                                   Reference to a Secret containing a key with the token value to use for
                                   authentication when connecting to the Provider.
+                                  Mutually exclusive with Token.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must
@@ -116,6 +119,7 @@ spec:
                                 x-kubernetes-map-type: atomic
                             required:
                             - scheme
+                            - tokenSecretRef
                             type: object
                         type: object
                         x-kubernetes-validations:
@@ -133,37 +137,40 @@ spec:
                           from the HTTP endpoint
                         properties:
                           address:
-                            description: JSONPath expression to extract the target
-                              address from the response list
+                            description: |-
+                              CEL expression to extract the target Address from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           labels:
                             additionalProperties:
                               type: string
-                            description: JSONPath expression to extract the target
-                              labels from the response list
+                            description: |-
+                              CEL expression to extract the target labels from the response
+                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+                              with values from the response taking precedence in case of conflicts.
                             type: object
                           name:
-                            description: JSONPath expression to extract the target
-                              name from the response list
+                            description: |-
+                              CEL expression to extract the target name from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           port:
-                            description: JSONPath expression to extract the target
-                              port from the response list
+                            description: |-
+                              CEL expression to extract the target port from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           targetProfile:
-                            description: JSONPath expression to extract the target
-                              profile from the response list
+                            description: |-
+                              CEL expression to extract the target profile from the response
+                              If TargetsField is specified, this should be relative to TargetsField
                             type: string
                           targetsField:
                             description: |-
                               Field name in the JSON response that contains the list of items (targets).
                               If not specified, the entire response is expected to be a list of items.
                               All subsequent fields are specified relative to this field
-                              Example: "results"
+                              Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
                             type: string
-                        required:
-                        - address
-                        - name
                         type: object
                       pagination:
                         description: Optional pagination configuration for parsing
@@ -180,65 +187,8 @@ spec:
                               Example: "next" or "nextToken"
                             type: string
                         type: object
-                      timeout:
-                        default: 10s
-                        description: Optional timeout for HTTP requests to the endpoint
-                        type: string
-                      tls:
-                        description: |-
-                          Optional TLS configuration for connecting to the HTTP endpoint
-                          If it is an HTTP endpoint, this will be ignored
-                        properties:
-                          caBundle:
-                            description: |-
-                              Base64-encoded bundle of PEM CAs which will be used to validate the certificate
-                              chain presented by the Provider. Only used if using HTTPS to connect to Provider and
-                              ignored for HTTP connections.
-                              Mutually exclusive with CABundleSecretRef.
-                            format: byte
-                            type: string
-                          caBundleSecretRef:
-                            description: |-
-                              Reference to a Secret containing a bundle of PEM-encoded CAs to use when
-                              verifying the certificate chain presented by the Provider when using HTTPS.
-                              Mutually exclusive with CABundle.
-                            properties:
-                              key:
-                                description: The key of the secret to select from.  Must
-                                  be a valid secret key.
-                                type: string
-                              name:
-                                default: ""
-                                description: |-
-                                  Name of the referent.
-                                  This field is effectively required, but due to backwards compatibility is
-                                  allowed to be empty. Instances of this type with an empty value here are
-                                  almost certainly wrong.
-                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                                type: string
-                              optional:
-                                description: Specify whether the Secret or its key
-                                  must be defined
-                                type: boolean
-                            required:
-                            - key
-                            type: object
-                            x-kubernetes-map-type: atomic
-                          insecureSkipVerify:
-                            default: false
-                            description: Skip TLS verification of the Provider's certificate.
-                            type: boolean
-                        type: object
-                        x-kubernetes-validations:
-                        - message: caBundle and caBundleSecretRef are mutually exclusive
-                          rule: '!(has(self.caBundle) && has(self.caBundleSecretRef))'
-                      url:
-                        description: |-
-                          URL of the HTTP endpoint to pull targets from
-                          If defined, the loader will periodically poll this endpoint for targets
-                        type: string
-                      webhook:
-                        description: Optional configuration to enable webhooks
+                      push:
+                        description: Optional configuration to enable push
                         properties:
                           auth:
                             properties:
@@ -325,6 +275,51 @@ spec:
                         required:
                         - enabled
                         type: object
+                      timeout:
+                        default: 10s
+                        description: Optional timeout for HTTP requests to the endpoint
+                        type: string
+                      tls:
+                        description: |-
+                          Optional TLS configuration for connecting to the HTTP endpoint
+                          If it is an HTTP endpoint, this will be ignored
+                        properties:
+                          caBundleSecretRef:
+                            description: |-
+                              Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
+                              verifying the certificate chain presented by the Provider when using HTTPS.
+                              Mutually exclusive with CABundle.
+                            properties:
+                              key:
+                                description: The key to select.
+                                type: string
+                              name:
+                                default: ""
+                                description: |-
+                                  Name of the referent.
+                                  This field is effectively required, but due to backwards compatibility is
+                                  allowed to be empty. Instances of this type with an empty value here are
+                                  almost certainly wrong.
+                                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                type: string
+                              optional:
+                                description: Specify whether the ConfigMap or its
+                                  key must be defined
+                                type: boolean
+                            required:
+                            - key
+                            type: object
+                            x-kubernetes-map-type: atomic
+                          insecureSkipVerify:
+                            default: false
+                            description: Skip TLS verification of the Provider's certificate.
+                            type: boolean
+                        type: object
+                      url:
+                        description: |-
+                          URL of the HTTP endpoint to pull targets from
+                          If defined, the loader will periodically poll this endpoint for targets
+                        type: string
                     type: object
                     x-kubernetes-validations:
                     - message: at least one of the fields in [url webhook] must be
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index dce3928..af014a5 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -13,7 +13,7 @@ func NewLoader(cfg *core.CommonLoaderConfig, spec gnmicv1alpha1.TargetSourceSpec
 
 	switch {
 	case spec.Provider.HTTP != nil:
-		cfg.AcceptPush = spec.Provider.HTTP.Webhook.Enabled
+		cfg.AcceptPush = spec.Provider.HTTP.Push.Enabled
 		return http.New(*cfg), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)

From 214810c45c6d5361651958354710a6f4e445dd8a Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Fri, 22 May 2026 08:16:35 -0600
Subject: [PATCH 146/165] removed old webhook statements

---
 api/v1alpha1/targetsource_types.go            |  14 +-
 api/v1alpha1/zz_generated.deepcopy.go         | 132 +++++++++---------
 .../operator.gnmic.dev_targetsources.yaml     |   5 +-
 3 files changed, 75 insertions(+), 76 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 8cfb6ad..56afe46 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -53,7 +53,7 @@ type ProviderSpec struct {
 }
 
 // HTTPConfig defines the configuration for the HTTP provider
-// +kubebuilder:validation:AtLeastOneOf:=url;webhook
+// +kubebuilder:validation:AtLeastOneOf:=url;push
 type HTTPConfig struct {
 	// URL of the HTTP endpoint to pull targets from
 	// If defined, the loader will periodically poll this endpoint for targets
@@ -188,22 +188,22 @@ type PushSpec struct {
 	Enabled bool `json:"enabled"`
 
 	// +kubebuilder:validation:Optional
-	Auth *WebhookAuthSpec `json:"auth,omitempty"`
+	Auth *PushAuthSpec `json:"auth,omitempty"`
 }
 
 // +kubebuilder:validation:ExactlyOneOf:=bearer;signature
-type WebhookAuthSpec struct {
-	Bearer    *WebhookBearerAuthSpec    `json:"bearer,omitempty"`
-	Signature *WebhookSignatureAuthSpec `json:"signature,omitempty"`
+type PushAuthSpec struct {
+	Bearer    *PushBearerAuthSpec    `json:"bearer,omitempty"`
+	Signature *PushSignatureAuthSpec `json:"signature,omitempty"`
 }
 
 // +kubebuilder:validation:Required
-type WebhookBearerAuthSpec struct {
+type PushBearerAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
 // +kubebuilder:validation:Required
-type WebhookSignatureAuthSpec struct {
+type PushSignatureAuthSpec struct {
 	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
 
 	// Header containing the signature
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index c9a7235..5567a56 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -938,12 +938,77 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushAuthSpec) DeepCopyInto(out *PushAuthSpec) {
+	*out = *in
+	if in.Bearer != nil {
+		in, out := &in.Bearer, &out.Bearer
+		*out = new(PushBearerAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Signature != nil {
+		in, out := &in.Signature, &out.Signature
+		*out = new(PushSignatureAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushAuthSpec.
+func (in *PushAuthSpec) DeepCopy() *PushAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushBearerAuthSpec) DeepCopyInto(out *PushBearerAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushBearerAuthSpec.
+func (in *PushBearerAuthSpec) DeepCopy() *PushBearerAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushBearerAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushSignatureAuthSpec) DeepCopyInto(out *PushSignatureAuthSpec) {
+	*out = *in
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSignatureAuthSpec.
+func (in *PushSignatureAuthSpec) DeepCopy() *PushSignatureAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushSignatureAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PushSpec) DeepCopyInto(out *PushSpec) {
 	*out = *in
 	if in.Auth != nil {
 		in, out := &in.Auth, &out.Auth
-		*out = new(WebhookAuthSpec)
+		*out = new(PushAuthSpec)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -1634,68 +1699,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookAuthSpec) DeepCopyInto(out *WebhookAuthSpec) {
-	*out = *in
-	if in.Bearer != nil {
-		in, out := &in.Bearer, &out.Bearer
-		*out = new(WebhookBearerAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.Signature != nil {
-		in, out := &in.Signature, &out.Signature
-		*out = new(WebhookSignatureAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookAuthSpec.
-func (in *WebhookAuthSpec) DeepCopy() *WebhookAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookBearerAuthSpec) DeepCopyInto(out *WebhookBearerAuthSpec) {
-	*out = *in
-	if in.TokenSecretRef != nil {
-		in, out := &in.TokenSecretRef, &out.TokenSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookBearerAuthSpec.
-func (in *WebhookBearerAuthSpec) DeepCopy() *WebhookBearerAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookBearerAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookSignatureAuthSpec) DeepCopyInto(out *WebhookSignatureAuthSpec) {
-	*out = *in
-	if in.SecretRef != nil {
-		in, out := &in.SecretRef, &out.SecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSignatureAuthSpec.
-func (in *WebhookSignatureAuthSpec) DeepCopy() *WebhookSignatureAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookSignatureAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index b3eaa7d..5ee51af 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -322,9 +322,8 @@ spec:
                         type: string
                     type: object
                     x-kubernetes-validations:
-                    - message: at least one of the fields in [url webhook] must be
-                        set
-                      rule: '[has(self.url),has(self.webhook)].filter(x,x==true).size()
+                    - message: at least one of the fields in [url push] must be set
+                      rule: '[has(self.url),has(self.push)].filter(x,x==true).size()
                         >= 1'
                 type: object
                 x-kubernetes-validations:

From 89bfbf54d76b6d593b003a051cee24a2932b4b26 Mon Sep 17 00:00:00 2001
From: Valentino Diller <valentino@diller.gl>
Date: Fri, 22 May 2026 08:25:50 -0600
Subject: [PATCH 147/165] changed pushSpec comment

---
 api/v1alpha1/targetsource_types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 56afe46..320482c 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -182,7 +182,7 @@ type ResponseMappingSpec struct {
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
-// PushSpec defines the settings for event-based update mechanism (i.e. push-based)
+// PushSpec defines the settings for event-based update mechanism (i.e. webhooks sent from the server)
 type PushSpec struct {
 	// +kubebuilder:default=false
 	Enabled bool `json:"enabled"`

From 0e147516f06d5745c54c45855c011e01bc5be6d8 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 14:34:19 +0000
Subject: [PATCH 148/165] update webhook to push

---
 api/v1alpha1/targetsource_types.go            |  14 +-
 api/v1alpha1/zz_generated.deepcopy.go         | 132 +++++++++---------
 .../operator.gnmic.dev_targetsources.yaml     |   5 +-
 3 files changed, 75 insertions(+), 76 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 8cfb6ad..56afe46 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -53,7 +53,7 @@ type ProviderSpec struct {
 }
 
 // HTTPConfig defines the configuration for the HTTP provider
-// +kubebuilder:validation:AtLeastOneOf:=url;webhook
+// +kubebuilder:validation:AtLeastOneOf:=url;push
 type HTTPConfig struct {
 	// URL of the HTTP endpoint to pull targets from
 	// If defined, the loader will periodically poll this endpoint for targets
@@ -188,22 +188,22 @@ type PushSpec struct {
 	Enabled bool `json:"enabled"`
 
 	// +kubebuilder:validation:Optional
-	Auth *WebhookAuthSpec `json:"auth,omitempty"`
+	Auth *PushAuthSpec `json:"auth,omitempty"`
 }
 
 // +kubebuilder:validation:ExactlyOneOf:=bearer;signature
-type WebhookAuthSpec struct {
-	Bearer    *WebhookBearerAuthSpec    `json:"bearer,omitempty"`
-	Signature *WebhookSignatureAuthSpec `json:"signature,omitempty"`
+type PushAuthSpec struct {
+	Bearer    *PushBearerAuthSpec    `json:"bearer,omitempty"`
+	Signature *PushSignatureAuthSpec `json:"signature,omitempty"`
 }
 
 // +kubebuilder:validation:Required
-type WebhookBearerAuthSpec struct {
+type PushBearerAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
 // +kubebuilder:validation:Required
-type WebhookSignatureAuthSpec struct {
+type PushSignatureAuthSpec struct {
 	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
 
 	// Header containing the signature
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index c9a7235..5567a56 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -938,12 +938,77 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushAuthSpec) DeepCopyInto(out *PushAuthSpec) {
+	*out = *in
+	if in.Bearer != nil {
+		in, out := &in.Bearer, &out.Bearer
+		*out = new(PushBearerAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Signature != nil {
+		in, out := &in.Signature, &out.Signature
+		*out = new(PushSignatureAuthSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushAuthSpec.
+func (in *PushAuthSpec) DeepCopy() *PushAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushBearerAuthSpec) DeepCopyInto(out *PushBearerAuthSpec) {
+	*out = *in
+	if in.TokenSecretRef != nil {
+		in, out := &in.TokenSecretRef, &out.TokenSecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushBearerAuthSpec.
+func (in *PushBearerAuthSpec) DeepCopy() *PushBearerAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushBearerAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushSignatureAuthSpec) DeepCopyInto(out *PushSignatureAuthSpec) {
+	*out = *in
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(v1.SecretKeySelector)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSignatureAuthSpec.
+func (in *PushSignatureAuthSpec) DeepCopy() *PushSignatureAuthSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushSignatureAuthSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *PushSpec) DeepCopyInto(out *PushSpec) {
 	*out = *in
 	if in.Auth != nil {
 		in, out := &in.Auth, &out.Auth
-		*out = new(WebhookAuthSpec)
+		*out = new(PushAuthSpec)
 		(*in).DeepCopyInto(*out)
 	}
 }
@@ -1634,68 +1699,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookAuthSpec) DeepCopyInto(out *WebhookAuthSpec) {
-	*out = *in
-	if in.Bearer != nil {
-		in, out := &in.Bearer, &out.Bearer
-		*out = new(WebhookBearerAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.Signature != nil {
-		in, out := &in.Signature, &out.Signature
-		*out = new(WebhookSignatureAuthSpec)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookAuthSpec.
-func (in *WebhookAuthSpec) DeepCopy() *WebhookAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookBearerAuthSpec) DeepCopyInto(out *WebhookBearerAuthSpec) {
-	*out = *in
-	if in.TokenSecretRef != nil {
-		in, out := &in.TokenSecretRef, &out.TokenSecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookBearerAuthSpec.
-func (in *WebhookBearerAuthSpec) DeepCopy() *WebhookBearerAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookBearerAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *WebhookSignatureAuthSpec) DeepCopyInto(out *WebhookSignatureAuthSpec) {
-	*out = *in
-	if in.SecretRef != nil {
-		in, out := &in.SecretRef, &out.SecretRef
-		*out = new(v1.SecretKeySelector)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSignatureAuthSpec.
-func (in *WebhookSignatureAuthSpec) DeepCopy() *WebhookSignatureAuthSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(WebhookSignatureAuthSpec)
-	in.DeepCopyInto(out)
-	return out
-}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index b3eaa7d..5ee51af 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -322,9 +322,8 @@ spec:
                         type: string
                     type: object
                     x-kubernetes-validations:
-                    - message: at least one of the fields in [url webhook] must be
-                        set
-                      rule: '[has(self.url),has(self.webhook)].filter(x,x==true).size()
+                    - message: at least one of the fields in [url push] must be set
+                      rule: '[has(self.url),has(self.push)].filter(x,x==true).size()
                         >= 1'
                 type: object
                 x-kubernetes-validations:

From e8c298e85d492ea5cfcf34b63858feae597d4780 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 14:38:15 +0000
Subject: [PATCH 149/165] fix after unlean merge

---
 internal/controller/discovery/client.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
index 28824cd..169166a 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/client.go
@@ -24,9 +24,6 @@ func fetchExistingTargets(
 
 	var targetList gnmicv1alpha1.TargetList
 
-	err := c.List(
-		ctx,
-		&targetList,
 	err := c.List(
 		ctx,
 		&targetList,

From b62f3f5a635eae07060f7b43b3d611337ea001cd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 15:01:05 +0000
Subject: [PATCH 150/165] refactor CRD

---
 api/v1alpha1/targetsource_types.go                     | 9 +++++----
 api/v1alpha1/zz_generated.deepcopy.go                  | 4 ++--
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 4 ++--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 320482c..b34b096 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -65,10 +65,10 @@ type HTTPConfig struct {
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
 
 	// Optional interval for polling the HTTP endpoint for targets
-	// TODO: increase default value
-	// +kubebuilder:default="30s"
+	// TODO: document about default value
+	// +kubebuilder:default="6h"
 	// +kubebuilder:validation:Optional
-	PollInterval *metav1.Duration `json:"interval,omitempty"`
+	Interval *metav1.Duration `json:"interval,omitempty"`
 
 	// Optional timeout for HTTP requests to the endpoint
 	// +kubebuilder:default="10s"
@@ -101,7 +101,8 @@ type ClientTLSConfig struct {
 	// Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
 	// verifying the certificate chain presented by the Provider when using HTTPS.
 	// Mutually exclusive with CABundle.
-	CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleSecretRef,omitempty"`
+	// +kubebuilder:validation:Optional
+	CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleRef,omitempty"`
 }
 
 // AuthorizationSpec defines the configuration for authentication
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 5567a56..9051fc7 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -328,8 +328,8 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 		*out = new(AuthorizationSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.PollInterval != nil {
-		in, out := &in.PollInterval, &out.PollInterval
+	if in.Interval != nil {
+		in, out := &in.Interval, &out.Interval
 		*out = new(metav1.Duration)
 		**out = **in
 	}
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 5ee51af..d9c9184 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -128,7 +128,7 @@ spec:
                           rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
                             == 1'
                       interval:
-                        default: 30s
+                        default: 6h
                         description: Optional interval for polling the HTTP endpoint
                           for targets
                         type: string
@@ -284,7 +284,7 @@ spec:
                           Optional TLS configuration for connecting to the HTTP endpoint
                           If it is an HTTP endpoint, this will be ignored
                         properties:
-                          caBundleSecretRef:
+                          caBundleRef:
                             description: |-
                               Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
                               verifying the certificate chain presented by the Provider when using HTTPS.

From f390eaeabd7ed9baba0b48d886cfc3b44f1d0565 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 15:13:06 +0000
Subject: [PATCH 151/165] fixes after merge

---
 internal/controller/discovery/loaders/http/mapper_jsonpath.go | 2 +-
 internal/controller/targetsource_controller.go                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
index 85bf00a..c82534d 100644
--- a/internal/controller/discovery/loaders/http/mapper_jsonpath.go
+++ b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
@@ -42,7 +42,7 @@ func (g *jsonPathGetter) GetName() (string, error) {
 
 // GetIP extracts the IP using JSONPath
 func (g *jsonPathGetter) GetIP() (string, error) {
-	val, err := g.get(g.spec.IP)
+	val, err := g.get(g.spec.Address)
 	if err != nil {
 		return "", fmt.Errorf("IP mapping failed: %w", err)
 	}
diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go
index 9726ace..cfa5782 100644
--- a/internal/controller/targetsource_controller.go
+++ b/internal/controller/targetsource_controller.go
@@ -196,7 +196,7 @@ func (r *TargetSourceReconciler) startDiscovery(
 		targetSource,
 		targetChannel,
 	)
-	loader, err := discovery.NewLoader(reconcileCtx, r.Client, &loaderConfig, targetSource.Spec)
+	loader, err := discovery.NewLoader(ctx, r.Client, &loaderConfig, targetSource.Spec)
 	if err != nil {
 		logger.Error(err, "Target loader could not be created")
 		cleanup()

From 722f53c1693d75ebf17199b6398a38087fec346a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 16:04:42 +0000
Subject: [PATCH 152/165] fix ip/address after merge

---
 internal/controller/discovery/loaders/http/mapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index de618fa..cb936cb 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -52,7 +52,7 @@ func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, er
 
 	return core.DiscoveredTarget{
 		Name:          name,
-		IP:            ip,
+		Address:       ip,
 		Port:          port,
 		Labels:        labels,
 		TargetProfile: targetProfile,

From d5be9a815719552f8b3bdf314fc83951559a38e1 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 16:04:53 +0000
Subject: [PATCH 153/165] implement secret and configmap fetcher

---
 internal/controller/discovery/client.go       | 43 -----------
 .../discovery/core/ressource_fetcher.go       | 15 ++++
 internal/controller/discovery/core/types.go   |  7 +-
 internal/controller/discovery/loaders.go      |  3 +-
 .../controller/discovery/loaders/http/auth.go | 74 ++++++++++++-------
 .../discovery/loaders/http/loader.go          | 50 ++++++++-----
 .../controller/discovery/ressource_fetcher.go | 60 +++++++++++++++
 7 files changed, 161 insertions(+), 91 deletions(-)
 create mode 100644 internal/controller/discovery/core/ressource_fetcher.go
 create mode 100644 internal/controller/discovery/ressource_fetcher.go

diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go
index 169166a..74edf29 100644
--- a/internal/controller/discovery/client.go
+++ b/internal/controller/discovery/client.go
@@ -2,9 +2,7 @@ package discovery
 
 import (
 	"context"
-	"fmt"
 
-	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -94,44 +92,3 @@ func updateTargetSourceStatus(ctx context.Context, c client.Client, ts *gnmicv1a
 
 	return err
 }
-
-// Helper: GetSecretValues returns values from a secret
-// If keys are provided -> returns only those keys
-// If keys is empty -> returns entire secret data
-func GetSecretValues(
-	ctx context.Context,
-	c client.Client,
-	namespace string,
-	secretRef string,
-	keys ...string,
-) (map[string]string, error) {
-	var secret corev1.Secret
-	if err := c.Get(ctx,
-		client.ObjectKey{
-			Name:      secretRef,
-			Namespace: namespace,
-		}, &secret); err != nil {
-		return nil, fmt.Errorf("failed to get secret %s/%s: %w", namespace, secretRef, err)
-	}
-
-	result := make(map[string]string)
-
-	// Return full secret
-	if len(keys) == 0 {
-		for k, v := range secret.Data {
-			result[k] = string(v)
-		}
-		return result, nil
-	}
-
-	// Return specific keys
-	for _, key := range keys {
-		val, ok := secret.Data[key]
-		if !ok {
-			return nil, fmt.Errorf("key %s missing in secret %s/%s", key, namespace, secretRef)
-		}
-		result[key] = string(val)
-	}
-
-	return result, nil
-}
diff --git a/internal/controller/discovery/core/ressource_fetcher.go b/internal/controller/discovery/core/ressource_fetcher.go
new file mode 100644
index 0000000..31a82cf
--- /dev/null
+++ b/internal/controller/discovery/core/ressource_fetcher.go
@@ -0,0 +1,15 @@
+package core
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// ResourceFetcher provides read-only access to namespaced Secret and
+// ConfigMap data for loaders without requiring each loader to carry a
+// Kubernetes client.
+type ResourceFetcher interface {
+	GetSecretKey(ctx context.Context, namespace string, selector *corev1.SecretKeySelector) (string, error)
+	GetConfigMapKey(ctx context.Context, namespace string, selector *corev1.ConfigMapKeySelector) (string, error)
+}
diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go
index 8de38c1..a9a208f 100644
--- a/internal/controller/discovery/core/types.go
+++ b/internal/controller/discovery/core/types.go
@@ -19,9 +19,10 @@ type DiscoveryRegistryValue struct {
 }
 
 type CommonLoaderConfig struct {
-	TargetsourceNN types.NamespacedName
-	ChunkSize      int
-	AcceptPush     bool
+	TargetsourceNN  types.NamespacedName
+	ChunkSize       int
+	AcceptPush      bool
+	ResourceFetcher ResourceFetcher
 }
 
 // EventAction represents the type of a discovery event
diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index f8d8c7d..4ecf9dd 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -17,7 +17,8 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi
 	switch {
 	case spec.Provider.HTTP != nil:
 		httpSpec := *spec.Provider.HTTP
-		cfg.AcceptPush = httpSpec.AcceptPush
+		cfg.AcceptPush = httpSpec.Push.Enabled
+		cfg.ResourceFetcher = newK8sResourceFetcher(c)
 		return http.New(*cfg, httpSpec), nil
 	default:
 		return nil, fmt.Errorf("unknown targetsource provider, check TargetSource CRD for %s", cfg.TargetsourceNN)
diff --git a/internal/controller/discovery/loaders/http/auth.go b/internal/controller/discovery/loaders/http/auth.go
index 0af0556..109bbc8 100644
--- a/internal/controller/discovery/loaders/http/auth.go
+++ b/internal/controller/discovery/loaders/http/auth.go
@@ -1,38 +1,62 @@
 package http
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
+
+	corev1 "k8s.io/api/core/v1"
 )
 
-func (l *Loader) applyAuthorization(req *http.Request) {
+// fetchSecret uses the configured ResourceFetcher to resolve secret values.
+func (l *Loader) fetchSecret(ctx context.Context, sel *corev1.SecretKeySelector) (string, error) {
+	if l.loaderCfg.ResourceFetcher == nil {
+		return "", nil
+	}
+	return l.loaderCfg.ResourceFetcher.GetSecretKey(ctx, l.loaderCfg.TargetsourceNN.Namespace, sel)
+}
+
+func (l *Loader) applyAuthorization(req *http.Request) error {
 	auth := l.spec.Authorization
 	if auth == nil {
-		return
+		return nil
+	}
+	// Basic auth
+	if auth.Basic != nil {
+		// Secret-based credentials
+		if auth.Basic.CredentialsSecretRef != nil {
+			val, err := l.fetchSecret(req.Context(), auth.Basic.CredentialsSecretRef)
+			if err != nil {
+				return err
+			}
+			var cm map[string]string
+			if err := json.Unmarshal([]byte(val), &cm); err == nil {
+				username := cm["username"]
+				password := cm["password"]
+				if username != "" || password != "" {
+					req.SetBasicAuth(username, password)
+					return nil
+				}
+			}
+			return err
+		}
+		return fmt.Errorf("Basic auth enabled but no valid credentials provided")
 	}
 
-	switch {
-	case auth.Basic != nil:
-		req.SetBasicAuth(
-			auth.Basic.Username,
-			auth.Basic.Password,
-		)
-
-	case auth.Token != nil:
-		req.Header.Set(
-			"Authorization",
-			fmt.Sprintf("%s %s",
-				auth.Token.Scheme,
-				auth.Token.Token,
-			),
-		)
-
-		// case auth.JWT != nil:
-		// 	if auth.JWT.Token != "" {
-		// 		req.Header.Set(
-		// 			"Authorization",
-		// 			fmt.Sprintf("Bearer %s", auth.JWT.Token),
-		// 		)
-		// 	}
+	// Token-based auth: prefer secret ref if present
+	if auth.Token != nil {
+		if auth.Token.TokenSecretRef != nil {
+			token, err := l.fetchSecret(req.Context(), auth.Token.TokenSecretRef)
+			if err != nil {
+				return err
+			}
+			req.Header.Set("Authorization", fmt.Sprintf("%s %s", auth.Token.Scheme, token))
+			return nil
+		}
+		return fmt.Errorf("Token auth enabled but no valid token secret reference provided")
 	}
+
+	// No supported auth method configured
+	return fmt.Errorf("no supported authentication method configured")
 }
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 543b7d8..039d023 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -59,11 +59,11 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 
 	logger.Info("HTTP loader started")
 
-	client, err := l.buildHTTPClient()
+	client, err := l.buildHTTPClient(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to build HTTP client: %w", err)
 	}
-	interval := l.spec.PollInterval.Duration
+	interval := l.spec.Interval.Duration
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
@@ -90,7 +90,7 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 			logger.Info(
 				"Discovered target",
 				"name", t.Name,
-				"ip", t.IP,
+				"address", t.Address,
 				"port", t.Port,
 				"labels", t.Labels,
 				"profile", t.TargetProfile,
@@ -133,18 +133,28 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er
 }
 
 // buildHTTPClient constructs an HTTP client with optional configuration
-func (l *Loader) buildHTTPClient() (*http.Client, error) {
+func (l *Loader) buildHTTPClient(ctx context.Context) (*http.Client, error) {
 	tlsConfig := &tls.Config{
 		InsecureSkipVerify: l.spec.TLS != nil && l.spec.TLS.InsecureSkipVerify,
 	}
 
-	// If a CA bundle is provided, add it to the TLS config
-	if l.spec.TLS != nil && len(l.spec.TLS.CABundle) > 0 {
-		certPool := x509.NewCertPool()
-		if ok := certPool.AppendCertsFromPEM([]byte(l.spec.TLS.CABundle)); !ok {
-			return nil, fmt.Errorf("Failed to parse CA bundle for TargetSource %s/%s\n", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
+	// If a CA bundle is provided, add it to the TLS config.
+	if l.spec.TLS != nil {
+		var caBundle string
+		if l.spec.TLS.CABundleRef != nil {
+			var err error
+			caBundle, err = l.loaderCfg.ResourceFetcher.GetConfigMapKey(ctx, l.loaderCfg.TargetsourceNN.Namespace, l.spec.TLS.CABundleRef)
+			if err != nil {
+				return nil, fmt.Errorf("failed to fetch CA bundle from config map ref: %w", err)
+			}
+		}
+		if len(caBundle) > 0 {
+			certPool := x509.NewCertPool()
+			if ok := certPool.AppendCertsFromPEM([]byte(caBundle)); !ok {
+				return nil, fmt.Errorf("failed to parse CA bundle for TargetSource %s/%s", l.loaderCfg.TargetsourceNN.Namespace, l.loaderCfg.TargetsourceNN.Name)
+			}
+			tlsConfig.RootCAs = certPool
 		}
-		tlsConfig.RootCAs = certPool
 	}
 
 	// Build the HTTP client with the specified timeout and TLS config
@@ -205,7 +215,9 @@ func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string)
 		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
 	}
 	req.Header.Set("Accept", "application/json")
-	l.applyAuthorization(req)
+	if err := l.applyAuthorization(req); err != nil {
+		return nil, fmt.Errorf("applying authorization to HTTP request failed: %w", err)
+	}
 
 	// Execute HTTP request
 	resp, err := client.Do(req)
@@ -231,22 +243,22 @@ func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string)
 func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
 
-	if l.spec.TargetsField != "" {
+	if l.spec.ResponseMapping.TargetsField != "" {
 		obj, ok := raw.(map[string]interface{})
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: expected JSON object when itemsField '%s' is configured (e.g. {\"%s\": [...]})",
-				l.spec.TargetsField,
-				l.spec.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
 			)
 		}
 
-		results, ok := obj[l.spec.TargetsField]
+		results, ok := obj[l.spec.ResponseMapping.TargetsField]
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: itemsField '%s' not found. ensure the API response contains this field (e.g. {\"%s\": [...]})",
-				l.spec.TargetsField,
-				l.spec.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
 			)
 		}
 
@@ -254,8 +266,8 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger)
 		if !ok {
 			return nil, fmt.Errorf(
 				"invalid HTTP response: itemsField '%s' must be an array of objects (e.g. {\"%s\": [...]})",
-				l.spec.TargetsField,
-				l.spec.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
+				l.spec.ResponseMapping.TargetsField,
 			)
 		}
 
diff --git a/internal/controller/discovery/ressource_fetcher.go b/internal/controller/discovery/ressource_fetcher.go
new file mode 100644
index 0000000..c544b30
--- /dev/null
+++ b/internal/controller/discovery/ressource_fetcher.go
@@ -0,0 +1,60 @@
+package discovery
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+)
+
+// k8sResourceFetcher implements core.ResourceFetcher using a controller runtime client
+type k8sResourceFetcher struct {
+	client client.Client
+}
+
+// GetSecretKey retrieves the value of a specific key from a Kubernetes Secret
+func (f *k8sResourceFetcher) GetSecretKey(ctx context.Context, namespace string, selector *corev1.SecretKeySelector) (string, error) {
+	if selector == nil {
+		return "", nil
+	}
+	var secret corev1.Secret
+	key := client.ObjectKey{Namespace: namespace, Name: selector.Name}
+	if err := f.client.Get(ctx, key, &secret); err != nil {
+		return "", err
+	}
+	if selector.Key == "" {
+		return "", fmt.Errorf("secret key selector has empty key for secret %s/%s", namespace, selector.Name)
+	}
+	val, ok := secret.Data[selector.Key]
+	if !ok {
+		return "", fmt.Errorf("secret %s/%s does not contain key %s", namespace, selector.Name, selector.Key)
+	}
+	return string(val), nil
+}
+
+// GetConfigMapKey retrieves the value of a specific key from a Kubernetes ConfigMap
+func (f *k8sResourceFetcher) GetConfigMapKey(ctx context.Context, namespace string, selector *corev1.ConfigMapKeySelector) (string, error) {
+	if selector == nil {
+		return "", nil
+	}
+	var cm corev1.ConfigMap
+	key := client.ObjectKey{Namespace: namespace, Name: selector.Name}
+	if err := f.client.Get(ctx, key, &cm); err != nil {
+		return "", err
+	}
+	if selector.Key == "" {
+		return "", fmt.Errorf("config map key selector has empty key for config map %s/%s", namespace, selector.Name)
+	}
+	val, ok := cm.Data[selector.Key]
+	if !ok {
+		return "", fmt.Errorf("config map %s/%s does not contain key %s", namespace, selector.Name, selector.Key)
+	}
+	return val, nil
+}
+
+func newK8sResourceFetcher(c client.Client) core.ResourceFetcher {
+	return &k8sResourceFetcher{client: c}
+}

From 647e2a9eec4e750621c8f84cddf20f3d730fe870 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 20:13:43 +0000
Subject: [PATCH 154/165] replace JSONPath mapping with CEL mapping

---
 api/v1alpha1/targetsource_types.go            | 124 ++++++--
 go.mod                                        |   7 +-
 go.sum                                        |  15 +-
 .../discovery/loaders/http/loader.go          |  72 ++---
 .../discovery/loaders/http/mapper.go          | 275 +++++++++++++++---
 .../discovery/loaders/http/mapper_direct.go   |  82 ------
 .../discovery/loaders/http/mapper_jsonpath.go | 106 -------
 7 files changed, 379 insertions(+), 302 deletions(-)
 delete mode 100644 internal/controller/discovery/loaders/http/mapper_direct.go
 delete mode 100644 internal/controller/discovery/loaders/http/mapper_jsonpath.go

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index b34b096..426731d 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -146,39 +146,125 @@ type PaginationSpec struct {
 	NextField string `json:"nextField,omitempty"`
 }
 
-// CEL expressions to extract target fields from the response
-// and map them to the corresponding Target fields.
+// ResponseMappingSpec controls how targets are extracted from an HTTP JSON response.
+//
+// This allows you to map fields from a JSON API into targets using either:
+//   - simple direct field access (e.g. item["name"])
+//   - or CEL expressions for more advanced logic
+//
+// General behavior:
+//
+//  1. Selecting targets:
+//     - `targetsField` is a CEL expression that selects the list of targets
+//     - It runs once on the full response (`self`) and MUST return a list
+//     - If not set, the response itself must be a JSON array
+//
+//  2. Extracting fields:
+//     - Each field (name, address, port, labels, etc.) is handled independently
+//     - If a CEL expression is provided → it is evaluated
+//     - If not provided → the value is read directly from the target object
+//
+//  3. Available variables in CEL:
+//     - item -> the current target object
+//     - self -> the full HTTP response JSON
+//
+// Example:
+//
+//	Response:
+//	{
+//	  "results": [
+//	    { "name": "device1", "ip": "10.0.0.1", "env": "prod" }
+//	  ],
+//	  "meta": { "region": "eu-west" }
+//	}
+//
+//	Mapping:
+//	targetsField: "self.results"
+//
+//	name: ""            # direct → item["name"]
+//	address: "item.ip"  # CEL
+//
+//	labels:
+//	  env:    "item.env"
+//	  region: "self.meta.region"
 type ResponseMappingSpec struct {
-	// Field name in the JSON response that contains the list of items (targets).
-	// If not specified, the entire response is expected to be a list of items.
-	// All subsequent fields are specified relative to this field
-	// Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
+	// CEL expression that selects the list of target objects from the response.
+	//
+	// This is evaluated once using:
+	//   self -> full JSON response
+	//
+	// Example:
+	//   targetsField: "self.results"
+	//
+	// If not set, the response itself must be a JSON array with the targets.
+	//
 	// +kubebuilder:validation:Optional
 	TargetsField string `json:"targetsField,omitempty"`
 
-	// CEL expression to extract the target name from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target name.
+	//
+	// If not set, defaults to:
+	//   item["name"]
+	//
+	// Example:
+	//   "item.hostname"
+	//
 	// +kubebuilder:validation:Optional
-	Name string `json:"name"`
+	Name string `json:"name,omitempty"`
 
-	// CEL expression to extract the target Address from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target address.
+	//
+	// If not set, defaults to:
+	//   item["address"]
+	//
+	// Example:
+	//   "item.ip"
+	//
 	// +kubebuilder:validation:Optional
-	Address string `json:"address"`
+	Address string `json:"address,omitempty"`
 
-	// CEL expression to extract the target port from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target port.
+	//
+	// If not set, defaults to:
+	//   item["port"]
+	//
+	// Example:
+	//   "item.port"
+	//
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// CEL expression to extract the target labels from the response
-	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
-	// with values from the response taking precedence in case of conflicts.
+	// Defines labels to attach to the target.
+	//
+	// Each entry defines:
+	//   key   -> label name
+	//   value -> CEL expression
+	//
+	// Expressions can use both:
+	//   item -> current target
+	//   self -> full response
+	//
+	// Example:
+	//   labels:
+	//     env:    "item.environment"
+	//     region: "self.meta.region"
+	//
+	// If not set, defaults to:
+	//   item["labels"]
+	//
+	// Dynamic labels override static labels defined in the TargetSource.
+	//
 	// +kubebuilder:validation:Optional
 	Labels map[string]string `json:"labels,omitempty"`
 
-	// CEL expression to extract the target profile from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target profile.
+	//
+	// If not set, defaults to:
+	//   item["targetProfile"]
+	//
+	// Example:
+	//   "item.type == 'edge' ? 'edge-profile' : 'default'"
+	//
 	// +kubebuilder:validation:Optional
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
diff --git a/go.mod b/go.mod
index c877a7b..c08b9b8 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.25.5
 require (
 	github.com/cert-manager/cert-manager v1.19.3
 	github.com/go-logr/logr v1.4.3
+	github.com/google/cel-go v0.28.1
 	github.com/google/uuid v1.6.0
 	github.com/onsi/ginkgo/v2 v2.28.3
 	github.com/onsi/gomega v1.40.0
@@ -19,10 +20,10 @@ require (
 )
 
 require (
+	cel.dev/expr v0.25.1 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
-	github.com/PaesslerAG/gval v1.0.0 // indirect
-	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -65,6 +66,7 @@ require (
 	go.uber.org/zap v1.27.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
 	golang.org/x/mod v0.35.0 // indirect
 	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
@@ -75,6 +77,7 @@ require (
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
 	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
diff --git a/go.sum b/go.sum
index d900003..0a845c4 100644
--- a/go.sum
+++ b/go.sum
@@ -1,12 +1,11 @@
+cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
+cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
-github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
-github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
-github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
-github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
-github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs=
@@ -81,6 +80,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
 github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM=
+github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8=
 github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c=
 github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -177,6 +178,8 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
 golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
 golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
@@ -199,6 +202,8 @@ gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0
 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 039d023..341feac 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -157,9 +157,10 @@ func (l *Loader) buildHTTPClient(ctx context.Context) (*http.Client, error) {
 		}
 	}
 
+	timeout := l.spec.Timeout.Duration
 	// Build the HTTP client with the specified timeout and TLS config
 	return &http.Client{
-		Timeout: l.spec.Timeout.Duration,
+		Timeout: timeout,
 		Transport: &http.Transport{
 			TLSClientConfig: tlsConfig,
 		},
@@ -238,70 +239,43 @@ func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string)
 	return raw, nil
 }
 
-// extractTargetsFromResponse extracts items from the response
-// and maps each item into a DiscoveredTarget
+// extractTargetsFromResponse extracts items from the response and maps each item into a DiscoveredTarget
 func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
 	var items []interface{}
-
-	if l.spec.ResponseMapping.TargetsField != "" {
-		obj, ok := raw.(map[string]interface{})
-		if !ok {
-			return nil, fmt.Errorf(
-				"invalid HTTP response: expected JSON object when itemsField '%s' is configured (e.g. {\"%s\": [...]})",
-				l.spec.ResponseMapping.TargetsField,
-				l.spec.ResponseMapping.TargetsField,
-			)
+	// If ResponseMapping is configured and TargetsField is provided we treat
+	// it as a CEL expression that evaluates against the whole response and
+	// must return an array of items.
+	if l.spec.ResponseMapping != nil && l.spec.ResponseMapping.TargetsField != "" {
+		prog, err := compileCEL(l.spec.ResponseMapping.TargetsField)
+		if err != nil {
+			return nil, fmt.Errorf("invalid TargetsField CEL expression: %w", err)
 		}
-
-		results, ok := obj[l.spec.ResponseMapping.TargetsField]
-		if !ok {
-			return nil, fmt.Errorf(
-				"invalid HTTP response: itemsField '%s' not found. ensure the API response contains this field (e.g. {\"%s\": [...]})",
-				l.spec.ResponseMapping.TargetsField,
-				l.spec.ResponseMapping.TargetsField,
-			)
+		out, _, err := prog.Eval(map[string]interface{}{"self": raw})
+		if err != nil {
+			return nil, fmt.Errorf("evaluating TargetsField CEL expression failed: %w", err)
 		}
-
-		array, ok := results.([]interface{})
+		if out == nil {
+			return nil, fmt.Errorf("TargetsField expression returned nil")
+		}
+		array, ok := out.Value().([]interface{})
 		if !ok {
-			return nil, fmt.Errorf(
-				"invalid HTTP response: itemsField '%s' must be an array of objects (e.g. {\"%s\": [...]})",
-				l.spec.ResponseMapping.TargetsField,
-				l.spec.ResponseMapping.TargetsField,
-			)
+			return nil, fmt.Errorf("invalid HTTP response: targetsField expression must evaluate to an array of objects")
 		}
-
 		items = array
 	} else {
+		//If TargetsField is empty, the raw response is expected to be an array of items.
 		array, ok := raw.([]interface{})
 		if !ok {
-			return nil, fmt.Errorf("invalid HTTP response: expected a JSON array because itemsField is not set (e.g. [{...}, {...}])")
+			return nil, fmt.Errorf("invalid HTTP response: expected a JSON array when itemsField is not set")
 		}
 		items = array
 	}
 
 	// Map items to targets
 	var targets []core.DiscoveredTarget
-	for _, item := range items {
-		obj, ok := item.(map[string]interface{})
-		if !ok {
-			logger.Error(fmt.Errorf("invalid target format"),
-				"Failed to convert target to map",
-				"item", item,
-			)
-			continue
-		}
-
-		target, err := l.mapItem(obj)
-		if err != nil {
-			logger.Error(err,
-				"Failed to map target",
-				"item", obj,
-			)
-			continue
-		}
-
-		targets = append(targets, target)
+	targets, err := l.mapItemsToTargets(items, raw, logger)
+	if err != nil {
+		return nil, fmt.Errorf("mapping items to targets failed: %w", err)
 	}
 
 	return targets, nil
diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index cb936cb..a14b5cb 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -1,71 +1,266 @@
 package http
 
 import (
+	"fmt"
 	"math"
 	"strconv"
 
 	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/go-logr/logr"
+	"github.com/google/cel-go/cel"
 )
 
-// valueGetter defines the contract for extracting values from a response item
-type valueGetter interface {
-	GetName() (string, error)
-	GetIP() (string, error)
-	GetPort() int32
-	GetLabels() map[string]string
-	GetTargetProfile() string
-}
+// mapItemsToTargets converts a list of raw JSON items into DiscoveredTargets using the configured mapping rules
+func (l *Loader) mapItemsToTargets(items []interface{}, full interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
+	// Compile CEL expressions once for efficiency
+	compiled, err := l.compileMapping()
+	if err != nil {
+		return nil, fmt.Errorf("compile mapping: %w", err)
+	}
 
-// getGetter selects the extraction strategy based on the spec
-// If no ResponseMapping is defined -> use direct mapping
-func (l *Loader) getGetter(item map[string]interface{}) valueGetter {
-	if l.spec.ResponseMapping == nil {
-		return &directGetter{
-			item: item,
+	// Map items to targets
+	targets := make([]core.DiscoveredTarget, 0, len(items))
+	for _, item := range items {
+		obj, ok := item.(map[string]interface{})
+		if !ok {
+			logger.Error(fmt.Errorf("invalid target format"),
+				"failed to convert target to map",
+				"item", item,
+			)
+			continue
+		}
+		target, err := l.mapItemToTarget(obj, full, compiled)
+		if err != nil {
+			logger.Error(err,
+				"failed to map target",
+				"item", obj,
+			)
+			continue
 		}
-	}
 
-	return &jsonPathGetter{
-		item: item,
-		spec: l.spec.ResponseMapping,
+		targets = append(targets, target)
 	}
+
+	return targets, nil
 }
 
-// mapItem is the mapping entrypoint used by the loader
-// It uses the selected valueGetter and produces a DiscoveredTarget
-func (l *Loader) mapItem(item map[string]interface{}) (core.DiscoveredTarget, error) {
-	getter := l.getGetter(item)
+type compiledMapping struct {
+	name    cel.Program
+	address cel.Program
+	port    cel.Program
+
+	targetProfile cel.Program
+	labels        map[string]cel.Program
+}
+
+func (l *Loader) compileMapping() (*compiledMapping, error) {
+	rm := l.spec.ResponseMapping
+	cm := &compiledMapping{
+		labels: make(map[string]cel.Program),
+	}
+	if rm == nil {
+		return cm, nil
+	}
+
+	var err error
+	if rm.Name != "" {
+		cm.name, err = compileCEL(rm.Name)
+		if err != nil {
+			return nil, fmt.Errorf("name: %w", err)
+		}
+	}
+	if rm.Address != "" {
+		cm.address, err = compileCEL(rm.Address)
+		if err != nil {
+			return nil, fmt.Errorf("address: %w", err)
+		}
+	}
+	if rm.Port != "" {
+		cm.port, err = compileCEL(rm.Port)
+		if err != nil {
+			return nil, fmt.Errorf("port: %w", err)
+		}
+	}
+	if rm.TargetProfile != "" {
+		cm.targetProfile, err = compileCEL(rm.TargetProfile)
+		if err != nil {
+			return nil, fmt.Errorf("targetProfile: %w", err)
+		}
+	}
+	for key, expr := range rm.Labels {
+		p, err := compileCEL(expr)
+		if err != nil {
+			return nil, fmt.Errorf("label %s: %w", key, err)
+		}
+		cm.labels[key] = p
+	}
+
+	return cm, nil
+}
 
-	name, err := getter.GetName()
+// mapItemToTarget converts a raw JSON object into a DiscoveredTarget
+func (l *Loader) mapItemToTarget(item map[string]interface{}, full interface{}, cm *compiledMapping) (core.DiscoveredTarget, error) {
+	name, err := l.getName(item, full, cm)
 	if err != nil {
 		return core.DiscoveredTarget{}, err
 	}
 
-	ip, err := getter.GetIP()
+	address, err := l.getAddress(item, full, cm)
 	if err != nil {
 		return core.DiscoveredTarget{}, err
 	}
 
-	port := getter.GetPort()
-	labels := getter.GetLabels()
-	targetProfile := getter.GetTargetProfile()
-
 	return core.DiscoveredTarget{
 		Name:          name,
-		Address:       ip,
-		Port:          port,
-		Labels:        labels,
-		TargetProfile: targetProfile,
+		Address:       address,
+		Port:          l.getPort(item, full, cm),
+		Labels:        l.getLabels(item, full, cm),
+		TargetProfile: l.getTargetProfile(item, full, cm),
 	}, nil
 }
 
-// extractPort attempts to normalize different JSON types into int32
-//
-// Supports:
-// - float64 (default JSON number type)
-// - string ("1234")
-//
-// Returns 0 if conversion fails (treated as "no port specified").
+// getName extracts the target name from the item using the compiled CEL expression if provided,
+// otherwise it falls back to the default "name" field
+func (l *Loader) getName(item map[string]interface{}, full interface{}, cm *compiledMapping) (string, error) {
+	if cm.name != nil {
+		val, err := evalCEL(cm.name, item, full)
+		if err != nil {
+			return "", err
+		}
+
+		str, ok := val.(string)
+		if !ok || str == "" {
+			return "", fmt.Errorf("name must be non-empty string")
+		}
+		return str, nil
+	}
+
+	val, ok := item["name"].(string)
+	if !ok || val == "" {
+		return "", fmt.Errorf("name must be non-empty string")
+	}
+	return val, nil
+}
+
+// getAddress extracts the target address from the item using the compiled CEL expression if provided,
+// otherwise it falls back to the default "address" field
+func (l *Loader) getAddress(item map[string]interface{}, full interface{}, cm *compiledMapping) (string, error) {
+	if cm.address != nil {
+		val, err := evalCEL(cm.address, item, full)
+		if err != nil {
+			return "", err
+		}
+
+		str, ok := val.(string)
+		if !ok || str == "" {
+			return "", fmt.Errorf("address must be non-empty string")
+		}
+		return str, nil
+	}
+
+	val, ok := item["address"].(string)
+	if !ok || val == "" {
+		return "", fmt.Errorf("address must be non-empty string")
+	}
+	return val, nil
+}
+
+// getPort extracts the target port from the item using the compiled CEL expression if provided,
+// otherwise it falls back to the default "port" field
+func (l *Loader) getPort(item map[string]interface{}, full interface{}, cm *compiledMapping) int32 {
+	if cm.port != nil {
+		val, err := evalCEL(cm.port, item, full)
+		if err == nil {
+			return extractPort(val)
+		}
+		return 0
+	}
+
+	return extractPort(item["port"])
+}
+
+// getLabels extracts the target labels from the item using the compiled CEL expressions if provided,
+// otherwise it falls back to the default "labels" field
+func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *compiledMapping) map[string]string {
+	labels := make(map[string]string)
+
+	if len(cm.labels) > 0 {
+		for label, prog := range cm.labels {
+			val, err := evalCEL(prog, item, full)
+			if err == nil {
+				labels[label] = fmt.Sprintf("%v", val)
+			}
+		}
+		return labels
+	}
+
+	if raw, ok := item["labels"].(map[string]interface{}); ok {
+		for key, val := range raw {
+			labels[key] = fmt.Sprintf("%v", val)
+		}
+	}
+	return labels
+}
+
+// getTargetProfile extracts the target profile from the item using the compiled CEL expression if provided,
+// otherwise it falls back to the default "targetProfile" field
+func (l *Loader) getTargetProfile(item map[string]interface{}, full interface{}, cm *compiledMapping) string {
+	if cm.targetProfile != nil {
+		val, err := evalCEL(cm.targetProfile, item, full)
+		if err == nil {
+			if str, ok := val.(string); ok {
+				return str
+			}
+		}
+		return ""
+	}
+
+	if val, ok := item["targetProfile"].(string); ok {
+		return val
+	}
+	return ""
+}
+
+var celEnv = mustNewEnv()
+
+// mustNewEnv creates a CEL environment with the necessary variable declarations for evaluating expressions
+func mustNewEnv() *cel.Env {
+	env, err := cel.NewEnv(
+		cel.Variable("self", cel.DynType),
+		cel.Variable("item", cel.DynType),
+	)
+	if err != nil {
+		panic(err)
+	}
+	return env
+}
+
+// compileCEL compiles a CEL expression into a program that can be evaluated against items
+func compileCEL(expr string) (cel.Program, error) {
+	ast, issues := celEnv.Compile(expr)
+	if issues != nil && issues.Err() != nil {
+		return nil, issues.Err()
+	}
+	return celEnv.Program(ast, cel.EvalOptions(cel.OptOptimize))
+}
+
+// evalCEL evaluates a compiled CEL program against an item
+func evalCEL(p cel.Program, item map[string]interface{}, full interface{}) (interface{}, error) {
+	out, _, err := p.Eval(map[string]interface{}{
+		"self": full,
+		"item": item,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if out == nil {
+		return nil, fmt.Errorf("CEL returned nil")
+	}
+	return out.Value(), nil
+}
+
+// extractPort converts a CEL evaluation result into an int32 port number,
+// handling both numeric and string representations
 func extractPort(val interface{}) int32 {
 	switch v := val.(type) {
 	case float64:
@@ -73,12 +268,14 @@ func extractPort(val interface{}) int32 {
 			return 0
 		}
 		return int32(v)
+
 	case string:
 		p, err := strconv.ParseInt(v, 10, 32)
 		if err != nil {
 			return 0
 		}
 		return int32(p)
+
 	default:
 		return 0
 	}
diff --git a/internal/controller/discovery/loaders/http/mapper_direct.go b/internal/controller/discovery/loaders/http/mapper_direct.go
deleted file mode 100644
index 185e1cb..0000000
--- a/internal/controller/discovery/loaders/http/mapper_direct.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package http
-
-import (
-	"fmt"
-)
-
-// directGetter extracts values via direct map access
-// Example input:
-//
-//	{
-//	  "name": "router1",
-//	  "ip": "10.0.0.1",
-//	  "port": 57400,
-//	  "labels": { ... },
-//	  "targetProfile": "profile1"
-//	}
-type directGetter struct {
-	item map[string]interface{}
-}
-
-// GetName extracts the "name" field directly
-func (g *directGetter) GetName() (string, error) {
-	val, ok := g.item["name"].(string)
-	if !ok || val == "" {
-		return "", fmt.Errorf("name must be a non-empty string")
-	}
-	return val, nil
-}
-
-// GetIP extracts the "ip" field directly.
-func (g *directGetter) GetIP() (string, error) {
-	val, ok := g.item["ip"].(string)
-	if !ok || val == "" {
-		return "", fmt.Errorf("ip must be a non-empty string")
-	}
-	return val, nil
-}
-
-// GetPort extracts and normalizes the "port" field
-//
-// Behavior:
-// - supports int, float64, string
-// - returns 0 if value is missing or invalid
-func (g *directGetter) GetPort() int32 {
-	if val, ok := g.item["port"]; ok {
-		return extractPort(val)
-	}
-	return 0
-}
-
-// GetLabels extracts labels from the "labels" field
-// Expected format:
-//
-//	"labels": {
-//	  "key": "value"
-//	}
-//
-// Non-string values are converted to string
-func (g *directGetter) GetLabels() map[string]string {
-	labels := make(map[string]string)
-
-	if val, ok := g.item["labels"]; ok {
-		if m, ok := val.(map[string]interface{}); ok {
-			for k, v := range m {
-				labels[k] = fmt.Sprintf("%v", v)
-			}
-		}
-	}
-
-	return labels
-}
-
-// GetTargetProfile extracts the "targetProfile" field directly
-//
-// Behavior:
-// - returns "" if value is missing or invalid
-func (g *directGetter) GetTargetProfile() string {
-	if val, ok := g.item["targetProfile"].(string); ok {
-		return val
-	}
-	return ""
-}
diff --git a/internal/controller/discovery/loaders/http/mapper_jsonpath.go b/internal/controller/discovery/loaders/http/mapper_jsonpath.go
deleted file mode 100644
index c82534d..0000000
--- a/internal/controller/discovery/loaders/http/mapper_jsonpath.go
+++ /dev/null
@@ -1,106 +0,0 @@
-package http
-
-import (
-	"fmt"
-
-	"github.com/PaesslerAG/jsonpath"
-	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
-)
-
-// jsonPathGetter extracts values using JSONPath expressions defined in the CR
-// Example mapping:
-//
-//	name: "$.hostname"
-//	ip: "$.ip"
-//	port: "$.port"
-//	labels:
-//	  rack: "$.meta.rack"
-type jsonPathGetter struct {
-	item map[string]interface{}
-	spec *gnmicv1alpha1.ResponseMappingSpec
-}
-
-// helper function to execute JSONPath queries
-func (g *jsonPathGetter) get(expr string) (interface{}, error) {
-	return jsonpath.Get(expr, g.item)
-}
-
-// GetName extracts the target name using JSONPath
-func (g *jsonPathGetter) GetName() (string, error) {
-	val, err := g.get(g.spec.Name)
-	if err != nil {
-		return "", fmt.Errorf("name mapping failed: %w", err)
-	}
-
-	str, ok := val.(string)
-	if !ok || str == "" {
-		return "", fmt.Errorf("name must be a non-empty string")
-	}
-
-	return str, nil
-}
-
-// GetIP extracts the IP using JSONPath
-func (g *jsonPathGetter) GetIP() (string, error) {
-	val, err := g.get(g.spec.Address)
-	if err != nil {
-		return "", fmt.Errorf("IP mapping failed: %w", err)
-	}
-
-	str, ok := val.(string)
-	if !ok || str == "" {
-		return "", fmt.Errorf("IP must be a non-empty string")
-	}
-
-	return str, nil
-}
-
-// GetPort extracts the port using JSONPath
-//
-// Behavior:
-// - returns 0 if no port mapping defined
-// - returns 0 if extraction fails or value invalid
-func (g *jsonPathGetter) GetPort() int32 {
-	if g.spec.Port == "" {
-		return 0
-	}
-
-	val, err := g.get(g.spec.Port)
-	if err != nil {
-		return 0
-	}
-
-	return extractPort(val)
-}
-
-// GetLabels extracts labels using JSONPath expressions defined per label key
-func (g *jsonPathGetter) GetLabels() map[string]string {
-	labels := make(map[string]string)
-
-	for key, expr := range g.spec.Labels {
-		if val, err := g.get(expr); err == nil {
-			labels[key] = fmt.Sprintf("%v", val)
-		}
-	}
-
-	return labels
-}
-
-// GetTargetProfile extracts the target profile using JSONPath
-//
-// Behavior:
-// - returns "" if no target profile mapping defined
-// - returns "" if extraction fails or value invalid
-func (g *jsonPathGetter) GetTargetProfile() string {
-	val, err := g.get(g.spec.TargetProfile)
-	if err != nil {
-		return ""
-	}
-
-	str, ok := val.(string)
-	if !ok {
-		return ""
-	}
-
-	return str
-}

From 1a486b1cd16af2d6daffdec58ef917a5de1b99ec Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Fri, 22 May 2026 20:21:07 +0000
Subject: [PATCH 155/165] update targetsource base

---
 .../operator.gnmic.dev_targetsources.yaml     | 71 +++++++++++++++----
 1 file changed, 56 insertions(+), 15 deletions(-)

diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index d9c9184..ca6104b 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -138,38 +138,79 @@ spec:
                         properties:
                           address:
                             description: |-
-                              CEL expression to extract the target Address from the response
-                              If TargetsField is specified, this should be relative to TargetsField
+                              CEL expression for the target address.
+
+                              If not set, defaults to:
+                                item["address"]
+
+                              Example:
+                                "item.ip"
                             type: string
                           labels:
                             additionalProperties:
                               type: string
                             description: |-
-                              CEL expression to extract the target labels from the response
-                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
-                              with values from the response taking precedence in case of conflicts.
+                              Defines labels to attach to the target.
+
+                              Each entry defines:
+                                key   -> label name
+                                value -> CEL expression
+
+                              Expressions can use both:
+                                item -> current target
+                                self -> full response
+
+                              Example:
+                                labels:
+                                  env:    "item.environment"
+                                  region: "self.meta.region"
+
+                              If not set, defaults to:
+                                item["labels"]
+
+                              Dynamic labels override static labels defined in the TargetSource.
                             type: object
                           name:
                             description: |-
-                              CEL expression to extract the target name from the response
-                              If TargetsField is specified, this should be relative to TargetsField
+                              CEL expression for the target name.
+
+                              If not set, defaults to:
+                                item["name"]
+
+                              Example:
+                                "item.hostname"
                             type: string
                           port:
                             description: |-
-                              CEL expression to extract the target port from the response
-                              If TargetsField is specified, this should be relative to TargetsField
+                              CEL expression for the target port.
+
+                              If not set, defaults to:
+                                item["port"]
+
+                              Example:
+                                "item.port"
                             type: string
                           targetProfile:
                             description: |-
-                              CEL expression to extract the target profile from the response
-                              If TargetsField is specified, this should be relative to TargetsField
+                              CEL expression for the target profile.
+
+                              If not set, defaults to:
+                                item["targetProfile"]
+
+                              Example:
+                                "item.type == 'edge' ? 'edge-profile' : 'default'"
                             type: string
                           targetsField:
                             description: |-
-                              Field name in the JSON response that contains the list of items (targets).
-                              If not specified, the entire response is expected to be a list of items.
-                              All subsequent fields are specified relative to this field
-                              Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
+                              CEL expression that selects the list of target objects from the response.
+
+                              This is evaluated once using:
+                                self -> full JSON response
+
+                              Example:
+                                targetsField: "self.results"
+
+                              If not set, the response itself must be a JSON array with the targets.
                             type: string
                         type: object
                       pagination:

From b931a02541bcb5ba58d4ad8c73daaf6a8d9c894c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Sat, 23 May 2026 07:34:19 +0000
Subject: [PATCH 156/165] mapping of labels via CEL

---
 api/v1alpha1/targetsource_types.go            | 27 ++++++------
 api/v1alpha1/zz_generated.deepcopy.go         |  9 +---
 .../operator.gnmic.dev_targetsources.yaml     | 29 ++++++-------
 .../discovery/loaders/http/mapper.go          | 41 ++++++++++---------
 4 files changed, 49 insertions(+), 57 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 426731d..c96f8fa 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -234,28 +234,27 @@ type ResponseMappingSpec struct {
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// Defines labels to attach to the target.
-	//
-	// Each entry defines:
-	//   key   -> label name
-	//   value -> CEL expression
-	//
-	// Expressions can use both:
-	//   item -> current target
-	//   self -> full response
+	// CEL expression that returns a map of labels.
+	// The expression must evaluate to an object (map).
 	//
 	// Example:
-	//   labels:
-	//     env:    "item.environment"
-	//     region: "self.meta.region"
+	//
+	//   labels: |
+	//     {
+	//       "env": item.environment,
+	//       "region": self.meta.region,
+	//       item.dynamicKey: "value"
+	//     }
 	//
 	// If not set, defaults to:
 	//   item["labels"]
 	//
-	// Dynamic labels override static labels defined in the TargetSource.
+	// The resulting map will be converted into labels.
+	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+	// with values from the response taking precedence in case of conflicts.
 	//
 	// +kubebuilder:validation:Optional
-	Labels map[string]string `json:"labels,omitempty"`
+	Labels string `json:"labels,omitempty"`
 
 	// CEL expression for the target profile.
 	//
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 9051fc7..3043155 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -351,7 +351,7 @@ func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	if in.ResponseMapping != nil {
 		in, out := &in.ResponseMapping, &out.ResponseMapping
 		*out = new(ResponseMappingSpec)
-		(*in).DeepCopyInto(*out)
+		**out = **in
 	}
 	if in.Push != nil {
 		in, out := &in.Push, &out.Push
@@ -1026,13 +1026,6 @@ func (in *PushSpec) DeepCopy() *PushSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ResponseMappingSpec) DeepCopyInto(out *ResponseMappingSpec) {
 	*out = *in
-	if in.Labels != nil {
-		in, out := &in.Labels, &out.Labels
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResponseMappingSpec.
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index ca6104b..8232ec3 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -147,29 +147,26 @@ spec:
                                 "item.ip"
                             type: string
                           labels:
-                            additionalProperties:
-                              type: string
                             description: |-
-                              Defines labels to attach to the target.
-
-                              Each entry defines:
-                                key   -> label name
-                                value -> CEL expression
-
-                              Expressions can use both:
-                                item -> current target
-                                self -> full response
+                              CEL expression that returns a map of labels.
+                              The expression must evaluate to an object (map).
 
                               Example:
-                                labels:
-                                  env:    "item.environment"
-                                  region: "self.meta.region"
+
+                                labels: |
+                                  {
+                                    "env": item.environment,
+                                    "region": self.meta.region,
+                                    item.dynamicKey: "value"
+                                  }
 
                               If not set, defaults to:
                                 item["labels"]
 
-                              Dynamic labels override static labels defined in the TargetSource.
-                            type: object
+                              The resulting map will be converted into labels.
+                              The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+                              with values from the response taking precedence in case of conflicts.
+                            type: string
                           name:
                             description: |-
                               CEL expression for the target name.
diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index a14b5cb..a89196e 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -50,14 +50,12 @@ type compiledMapping struct {
 	port    cel.Program
 
 	targetProfile cel.Program
-	labels        map[string]cel.Program
+	labels        cel.Program
 }
 
 func (l *Loader) compileMapping() (*compiledMapping, error) {
 	rm := l.spec.ResponseMapping
-	cm := &compiledMapping{
-		labels: make(map[string]cel.Program),
-	}
+	cm := &compiledMapping{}
 	if rm == nil {
 		return cm, nil
 	}
@@ -87,12 +85,11 @@ func (l *Loader) compileMapping() (*compiledMapping, error) {
 			return nil, fmt.Errorf("targetProfile: %w", err)
 		}
 	}
-	for key, expr := range rm.Labels {
-		p, err := compileCEL(expr)
+	if rm.Labels != "" {
+		cm.labels, err = compileCEL(rm.Labels)
 		if err != nil {
-			return nil, fmt.Errorf("label %s: %w", key, err)
+			return nil, fmt.Errorf("labels: %w", err)
 		}
-		cm.labels[key] = p
 	}
 
 	return cm, nil
@@ -182,24 +179,30 @@ func (l *Loader) getPort(item map[string]interface{}, full interface{}, cm *comp
 // getLabels extracts the target labels from the item using the compiled CEL expressions if provided,
 // otherwise it falls back to the default "labels" field
 func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *compiledMapping) map[string]string {
-	labels := make(map[string]string)
+	result := make(map[string]string)
 
-	if len(cm.labels) > 0 {
-		for label, prog := range cm.labels {
-			val, err := evalCEL(prog, item, full)
-			if err == nil {
-				labels[label] = fmt.Sprintf("%v", val)
-			}
+	if cm != nil && cm.labels != nil {
+		val, err := evalCEL(cm.labels, item, full)
+		if err != nil {
+			return result
+		}
+		raw, ok := val.(map[string]interface{})
+		if !ok {
+			return result
+		}
+		for k, v := range raw {
+			result[k] = fmt.Sprintf("%v", v)
 		}
-		return labels
+		return result
 	}
 
+	// fallback: direct
 	if raw, ok := item["labels"].(map[string]interface{}); ok {
-		for key, val := range raw {
-			labels[key] = fmt.Sprintf("%v", val)
+		for k, v := range raw {
+			result[k] = fmt.Sprintf("%v", v)
 		}
 	}
-	return labels
+	return result
 }
 
 // getTargetProfile extracts the target profile from the item using the compiled CEL expression if provided,

From c2d49cb012b24136fb4ac921a799f9b55764b3c8 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 07:47:13 +0000
Subject: [PATCH 157/165] fix nil pointer issue

---
 internal/controller/discovery/loaders.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go
index 4ecf9dd..2644db3 100644
--- a/internal/controller/discovery/loaders.go
+++ b/internal/controller/discovery/loaders.go
@@ -17,7 +17,7 @@ func NewLoader(ctx context.Context, c client.Client, cfg *core.CommonLoaderConfi
 	switch {
 	case spec.Provider.HTTP != nil:
 		httpSpec := *spec.Provider.HTTP
-		cfg.AcceptPush = httpSpec.Push.Enabled
+		cfg.AcceptPush = httpSpec.Push != nil && httpSpec.Push.Enabled
 		cfg.ResourceFetcher = newK8sResourceFetcher(c)
 		return http.New(*cfg, httpSpec), nil
 	default:

From a9ddacd9e6fd9325a846e26888001f4a2593419a Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 07:47:33 +0000
Subject: [PATCH 158/165] remove missleading old comment

---
 api/v1alpha1/targetsource_types.go                     | 1 -
 config/crd/bases/operator.gnmic.dev_targetsources.yaml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index c96f8fa..66265ab 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -129,7 +129,6 @@ type TokenAuthSpec struct {
 	Scheme string `json:"scheme"`
 	// Reference to a Secret containing a key with the token value to use for
 	// authentication when connecting to the Provider.
-	// Mutually exclusive with Token.
 	// +kubebuilder:validation:Required
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 8232ec3..468fa76 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -94,7 +94,6 @@ spec:
                                 description: |-
                                   Reference to a Secret containing a key with the token value to use for
                                   authentication when connecting to the Provider.
-                                  Mutually exclusive with Token.
                                 properties:
                                   key:
                                     description: The key of the secret to select from.  Must

From f4fcf789d77dfe89780ee492e945f3c6199748bd Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 08:29:08 +0000
Subject: [PATCH 159/165] enable CEL extensions for advanced mapping

---
 internal/controller/discovery/loaders/http/mapper.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index a89196e..0d0ec26 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -8,6 +8,7 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/go-logr/logr"
 	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/ext"
 )
 
 // mapItemsToTargets converts a list of raw JSON items into DiscoveredTargets using the configured mapping rules
@@ -231,6 +232,14 @@ func mustNewEnv() *cel.Env {
 	env, err := cel.NewEnv(
 		cel.Variable("self", cel.DynType),
 		cel.Variable("item", cel.DynType),
+		// TODO: document what extensions are included
+		// Include standard CEL declarations for common operations and types
+		ext.Strings(),
+		ext.Math(),
+		ext.Lists(),
+		ext.Sets(),
+		ext.Regex(),
+		ext.Bindings(),
 	)
 	if err != nil {
 		panic(err)

From 0995dfe31bc6ac2406da0dd43519b180a74a1084 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 08:44:23 +0000
Subject: [PATCH 160/165] fix: add optional types for CEL regex

---
 internal/controller/discovery/loaders/http/mapper.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index 0d0ec26..b7ba634 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -240,6 +240,8 @@ func mustNewEnv() *cel.Env {
 		ext.Sets(),
 		ext.Regex(),
 		ext.Bindings(),
+		// Required for ext.Regex
+		cel.OptionalTypes(),
 	)
 	if err != nil {
 		panic(err)

From 5c337e64b787d56b81aac6ac66b24f850071980c Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 09:30:23 +0000
Subject: [PATCH 161/165] fix CEL OptionalTypes order

---
 internal/controller/discovery/loaders/http/mapper.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index b7ba634..8ff34a4 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -232,6 +232,8 @@ func mustNewEnv() *cel.Env {
 	env, err := cel.NewEnv(
 		cel.Variable("self", cel.DynType),
 		cel.Variable("item", cel.DynType),
+		// Required for ext.Regex
+		cel.OptionalTypes(),
 		// TODO: document what extensions are included
 		// Include standard CEL declarations for common operations and types
 		ext.Strings(),
@@ -240,8 +242,6 @@ func mustNewEnv() *cel.Env {
 		ext.Sets(),
 		ext.Regex(),
 		ext.Bindings(),
-		// Required for ext.Regex
-		cel.OptionalTypes(),
 	)
 	if err != nil {
 		panic(err)

From eecee725f5eb413af8c9b24b9c9fb296a10051c3 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 09:33:13 +0000
Subject: [PATCH 162/165] add support for POST and custom headers and body

---
 api/v1alpha1/targetsource_types.go            | 45 ++++++++++++++++++
 api/v1alpha1/zz_generated.deepcopy.go         |  7 +++
 .../operator.gnmic.dev_targetsources.yaml     | 46 +++++++++++++++++++
 .../discovery/loaders/http/loader.go          | 28 +++++++++--
 4 files changed, 123 insertions(+), 3 deletions(-)

diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go
index 66265ab..d530033 100644
--- a/api/v1alpha1/targetsource_types.go
+++ b/api/v1alpha1/targetsource_types.go
@@ -60,6 +60,51 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	URL string `json:"url,omitempty"`
 
+	// HTTP method used for the request.
+	//
+	// Defaults to GET if not specified.
+	//
+	// Supported values:
+	// - GET  (default, no request body)
+	// - POST (supports request body)
+	//
+	// +kubebuilder:validation:Enum=GET;POST
+	// +kubebuilder:validation:Optional
+	Method string `json:"method,omitempty"`
+
+	// Optional HTTP headers to include in the request.
+	//
+	// These map directly to HTTP headers (key-value pairs).
+	//
+	// Example:
+	//   headers:
+	//     Content-Type: application/json
+	//     X-Custom-Header: value
+	//
+	// Precedence:
+	// - Authorization configuration overrides any conflicting headers
+	//
+	// +kubebuilder:validation:Optional
+	Headers map[string]string `json:"headers,omitempty"`
+
+	// Optional raw request body.
+	//
+	// Typically used with POST requests and contains JSON payload.
+	//
+	// Example:
+	//   body: |
+	//     {
+	//       "limit": 100,
+	//       "status": "active"
+	//     }
+	//
+	// Notes:
+	// - Ignored for GET requests
+	// - User must set appropriate Content-Type header if needed
+	//
+	// +kubebuilder:validation:Optional
+	Body string `json:"body,omitempty"`
+
 	// Optional authorization configuration for accessing the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 3043155..1628621 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -323,6 +323,13 @@ func (in *GRPCTunnelConfig) DeepCopy() *GRPCTunnelConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HTTPConfig) DeepCopyInto(out *HTTPConfig) {
 	*out = *in
+	if in.Headers != nil {
+		in, out := &in.Headers, &out.Headers
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.Authorization != nil {
 		in, out := &in.Authorization, &out.Authorization
 		*out = new(AuthorizationSpec)
diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
index 468fa76..4fc6c32 100644
--- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml
+++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml
@@ -126,6 +126,39 @@ spec:
                             be set
                           rule: '[has(self.basic),has(self.token)].filter(x,x==true).size()
                             == 1'
+                      body:
+                        description: |-
+                          Optional raw request body.
+
+                          Typically used with POST requests and contains JSON payload.
+
+                          Example:
+                            body: |
+                              {
+                                "limit": 100,
+                                "status": "active"
+                              }
+
+                          Notes:
+                          - Ignored for GET requests
+                          - User must set appropriate Content-Type header if needed
+                        type: string
+                      headers:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Optional HTTP headers to include in the request.
+
+                          These map directly to HTTP headers (key-value pairs).
+
+                          Example:
+                            headers:
+                              Content-Type: application/json
+                              X-Custom-Header: value
+
+                          Precedence:
+                          - Authorization configuration overrides any conflicting headers
+                        type: object
                       interval:
                         default: 6h
                         description: Optional interval for polling the HTTP endpoint
@@ -209,6 +242,19 @@ spec:
                               If not set, the response itself must be a JSON array with the targets.
                             type: string
                         type: object
+                      method:
+                        description: |-
+                          HTTP method used for the request.
+
+                          Defaults to GET if not specified.
+
+                          Supported values:
+                          - GET  (default, no request body)
+                          - POST (supports request body)
+                        enum:
+                        - GET
+                        - POST
+                        type: string
                       pagination:
                         description: Optional pagination configuration for parsing
                           responses from the HTTP endpoint
diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 341feac..7891d23 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -1,6 +1,7 @@
 package http
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
@@ -177,7 +178,7 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 	currentURL := l.spec.URL
 
 	for {
-		raw, err := l.fetchPage(ctx, client, currentURL)
+		raw, err := l.fetchPage(ctx, client, currentURL, logger)
 		if err != nil {
 			logger.Error(err,
 				"Failed to fetch page from HTTP endpoint",
@@ -209,13 +210,34 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 
 // fetchPage performs an HTTP GET request to the specified URL and decodes the JSON response
 // and returns the raw response as an interface{}
-func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string) (interface{}, error) {
+func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string, logger logr.Logger) (interface{}, error) {
+	// Determine HTTP method (default GET)
+	method := l.spec.Method
+	if method == "" {
+		method = http.MethodGet
+	}
+
+	// Build request body (only for POST)
+	if method == http.MethodGet && l.spec.Body != "" {
+		logger.Info("ignoring body for GET request")
+	}
+	var bodyReader *bytes.Reader
+	if method == http.MethodPost && l.spec.Body != "" {
+		bodyReader = bytes.NewReader([]byte(l.spec.Body))
+	} else {
+		bodyReader = bytes.NewReader(nil)
+	}
+
 	// Build HTTP request
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
 	if err != nil {
 		return nil, fmt.Errorf("creating HTTP request failed: %w", err)
 	}
 	req.Header.Set("Accept", "application/json")
+	// Apply user-defined headers
+	for key, val := range l.spec.Headers {
+		req.Header.Set(key, val)
+	}
 	if err := l.applyAuthorization(req); err != nil {
 		return nil, fmt.Errorf("applying authorization to HTTP request failed: %w", err)
 	}

From 8a17aad49d457d871d4ce9f9c855ab6722e1b549 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 10:04:56 +0000
Subject: [PATCH 163/165] Handle different map representations returned by CEL

---
 .../discovery/loaders/http/mapper.go          | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index 8ff34a4..5daefe7 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -187,14 +187,22 @@ func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *co
 		if err != nil {
 			return result
 		}
-		raw, ok := val.(map[string]interface{})
-		if !ok {
+		// Handle different map representations returned by CEL
+		// Labels must be a map of string keys and string values
+		switch labels := val.(type) {
+		case map[string]interface{}:
+			for key, val := range labels {
+				result[key] = fmt.Sprintf("%v", val)
+			}
+			return result
+		case map[interface{}]interface{}:
+			for key, val := range labels {
+				result[fmt.Sprintf("%v", key)] = fmt.Sprintf("%v", val)
+			}
+			return result
+		default:
 			return result
 		}
-		for k, v := range raw {
-			result[k] = fmt.Sprintf("%v", v)
-		}
-		return result
 	}
 
 	// fallback: direct

From 1316082dec1c55eb133c173a0f7e253fe7c8ffff Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 11:01:49 +0000
Subject: [PATCH 164/165] changed interface{} to any

---
 .../discovery/loaders/http/loader.go          | 18 +++++------
 .../discovery/loaders/http/mapper.go          | 32 +++++++++----------
 .../discovery/loaders/http/pagination.go      |  4 +--
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go
index 7891d23..b8eb1ef 100644
--- a/internal/controller/discovery/loaders/http/loader.go
+++ b/internal/controller/discovery/loaders/http/loader.go
@@ -209,8 +209,8 @@ func (l *Loader) fetchTargetsFromHTTPEndpoint(
 }
 
 // fetchPage performs an HTTP GET request to the specified URL and decodes the JSON response
-// and returns the raw response as an interface{}
-func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string, logger logr.Logger) (interface{}, error) {
+// and returns the raw response
+func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string, logger logr.Logger) (any, error) {
 	// Determine HTTP method (default GET)
 	method := l.spec.Method
 	if method == "" {
@@ -253,7 +253,7 @@ func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string,
 	}
 
 	// Decode HTTP response
-	var raw interface{}
+	var raw any
 	if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
 		return nil, fmt.Errorf("failed to decode HTTP response: %w", err)
 	}
@@ -262,8 +262,8 @@ func (l *Loader) fetchPage(ctx context.Context, client *http.Client, url string,
 }
 
 // extractTargetsFromResponse extracts items from the response and maps each item into a DiscoveredTarget
-func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
-	var items []interface{}
+func (l *Loader) extractTargetsFromResponse(raw any, logger logr.Logger) ([]core.DiscoveredTarget, error) {
+	var items []any
 	// If ResponseMapping is configured and TargetsField is provided we treat
 	// it as a CEL expression that evaluates against the whole response and
 	// must return an array of items.
@@ -272,21 +272,21 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger)
 		if err != nil {
 			return nil, fmt.Errorf("invalid TargetsField CEL expression: %w", err)
 		}
-		out, _, err := prog.Eval(map[string]interface{}{"self": raw})
+		out, _, err := prog.Eval(map[string]any{"self": raw})
 		if err != nil {
 			return nil, fmt.Errorf("evaluating TargetsField CEL expression failed: %w", err)
 		}
 		if out == nil {
 			return nil, fmt.Errorf("TargetsField expression returned nil")
 		}
-		array, ok := out.Value().([]interface{})
+		array, ok := out.Value().([]any)
 		if !ok {
 			return nil, fmt.Errorf("invalid HTTP response: targetsField expression must evaluate to an array of objects")
 		}
 		items = array
 	} else {
 		//If TargetsField is empty, the raw response is expected to be an array of items.
-		array, ok := raw.([]interface{})
+		array, ok := raw.([]any)
 		if !ok {
 			return nil, fmt.Errorf("invalid HTTP response: expected a JSON array when itemsField is not set")
 		}
@@ -308,7 +308,7 @@ func (l *Loader) extractTargetsFromResponse(raw interface{}, logger logr.Logger)
 // - nextURL: next request
 // - stop: whether to terminate loop
 func (l *Loader) getNextURL(
-	raw interface{},
+	raw any,
 	currentURL string,
 	logger logr.Logger,
 ) (string, bool) {
diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index 5daefe7..ce1e2aa 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -12,7 +12,7 @@ import (
 )
 
 // mapItemsToTargets converts a list of raw JSON items into DiscoveredTargets using the configured mapping rules
-func (l *Loader) mapItemsToTargets(items []interface{}, full interface{}, logger logr.Logger) ([]core.DiscoveredTarget, error) {
+func (l *Loader) mapItemsToTargets(items []any, full any, logger logr.Logger) ([]core.DiscoveredTarget, error) {
 	// Compile CEL expressions once for efficiency
 	compiled, err := l.compileMapping()
 	if err != nil {
@@ -22,7 +22,7 @@ func (l *Loader) mapItemsToTargets(items []interface{}, full interface{}, logger
 	// Map items to targets
 	targets := make([]core.DiscoveredTarget, 0, len(items))
 	for _, item := range items {
-		obj, ok := item.(map[string]interface{})
+		obj, ok := item.(map[string]any)
 		if !ok {
 			logger.Error(fmt.Errorf("invalid target format"),
 				"failed to convert target to map",
@@ -97,7 +97,7 @@ func (l *Loader) compileMapping() (*compiledMapping, error) {
 }
 
 // mapItemToTarget converts a raw JSON object into a DiscoveredTarget
-func (l *Loader) mapItemToTarget(item map[string]interface{}, full interface{}, cm *compiledMapping) (core.DiscoveredTarget, error) {
+func (l *Loader) mapItemToTarget(item map[string]any, full any, cm *compiledMapping) (core.DiscoveredTarget, error) {
 	name, err := l.getName(item, full, cm)
 	if err != nil {
 		return core.DiscoveredTarget{}, err
@@ -119,7 +119,7 @@ func (l *Loader) mapItemToTarget(item map[string]interface{}, full interface{},
 
 // getName extracts the target name from the item using the compiled CEL expression if provided,
 // otherwise it falls back to the default "name" field
-func (l *Loader) getName(item map[string]interface{}, full interface{}, cm *compiledMapping) (string, error) {
+func (l *Loader) getName(item map[string]any, full any, cm *compiledMapping) (string, error) {
 	if cm.name != nil {
 		val, err := evalCEL(cm.name, item, full)
 		if err != nil {
@@ -142,7 +142,7 @@ func (l *Loader) getName(item map[string]interface{}, full interface{}, cm *comp
 
 // getAddress extracts the target address from the item using the compiled CEL expression if provided,
 // otherwise it falls back to the default "address" field
-func (l *Loader) getAddress(item map[string]interface{}, full interface{}, cm *compiledMapping) (string, error) {
+func (l *Loader) getAddress(item map[string]any, full any, cm *compiledMapping) (string, error) {
 	if cm.address != nil {
 		val, err := evalCEL(cm.address, item, full)
 		if err != nil {
@@ -165,7 +165,7 @@ func (l *Loader) getAddress(item map[string]interface{}, full interface{}, cm *c
 
 // getPort extracts the target port from the item using the compiled CEL expression if provided,
 // otherwise it falls back to the default "port" field
-func (l *Loader) getPort(item map[string]interface{}, full interface{}, cm *compiledMapping) int32 {
+func (l *Loader) getPort(item map[string]any, full any, cm *compiledMapping) int32 {
 	if cm.port != nil {
 		val, err := evalCEL(cm.port, item, full)
 		if err == nil {
@@ -179,7 +179,7 @@ func (l *Loader) getPort(item map[string]interface{}, full interface{}, cm *comp
 
 // getLabels extracts the target labels from the item using the compiled CEL expressions if provided,
 // otherwise it falls back to the default "labels" field
-func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *compiledMapping) map[string]string {
+func (l *Loader) getLabels(item map[string]any, full any, cm *compiledMapping) map[string]string {
 	result := make(map[string]string)
 
 	if cm != nil && cm.labels != nil {
@@ -190,12 +190,12 @@ func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *co
 		// Handle different map representations returned by CEL
 		// Labels must be a map of string keys and string values
 		switch labels := val.(type) {
-		case map[string]interface{}:
+		case map[string]any:
 			for key, val := range labels {
 				result[key] = fmt.Sprintf("%v", val)
 			}
 			return result
-		case map[interface{}]interface{}:
+		case map[any]any:
 			for key, val := range labels {
 				result[fmt.Sprintf("%v", key)] = fmt.Sprintf("%v", val)
 			}
@@ -206,9 +206,9 @@ func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *co
 	}
 
 	// fallback: direct
-	if raw, ok := item["labels"].(map[string]interface{}); ok {
-		for k, v := range raw {
-			result[k] = fmt.Sprintf("%v", v)
+	if raw, ok := item["labels"].(map[string]any); ok {
+		for key, val := range raw {
+			result[key] = fmt.Sprintf("%v", val)
 		}
 	}
 	return result
@@ -216,7 +216,7 @@ func (l *Loader) getLabels(item map[string]interface{}, full interface{}, cm *co
 
 // getTargetProfile extracts the target profile from the item using the compiled CEL expression if provided,
 // otherwise it falls back to the default "targetProfile" field
-func (l *Loader) getTargetProfile(item map[string]interface{}, full interface{}, cm *compiledMapping) string {
+func (l *Loader) getTargetProfile(item map[string]any, full any, cm *compiledMapping) string {
 	if cm.targetProfile != nil {
 		val, err := evalCEL(cm.targetProfile, item, full)
 		if err == nil {
@@ -267,8 +267,8 @@ func compileCEL(expr string) (cel.Program, error) {
 }
 
 // evalCEL evaluates a compiled CEL program against an item
-func evalCEL(p cel.Program, item map[string]interface{}, full interface{}) (interface{}, error) {
-	out, _, err := p.Eval(map[string]interface{}{
+func evalCEL(p cel.Program, item map[string]any, full any) (any, error) {
+	out, _, err := p.Eval(map[string]any{
 		"self": full,
 		"item": item,
 	})
@@ -283,7 +283,7 @@ func evalCEL(p cel.Program, item map[string]interface{}, full interface{}) (inte
 
 // extractPort converts a CEL evaluation result into an int32 port number,
 // handling both numeric and string representations
-func extractPort(val interface{}) int32 {
+func extractPort(val any) int32 {
 	switch v := val.(type) {
 	case float64:
 		if v < 0 || v > math.MaxInt32 {
diff --git a/internal/controller/discovery/loaders/http/pagination.go b/internal/controller/discovery/loaders/http/pagination.go
index 49ddf9b..7f7fd51 100644
--- a/internal/controller/discovery/loaders/http/pagination.go
+++ b/internal/controller/discovery/loaders/http/pagination.go
@@ -6,13 +6,13 @@ import (
 )
 
 // extractNextPageInfo extracts pagination information from a response
-func (l *Loader) extractNextPageInfo(raw interface{}) (string, error) {
+func (l *Loader) extractNextPageInfo(raw any) (string, error) {
 	if l.spec.Pagination == nil || l.spec.Pagination.NextField == "" {
 		return "", nil
 	}
 
 	// Only objects can have "next" fields
-	obj, ok := raw.(map[string]interface{})
+	obj, ok := raw.(map[string]any)
 	if !ok {
 		// array case -> no pagination
 		return "", nil

From c4a553c1dfe881f25993b47bdcc007a0e9db8a27 Mon Sep 17 00:00:00 2001
From: Daniel Schatzmann <daniel.schatzmann@ost.ch>
Date: Mon, 25 May 2026 12:15:42 +0000
Subject: [PATCH 165/165] normalize CEL returns

---
 .../discovery/loaders/http/mapper.go          | 57 ++++++++++++++-----
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/internal/controller/discovery/loaders/http/mapper.go b/internal/controller/discovery/loaders/http/mapper.go
index ce1e2aa..9fb60c7 100644
--- a/internal/controller/discovery/loaders/http/mapper.go
+++ b/internal/controller/discovery/loaders/http/mapper.go
@@ -8,6 +8,7 @@ import (
 	"github.com/gnmic/operator/internal/controller/discovery/core"
 	"github.com/go-logr/logr"
 	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types/ref"
 	"github.com/google/cel-go/ext"
 )
 
@@ -184,25 +185,16 @@ func (l *Loader) getLabels(item map[string]any, full any, cm *compiledMapping) m
 
 	if cm != nil && cm.labels != nil {
 		val, err := evalCEL(cm.labels, item, full)
+		fmt.Printf("DEBUG: CEL labels result = %#v (type: %T)\n", val, val)
 		if err != nil {
 			return result
 		}
-		// Handle different map representations returned by CEL
-		// Labels must be a map of string keys and string values
-		switch labels := val.(type) {
-		case map[string]any:
-			for key, val := range labels {
-				result[key] = fmt.Sprintf("%v", val)
+		if m, ok := val.(map[string]any); ok {
+			for k, v := range m {
+				result[k] = fmt.Sprintf("%v", v)
 			}
-			return result
-		case map[any]any:
-			for key, val := range labels {
-				result[fmt.Sprintf("%v", key)] = fmt.Sprintf("%v", val)
-			}
-			return result
-		default:
-			return result
 		}
+		return result
 	}
 
 	// fallback: direct
@@ -278,7 +270,42 @@ func evalCEL(p cel.Program, item map[string]any, full any) (any, error) {
 	if out == nil {
 		return nil, fmt.Errorf("CEL returned nil")
 	}
-	return out.Value(), nil
+
+	return normalizeCEL(out.Value()), nil
+}
+
+// normalizeCEL recursively converts CEL evaluation results into standard Go types
+func normalizeCEL(v any) any {
+	switch raw := v.(type) {
+	case ref.Val:
+		return normalizeCEL(raw.Value())
+	case map[ref.Val]ref.Val:
+		out := make(map[string]any)
+		for k, v := range raw {
+			key := fmt.Sprintf("%v", normalizeCEL(k))
+			out[key] = normalizeCEL(v)
+		}
+		return out
+	case map[string]any:
+		out := make(map[string]any)
+		for k, v := range raw {
+			out[k] = normalizeCEL(v)
+		}
+		return out
+	case map[any]any:
+		out := make(map[string]any)
+		for k, v := range raw {
+			out[fmt.Sprintf("%v", normalizeCEL(k))] = normalizeCEL(v)
+		}
+		return out
+	case []any:
+		for i := range raw {
+			raw[i] = normalizeCEL(raw[i])
+		}
+		return raw
+	default:
+		return raw
+	}
 }
 
 // extractPort converts a CEL evaluation result into an int32 port number,