Feature/pull interface

.gitignore

@@ -34,3 +34,10 @@ docs/public
 docs/resources/_gen/
 docs/.hugo_build.lock
 test/integration/**/clab-*
+
+
+# Only for development and testing purposes
+# To be removed after development of targetsource
+# ignored in order to not add unnecassary logging messages
+lab/dev/resources/targetsources
+.scannerwork/

api/v1alpha1/targetsource_types.go

@@ -17,37 +17,332 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // TargetSourceSpec defines the desired state of TargetSource
 // +kubebuilder:validation:Required
 type TargetSourceSpec struct {
+	// Provider defines the source of targets for this TargetSource
+	// Only one provider can be specified per TargetSource
+	// +kubebuilder:validation:Required
 	Provider *ProviderSpec `json:"provider"`
 
+	// TODO: implement in message processor
+	// Optional port to use for discovered targets if not specified by the provider
+	// +kubebuilder:validation:Optional
+	TargetPort int32 `json:"targetPort,omitempty"`
+
+	// Optional labels to apply to all targets discovered by this TargetSource
 	// +kubebuilder:validation:Optional
 	TargetLabels map[string]string `json:"targetLabels,omitempty"`
 
+	// The TargetProfile to use for targets discovered by this TargetSource
+	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinLength=1
 	TargetProfile string `json:"targetProfile"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf=http;consul
+// ProviderSpec defines the source of targets for a TargetSource
+// Only one provider can be specified per TargetSource
+// +kubebuilder:validation:ExactlyOneOf=http
 type ProviderSpec struct {
-	HTTP   *HTTPConfig   `json:"http,omitempty"`
-	Consul *ConsulConfig `json:"consul,omitempty"`
+	// HTTP defines the configuration for a HTTP provider
+	HTTP *HTTPConfig `json:"http,omitempty"`
 }
 
+// HTTPConfig defines the configuration for the HTTP provider
+// +kubebuilder:validation:AtLeastOneOf:=url;push
 type HTTPConfig struct {
+	// URL of the HTTP endpoint to pull targets from
+	// If defined, the loader will periodically poll this endpoint for targets
+	// +kubebuilder:validation:Optional
+	URL string `json:"url,omitempty"`
+
+	// HTTP method used for the request.
+	//
+	// Defaults to GET if not specified.
+	//
+	// Supported values:
+	// - GET  (default, no request body)
+	// - POST (supports request body)
+	//
+	// +kubebuilder:validation:Enum=GET;POST
+	// +kubebuilder:validation:Optional
+	Method string `json:"method,omitempty"`
+
+	// Optional HTTP headers to include in the request.
+	//
+	// These map directly to HTTP headers (key-value pairs).
+	//
+	// Example:
+	//   headers:
+	//     Content-Type: application/json
+	//     X-Custom-Header: value
+	//
+	// Precedence:
+	// - Authorization configuration overrides any conflicting headers
+	//
+	// +kubebuilder:validation:Optional
+	Headers map[string]string `json:"headers,omitempty"`
+
+	// Optional raw request body.
+	//
+	// Typically used with POST requests and contains JSON payload.
+	//
+	// Example:
+	//   body: |
+	//     {
+	//       "limit": 100,
+	//       "status": "active"
+	//     }
+	//
+	// Notes:
+	// - Ignored for GET requests
+	// - User must set appropriate Content-Type header if needed
+	//
+	// +kubebuilder:validation:Optional
+	Body string `json:"body,omitempty"`
+
+	// Optional authorization configuration for accessing the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	Authorization *AuthorizationSpec `json:"authorization,omitempty"`
+
+	// Optional interval for polling the HTTP endpoint for targets
+	// TODO: document about default value
+	// +kubebuilder:default="6h"
+	// +kubebuilder:validation:Optional
+	Interval *metav1.Duration `json:"interval,omitempty"`
+
+	// Optional timeout for HTTP requests to the endpoint
+	// +kubebuilder:default="10s"
+	// +kubebuilder:validation:Optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
+
+	// Optional TLS configuration for connecting to the HTTP endpoint
+	// If it is an HTTP endpoint, this will be ignored
+	// +kubebuilder:validation:Optional
+	TLS *ClientTLSConfig `json:"tls,omitempty"`
+
+	// Optional pagination configuration for parsing responses from the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	Pagination *PaginationSpec `json:"pagination,omitempty"`
+
+	// Optional mapping configuration for parsing responses from the HTTP endpoint
+	// +kubebuilder:validation:Optional
+	ResponseMapping *ResponseMappingSpec `json:"mapping,omitempty"`
+
+	// Optional configuration to enable push
+	// +kubebuilder:validation:Optional
+	Push *PushSpec `json:"push,omitempty"`
+}
+
+type ClientTLSConfig struct {
+	// Skip TLS verification of the Provider's certificate.
+	// +kubebuilder:default:=false
+	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
+
+	// Reference to a ConfigMap containing a bundle of PEM-encoded CAs to use when
+	// verifying the certificate chain presented by the Provider when using HTTPS.
+	// Mutually exclusive with CABundle.
+	// +kubebuilder:validation:Optional
+	CABundleRef *corev1.ConfigMapKeySelector `json:"caBundleRef,omitempty"`
+}
+
+// AuthorizationSpec defines the configuration for authentication
+// +kubebuilder:validation:ExactlyOneOf=basic;token
+type AuthorizationSpec struct {
+	// Basic authentication configuration
+	Basic *BasicAuthSpec `json:"basic,omitempty"`
+	// Token-based authentication configuration
+	Token *TokenAuthSpec `json:"token,omitempty"`
+}
+
+// BasicAuthSpec defines the configuration for basic authentication
+type BasicAuthSpec struct {
+	// Reference to a Secret containing "username" and "password" keys to use for
+	// basic authentication when connecting to the Provider.
+	// +kubebuilder:validation:Required
+	CredentialsSecretRef *corev1.SecretKeySelector `json:"credentialsSecretRef"`
+}
+
+// TokenAuthSpec defines the configuration for token-based authentication
+type TokenAuthSpec struct {
+	// Scheme for the token, e.g. "Bearer"
 	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url"`
+	Scheme string `json:"scheme"`
+	// Reference to a Secret containing a key with the token value to use for
+	// authentication when connecting to the Provider.
+	// +kubebuilder:validation:Required
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+// PaginationSpec defines the configuration for paginating through responses from providers
+type PaginationSpec struct {
+	// Field name in the JSON response that contains the next page reference.
+	// The value can be either:
+	// - a full URL (used directly for the next request), or
+	// - a pagination token (appended as a query parameter using this field name as the key).
+	//
+	// Must refer to a top-level key in the response object.
+	// Example: "next" or "nextToken"
+	NextField string `json:"nextField,omitempty"`
+}
+
+// ResponseMappingSpec controls how targets are extracted from an HTTP JSON response.
+//
+// This allows you to map fields from a JSON API into targets using either:
+//   - simple direct field access (e.g. item["name"])
+//   - or CEL expressions for more advanced logic
+//
+// General behavior:
+//
+//  1. Selecting targets:
+//     - `targetsField` is a CEL expression that selects the list of targets
+//     - It runs once on the full response (`self`) and MUST return a list
+//     - If not set, the response itself must be a JSON array
+//
+//  2. Extracting fields:
+//     - Each field (name, address, port, labels, etc.) is handled independently
+//     - If a CEL expression is provided → it is evaluated
+//     - If not provided → the value is read directly from the target object
+//
+//  3. Available variables in CEL:
+//     - item -> the current target object
+//     - self -> the full HTTP response JSON
+//
+// Example:
+//
+//	Response:
+//	{
+//	  "results": [
+//	    { "name": "device1", "ip": "10.0.0.1", "env": "prod" }
+//	  ],
+//	  "meta": { "region": "eu-west" }
+//	}
+//
+//	Mapping:
+//	targetsField: "self.results"
+//
+//	name: ""            # direct → item["name"]
+//	address: "item.ip"  # CEL
+//
+//	labels:
+//	  env:    "item.env"
+//	  region: "self.meta.region"
+type ResponseMappingSpec struct {
+	// CEL expression that selects the list of target objects from the response.
+	//
+	// This is evaluated once using:
+	//   self -> full JSON response
+	//
+	// Example:
+	//   targetsField: "self.results"
+	//
+	// If not set, the response itself must be a JSON array with the targets.
+	//
+	// +kubebuilder:validation:Optional
+	TargetsField string `json:"targetsField,omitempty"`
+
+	// CEL expression for the target name.
+	//
+	// If not set, defaults to:
+	//   item["name"]
+	//
+	// Example:
+	//   "item.hostname"
+	//
+	// +kubebuilder:validation:Optional
+	Name string `json:"name,omitempty"`
+
+	// CEL expression for the target address.
+	//
+	// If not set, defaults to:
+	//   item["address"]
+	//
+	// Example:
+	//   "item.ip"
+	//
+	// +kubebuilder:validation:Optional
+	Address string `json:"address,omitempty"`
+
+	// CEL expression for the target port.
+	//
+	// If not set, defaults to:
+	//   item["port"]
+	//
+	// Example:
+	//   "item.port"
+	//
+	// +kubebuilder:validation:Optional
+	Port string `json:"port,omitempty"`
+
+	// CEL expression that returns a map of labels.
+	// The expression must evaluate to an object (map).
+	//
+	// Example:
+	//
+	//   labels: |
+	//     {
+	//       "env": item.environment,
+	//       "region": self.meta.region,
+	//       item.dynamicKey: "value"
+	//     }
+	//
+	// If not set, defaults to:
+	//   item["labels"]
+	//
+	// The resulting map will be converted into labels.
+	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
+	// with values from the response taking precedence in case of conflicts.
+	//
 	// +kubebuilder:validation:Optional
-	AcceptPush bool `json:"acceptPush,omitempty"`
+	Labels string `json:"labels,omitempty"`
+
+	// CEL expression for the target profile.
+	//
+	// If not set, defaults to:
+	//   item["targetProfile"]
+	//
+	// Example:
+	//   "item.type == 'edge' ? 'edge-profile' : 'default'"
+	//
+	// +kubebuilder:validation:Optional
+	TargetProfile string `json:"targetProfile,omitempty"`
+}
+
+// PushSpec defines the settings for event-based update mechanism (i.e. webhooks sent from the server)
+type PushSpec struct {
+	// +kubebuilder:default=false
+	Enabled bool `json:"enabled"`
+
+	// +kubebuilder:validation:Optional
+	Auth *PushAuthSpec `json:"auth,omitempty"`
+}
+
+// +kubebuilder:validation:ExactlyOneOf:=bearer;signature
+type PushAuthSpec struct {
+	Bearer    *PushBearerAuthSpec    `json:"bearer,omitempty"`
+	Signature *PushSignatureAuthSpec `json:"signature,omitempty"`
 }
 
-type ConsulConfig struct {
+// +kubebuilder:validation:Required
+type PushBearerAuthSpec struct {
+	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
+}
+
+// +kubebuilder:validation:Required
+type PushSignatureAuthSpec struct {
+	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
+
+	// Header containing the signature
 	// +kubebuilder:validation:MinLength=1
-	URL string `json:"url,omitempty"`
+	Header string `json:"header"`
+
+	// +kubebuilder:default="sha512"
+	// +kubebuilder:validation:Enum=sha1;sha256;sha512
+	Algorithm string `json:"algorithm"`
 }
 
 // TargetSourceStatus defines the observed state of TargetSource