WIP: Move shards object ownership to shards

cmd/sharded-test-server/main.go

@@ -27,13 +27,11 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kuser "k8s.io/apiserver/pkg/authentication/user"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
 
 	"github.com/kcp-dev/sdk/apis/core"
@@ -316,7 +314,9 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb
 		return err
 	}
 
-	// Label region of shards
+	// Wait for all shards' Shard objects to be mirrored into the root
+	// workspace as representations before declaring the environment ready, so
+	// tests can rely on listing Shards there.
 	clientConfig, err := loadKubeConfig(filepath.Join(workDirPath, ".kcp", "admin.kubeconfig"), "base")
 	if err != nil {
 		return err
@@ -329,22 +329,15 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb
 	if err != nil {
 		return err
 	}
-	for i := range shards {
-		name := fmt.Sprintf("shard-%d", i)
-		if i == 0 {
-			name = "root"
-		}
-
-		if i >= len(regions) {
-			break
-		}
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"region":%q,"shared": "true"}}}`, regions[i])
-		if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
-			_, err := client.Cluster(core.RootCluster.Path()).CoreV1alpha1().Shards().Patch(ctx, name, types.MergePatchType, []byte(patch), metav1.PatchOptions{})
-			return err
-		}); err != nil {
-			return err
+	if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		shardList, err := client.Cluster(core.RootCluster.Path()).CoreV1alpha1().Shards().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			// the front-proxy or the root shard may not be fully ready yet, keep polling
+			return false, nil //nolint:nilerr
 		}
+		return len(shardList.Items) >= numberOfShards, nil
+	}); err != nil {
+		return fmt.Errorf("failed waiting for %d Shard representations in the root workspace: %w", numberOfShards, err)
 	}
 
 	readyToTestFile, err := os.Create(filepath.Join(workDirPath, ".kcp", "ready-to-test"))

cmd/sharded-test-server/shard.go

@@ -112,6 +112,11 @@ func newShard(ctx context.Context, n int, args []string, standaloneVW bool, serv
 		fmt.Sprintf("--service-account-signing-key-file=%s", filepath.Join(kcpDir, "service-account.key")),
 		"--audit-log-path", auditFilePath,
 		fmt.Sprintf("--shard-external-url=https://%s:%d", hostIP, 6443),
+	)
+	if n < len(regions) {
+		args = append(args, fmt.Sprintf("--shard-labels=region=%s,shared=true", regions[n]))
+	}
+	args = append(args,
 		fmt.Sprintf("--tls-cert-file=%s", filepath.Join(shardDir, "apiserver.crt")),
 		fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(shardDir, "apiserver.key")),
 		fmt.Sprintf("--secure-port=%d", 6444+n),

config/crds/core.kcp.io_shards.yaml

@@ -29,6 +29,10 @@ spec:
       jsonPath: .spec.externalURL
       name: External URL
       type: string
+    - description: Whether new workspaces are scheduled onto this shard
+      jsonPath: .status.conditions[?(@.type=="Schedulable")].status
+      name: Schedulable
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date

config/root-phase0/apiexport-shards.core.kcp.io.yaml

@@ -6,7 +6,7 @@ spec:
   resources:
   - group: core.kcp.io
     name: shards
-    schema: v260302-ae9fdd9c1.shards.core.kcp.io
+    schema: v260831-6f00c2057.shards.core.kcp.io
     storage:
       crd: {}
 status: {}

config/root-phase0/apiresourceschema-shards.core.kcp.io.yaml

@@ -1,7 +1,7 @@
 apiVersion: apis.kcp.io/v1alpha1
 kind: APIResourceSchema
 metadata:
-  name: v260302-ae9fdd9c1.shards.core.kcp.io
+  name: v260831-6f00c2057.shards.core.kcp.io
 spec:
   group: core.kcp.io
   names:
@@ -26,6 +26,10 @@ spec:
       jsonPath: .spec.externalURL
       name: External URL
       type: string
+    - description: Whether new workspaces are scheduled onto this shard
+      jsonPath: .status.conditions[?(@.type=="Schedulable")].status
+      name: Schedulable
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date

docs/content/concepts/sharding/shards.md

@@ -264,7 +264,19 @@ inferred only from request errors at the moment of access. Concretely:
   through the front-proxy until it returns.
 - **Workspace scheduling** continues to succeed for new workspaces as long as
   at least one valid shard is reachable; the random selector simply skips
-  shards annotated `kcp.io/unschedulable`.
+  shards annotated `experimental.core.kcp.io/unschedulable`. To cordon a
+  shard, set the annotation on its `Shard` representation in the `root`
+  workspace: the annotation is on a small allow-list that the shard mirror
+  syncs back onto the authoritative `Shard` object in the shard's
+  `system:shard` logical cluster (all other edits to a representation are
+  overwritten). Removing the annotation from the representation uncordons the
+  shard the same way. The kcp kubectl plugin wraps this, mirroring
+  `kubectl cordon`/`uncordon` for nodes: `kubectl kcp shard cordon <name>`
+  and `kubectl kcp shard uncordon <name>`. The owning shard acknowledges the signal node-style via
+  the `Schedulable` status condition on its `Shard` object — `False` with
+  reason `Cordoned` while cordoned, `True` otherwise — which is mirrored back
+  to the representation, so `kubectl get shard -o yaml` in `root` shows
+  whether the shard has actually applied the cordon.
 - **Bound APIs** on a workspace whose `APIExport` lives on an unavailable
   shard keep functioning: the CRD is materialized locally and the binding
   controller already has the schema cached. New bindings or schema updates

docs/content/concepts/workspaces/system-workspaces.md

@@ -103,6 +103,12 @@ Each bound CRD is:
 ## `system:shard`
 
 The `system:shard` workspace holds essential API bindings that every shard needs to function.
+It also holds the shard's own authoritative `Shard` object: each shard registers itself here
+at startup, the object is replicated to the cache server, and the root shard mirrors it into
+the root workspace as a read-only representation (marked with the
+`core.kcp.io/shard-representation` annotation). To change a shard's configuration (URLs,
+labels), change the shard's flags/deployment rather than editing the representation.
+
 During shard bootstrap, API bindings for the following API groups are created in this workspace:
 
 - `shards.core.kcp.io`

pkg/authorization/bootstrap/policy.go

@@ -108,6 +108,10 @@ func clusterRoles() []rbacv1.ClusterRole {
 			Rules: []rbacv1.PolicyRule{
 				rbacv1helpers.NewRule("*").Groups(core.GroupName).Resources("logicalclusters", "logicalclusters/status").RuleOrDie(),
 				rbacv1helpers.NewRule("delete", "update", "get").Groups(tenancy.GroupName).Resources("workspaces").RuleOrDie(),
+				// the shard mirror on the root shard back-syncs allow-listed
+				// annotations (e.g. cordoning) onto the shard-owned Shard
+				// object in each shard's system:shard logical cluster.
+				rbacv1helpers.NewRule("get", "update", "patch").Groups(core.GroupName).Resources("shards").RuleOrDie(),
 			},
 		},
 		{

pkg/authorization/workspace_content_authorizer.go

@@ -91,10 +91,22 @@ func (a *workspaceContentAuthorizer) Authorize(ctx context.Context, attr authori
 		return DelegateAuthorization("shard-level path", a.delegate).Authorize(ctx, attr)
 	}
 
-	// empty or system workspaces have no meaning in the context of authorizing workspace content.
-	// To access system workspaces, the user must be privileged such that authorization is skipped completely.
-	if cluster == nil || cluster.Name.Empty() || strings.HasPrefix(cluster.Name.String(), "system:") {
-		return authorizer.DecisionNoOpinion, "empty or system workspace", nil
+	// empty workspaces have no meaning in the context of authorizing workspace content.
+	if cluster == nil || cluster.Name.Empty() {
+		return authorizer.DecisionNoOpinion, "empty workspace", nil
+	}
+
+	// System workspaces have no meaning in the context of authorizing workspace
+	// content either. To access them, the user must be privileged such that
+	// authorization is skipped completely, with one exception: the
+	// logical-cluster-admin identity used for direct shard-to-shard requests
+	// (e.g. the shard mirror back-syncing cordon annotations onto the
+	// shard-owned Shard object in system:shard) is delegated to RBAC.
+	if strings.HasPrefix(cluster.Name.String(), "system:") {
+		if rbacregistryvalidation.EffectiveGroups(ctx, attr.GetUser()).Has(bootstrap.SystemLogicalClusterAdmin) {
+			return DelegateAuthorization("logical cluster admin access to system workspace", a.delegate).Authorize(ctx, attr)
+		}
+		return authorizer.DecisionNoOpinion, "system workspace", nil
 	}
 
 	isServiceAccount := rbacregistryvalidation.IsServiceAccount(attr.GetUser())

pkg/indexers/shard.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2026 The kcp Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package indexers
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"github.com/kcp-dev/logicalcluster/v3"
+	corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
+	corev1alpha1listers "github.com/kcp-dev/sdk/client/listers/core/v1alpha1"
+
+	configshard "github.com/kcp-dev/kcp/config/shard"
+)
+
+// ShardByName returns the Shard with the given name, regardless of the logical
+// cluster it lives in.
+func ShardByName(lister corev1alpha1listers.ShardClusterLister, name string) (*corev1alpha1.Shard, error) {
+	shards, err := lister.List(labels.Everything())
+	if err != nil {
+		return nil, err
+	}
+	var found *corev1alpha1.Shard
+	for _, shard := range shards {
+		if shard.Name != name {
+			continue
+		}
+		if logicalcluster.From(shard) == configshard.SystemShardCluster {
+			return shard, nil
+		}
+		found = shard
+	}
+	if found == nil {
+		return nil, errors.NewNotFound(corev1alpha1.Resource("shards"), name)
+	}
+	return found, nil
+}

pkg/indexers/shard_test.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2026 The kcp Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package indexers
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/cache"
+
+	kcpcache "github.com/kcp-dev/apimachinery/v2/pkg/cache"
+	"github.com/kcp-dev/logicalcluster/v3"
+	corev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
+	corev1alpha1listers "github.com/kcp-dev/sdk/client/listers/core/v1alpha1"
+)
+
+func shardInCluster(name, cluster string) *corev1alpha1.Shard {
+	return &corev1alpha1.Shard{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: map[string]string{logicalcluster.AnnotationKey: cluster},
+		},
+	}
+}
+
+func TestShardByName(t *testing.T) {
+	t.Parallel()
+	indexer := cache.NewIndexer(kcpcache.MetaClusterNamespaceKeyFunc, cache.Indexers{})
+	for _, shard := range []*corev1alpha1.Shard{
+		shardInCluster("alpha", "system:shard"),
+		shardInCluster("alpha", "root"), // legacy copy, authoritative wins
+		shardInCluster("legacy", "root"),
+	} {
+		if err := indexer.Add(shard); err != nil {
+			t.Fatal(err)
+		}
+	}
+	lister := corev1alpha1listers.NewShardClusterLister(indexer)
+
+	shard, err := ShardByName(lister, "alpha")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cluster := logicalcluster.From(shard); cluster != "system:shard" {
+		t.Errorf("expected the authoritative shard from system:shard, got the one from %q", cluster)
+	}
+
+	shard, err = ShardByName(lister, "legacy")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cluster := logicalcluster.From(shard); cluster != "root" {
+		t.Errorf("expected the legacy shard from root, got the one from %q", cluster)
+	}
+
+	if _, err := ShardByName(lister, "missing"); !errors.IsNotFound(err) {
+		t.Errorf("expected a NotFound error, got %v", err)
+	}
+}

pkg/reconciler/apis/apiexportendpointsliceurls/apiexportendpointsliceurls_controller.go

@@ -58,7 +58,7 @@ const (
 // NewController returns a new controller for APIExportEndpointSlices.
 // Shards and APIExports are read from the cache server.
 func NewController(
-	shardName string,
+	thisShard string,
 	apiExportEndpointSliceClusterInformer apisv1alpha1informers.APIExportEndpointSliceClusterInformer,
 	apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer,
 	globalAPIExportEndpointSliceClusterInformer apisv1alpha1informers.APIExportEndpointSliceClusterInformer,
@@ -67,7 +67,7 @@ func NewController(
 	clusterClient kcpclientset.ClusterInterface,
 ) (*controller, error) {
 	c := &controller{
-		shardName:     shardName,
+		thisShard:     thisShard,
 		clusterClient: clusterClient,
 		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
 			workqueue.DefaultTypedControllerRateLimiter[string](),
@@ -76,7 +76,7 @@ func NewController(
 			},
 		),
 		getMyShard: func() (*corev1alpha1.Shard, error) {
-			return globalShardClusterInformer.Cluster(core.RootCluster).Lister().Get(shardName)
+			return indexers.ShardByName(globalShardClusterInformer.Lister(), thisShard)
 		},
 		getAPIExportEndpointSlice: func(path logicalcluster.Path, name string) (*apisv1alpha1.APIExportEndpointSlice, error) {
 			obj, err := indexers.ByPathAndNameWithFallback[*apisv1alpha1.APIExportEndpointSlice](apisv1alpha1.Resource("apiexportendpointslices"), apiExportEndpointSliceClusterInformer.Informer().GetIndexer(), globalAPIExportEndpointSliceClusterInformer.Informer().GetIndexer(), path, name)
@@ -121,7 +121,7 @@ func NewController(
 		},
 		patchAPIExportEndpointSlice: func(ctx context.Context, cluster logicalcluster.Path, patch *apisv1alpha1apply.APIExportEndpointSliceApplyConfiguration) error {
 			_, err := clusterClient.ApisV1alpha1().APIExportEndpointSlices().Cluster(cluster).ApplyStatus(ctx, patch, metav1.ApplyOptions{
-				FieldManager: shardName,
+				FieldManager: thisShard,
 			})
 			return err
 		},
@@ -174,7 +174,7 @@ func NewController(
 // in the status of every APIExportEndpointSlices.
 type controller struct {
 	queue         workqueue.TypedRateLimitingInterface[string]
-	shardName     string
+	thisShard     string
 	clusterClient kcpclientset.ClusterInterface
 
 	getMyShard                  func() (*corev1alpha1.Shard, error)

pkg/reconciler/apis/apiexportendpointsliceurls/apiexportendpointsliceurls_controller_test.go

@@ -145,7 +145,7 @@ func TestReconcile(t *testing.T) {
 				},
 			},
 			endpointsReconciler: &endpointsReconciler{
-				shardName: "shard2",
+				thisShard: "shard2",
 				getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) {
 					return &apisv1alpha2.APIExport{}, nil
 				},
@@ -183,7 +183,7 @@ func TestReconcile(t *testing.T) {
 				},
 			},
 			endpointsReconciler: &endpointsReconciler{
-				shardName: "shard1",
+				thisShard: "shard1",
 				getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) {
 					return &apisv1alpha2.APIExport{}, nil
 				},
@@ -232,7 +232,7 @@ func TestReconcile(t *testing.T) {
 				},
 			},
 			endpointsReconciler: &endpointsReconciler{
-				shardName: "shard1",
+				thisShard: "shard1",
 				getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) {
 					return &apisv1alpha2.APIExport{}, nil
 				},
@@ -276,7 +276,7 @@ func TestReconcile(t *testing.T) {
 				},
 			},
 			endpointsReconciler: &endpointsReconciler{
-				shardName: "shard1",
+				thisShard: "shard1",
 				getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) {
 					return &apisv1alpha2.APIExport{
 						ObjectMeta: metav1.ObjectMeta{
@@ -325,7 +325,7 @@ func TestReconcile(t *testing.T) {
 				getAPIExport:                tc.endpointsReconciler.getAPIExport,
 				listAPIBindingsByAPIExport:  tc.endpointsReconciler.listAPIBindingsByAPIExport,
 				patchAPIExportEndpointSlice: tc.endpointsReconciler.patchAPIExportEndpointSlice,
-				shardName:                   tc.endpointsReconciler.shardName,
+				thisShard:                   tc.endpointsReconciler.thisShard,
 			}
 			input := tc.input.DeepCopy()
 			_, err := c.reconcile(context.Background(), input)