EKS MNG userns rootful DinD experiment

deploy/eks-mng-amazon-linux-userns-rootful-dind-experiment/cgroup-userns-probe.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cgroup-userns-probe
+  namespace: userns-rootful-dind
+spec:
+  hostUsers: false
+  restartPolicy: Never
+  nodeSelector:
+    experiment.coder.com/userns-dind: "true"
+  containers:
+    - name: probe
+      image: public.ecr.aws/docker/library/alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+      securityContext:
+        privileged: true
+        procMount: Unmasked
+      command: ["/bin/sh", "-c"]
+      args:
+        - |
+          set -eux
+          echo '--- uid map ---'
+          cat /proc/self/uid_map
+          echo '--- cgroup membership ---'
+          cat /proc/self/cgroup
+          echo '--- cgroup mounts ---'
+          mount | grep -E 'cgroup|/sys/fs/cgroup' || true
+          echo '--- cgroup root metadata ---'
+          ls -ld /sys/fs/cgroup
+          echo '--- cgroup-root write probe ---'
+          test_dir=/sys/fs/cgroup/userns-dind-write-probe
+          if mkdir "$test_dir"; then
+            echo writable
+            rmdir "$test_dir"
+          else
+            echo not-writable
+          fi
+          sleep 600

deploy/eks-mng-amazon-linux-userns-rootful-dind-experiment/cgroup-writable-runtime/cgroup-probe.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cgroup-writable-probe
+  namespace: userns-rootful-dind
+spec:
+  runtimeClassName: runc-cgroup-writable
+  hostUsers: false
+  restartPolicy: Never
+  containers:
+    - name: probe
+      image: public.ecr.aws/docker/library/alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+      securityContext:
+        privileged: false
+        allowPrivilegeEscalation: true
+        procMount: Unmasked
+        capabilities:
+          add: ["ALL"]
+        seccompProfile:
+          type: Unconfined
+      command: ["/bin/sh", "-c"]
+      args:
+        - |
+          set -eux
+          echo '--- identity and user namespace ---'
+          id
+          cat /proc/self/uid_map
+          cat /proc/self/gid_map
+          echo '--- cgroup namespace and mount ---'
+          cat /proc/self/cgroup
+          mount | grep -E 'cgroup|/sys/fs/cgroup'
+          ls -ldn /sys/fs/cgroup
+          echo '--- available/delegated controllers ---'
+          cat /sys/fs/cgroup/cgroup.controllers
+          cat /sys/fs/cgroup/cgroup.subtree_control
+          echo '--- create child cgroup ---'
+          mkdir /sys/fs/cgroup/userns-dind-write-probe
+          ls -ldn /sys/fs/cgroup/userns-dind-write-probe
+          rmdir /sys/fs/cgroup/userns-dind-write-probe
+          echo cgroup-writable-probe-ok
+          sleep 600

deploy/eks-mng-amazon-linux-userns-rootful-dind-experiment/cgroup-writable-runtime/compose-network-peer.yaml

@@ -0,0 +1,94 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: compose-network-target
+  namespace: userns-rootful-dind
+spec:
+  selector:
+    experiment.coder.com/compose-network-target: "true"
+  ports:
+    - name: http
+      port: 18080
+      targetPort: 18080
+---
+# This headless Service provides a stable discovery mechanism for the
+# workspace Pod IP. The peer resolves it, extracts the Pod IP, and then sends
+# HTTP directly to that IP rather than through the ClusterIP Service.
+apiVersion: v1
+kind: Service
+metadata:
+  name: compose-network-target-headless
+  namespace: userns-rootful-dind
+spec:
+  clusterIP: None
+  selector:
+    experiment.coder.com/compose-network-target: "true"
+  ports:
+    - name: http
+      port: 18080
+      targetPort: 18080
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: compose-network-peer-probe
+  namespace: userns-rootful-dind
+spec:
+  # The successful workspace is forced onto the cgroup-writable MNG by its
+  # RuntimeClass. Pinning this peer to the original MNG makes the two HTTP
+  # checks cross-node rather than merely same-node checks.
+  nodeSelector:
+    experiment.coder.com/userns-dind: "true"
+  automountServiceAccountToken: false
+  restartPolicy: Never
+  containers:
+    - name: probe
+      image: public.ecr.aws/docker/library/alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      env:
+        - name: PEER_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      command: ["/bin/sh", "-c"]
+      args:
+        - |
+          set -eux
+
+          fetch_and_verify() {
+            url="$1"
+            output="$2"
+            attempts=0
+            until wget -T 5 -t 1 -qO "$output" "$url"; do
+              attempts=$((attempts + 1))
+              [ "$attempts" -lt 120 ] || return 1
+              sleep 2
+            done
+            grep -q 'Welcome to nginx' "$output"
+          }
+
+          service_url='http://compose-network-target.userns-rootful-dind.svc.cluster.local:18080/'
+          fetch_and_verify "$service_url" /tmp/service-body
+          echo "cross-node-clusterip-service-ok node=$PEER_NODE_NAME url=$service_url"
+
+          headless_host='compose-network-target-headless.userns-rootful-dind.svc.cluster.local'
+          attempts=0
+          pod_ip=''
+          until pod_ip="$(getent hosts "$headless_host" | awk 'NR == 1 { print $1 }')" &&
+              [ -n "$pod_ip" ]; do
+            attempts=$((attempts + 1))
+            [ "$attempts" -lt 120 ] || exit 1
+            sleep 2
+          done
+
+          pod_url="http://${pod_ip}:18080/"
+          fetch_and_verify "$pod_url" /tmp/pod-ip-body
+          echo "cross-node-direct-pod-ip-ok node=$PEER_NODE_NAME url=$pod_url"
+          echo compose-cross-node-network-tests-ok

deploy/eks-mng-amazon-linux-userns-rootful-dind-experiment/cgroup-writable-runtime/nodegroup.yaml

@@ -0,0 +1,35 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+  name: userns-rootful-dind-mng-136
+  region: us-east-2
+  version: "1.36"
+
+managedNodeGroups:
+  - name: userns-dind-cgroup-writable
+    amiFamily: AmazonLinux2023
+    instanceType: m6i.large
+    minSize: 1
+    desiredCapacity: 1
+    maxSize: 1
+    labels:
+      experiment.coder.com/userns-dind-cgroup-writable: "true"
+    iam:
+      withAddonPolicies:
+        ebs: true
+    # eksctl prepends this partial NodeConfig to the EKS-generated NodeConfig;
+    # nodeadm merges it with the cluster connection details and normal AL2023
+    # defaults. Containerd 2.x uses the config-v3 CRI runtime plugin path.
+    overrideBootstrapCommand: |
+      apiVersion: node.eks.aws/v1alpha1
+      kind: NodeConfig
+      spec:
+        containerd:
+          config: |
+            [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc-cgroup-writable]
+              runtime_type = 'io.containerd.runc.v2'
+              cgroup_writable = true
+
+            [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc-cgroup-writable.options]
+              SystemdCgroup = true

deploy/eks-mng-amazon-linux-userns-rootful-dind-experiment/cgroup-writable-runtime/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: workspace-data-cgroup-writable
+  namespace: userns-rootful-dind
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: gp3-csi
+  resources:
+    requests:
+      storage: 30Gi