cat <<EOF | kubectl create -f -
apiVersion: v1
kind: Pod
metadata:
  name: netpod
  labels:
    app: netpod
spec:
  nodeName: k8s-s
  containers:
  - name: netshoot-pod
    image: nicolaka/netshoot
    command: ["tail"]
    args: ["-f", "/dev/null"]
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod1
  labels:
    app: webpod
spec:
  nodeName: k8s-w1
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod2
  labels:
    app: webpod
spec:
  nodeName: k8s-w2
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
EOF

# 확인
kubectl get pod -o wide
c0 status --verbose | grep Allocated -A5
c1 status --verbose | grep Allocated -A5
c2 status --verbose | grep Allocated -A5
 
kubectl get ciliumendpoints
kubectl get ciliumendpoints -A
c0 endpoint list
c0 bpf endpoint list
c0 map get cilium_lxc
c0 ip list

# 테스트 파드들 IP
NETPODIP=$(kubectl get pods netpod -o jsonpath='{.status.podIP}')
WEBPOD1IP=$(kubectl get pods webpod1 -o jsonpath='{.status.podIP}')
WEBPOD2IP=$(kubectl get pods webpod2 -o jsonpath='{.status.podIP}')
 
# 단축키(alias) 지정
alias p0="kubectl exec -it netpod  -- "
alias p1="kubectl exec -it webpod1 -- "
alias p2="kubectl exec -it webpod2 -- "

# netpod 네트워크 정보 확인
p0 ip -c -4 addr
p0 route -n
p0 ping -c 1 $WEBPOD1IP && p0 ping -c 1 $WEBPOD2IP
p0 curl -s $WEBPOD1IP && p0 curl -s $WEBPOD2IP
p0 curl -s $WEBPOD1IP:8080 ; p0 curl -s $WEBPOD2IP:8080
p0 ping -c 1 8.8.8.8 && p0 curl -s wttr.in/seoul
p0 ip -c neigh
 
# hubble cli 확인
hubble observe --pod netpod
hubble observe --pod webpod1
hubble observe --pod webpod2
 
# BPF maps : 목적지 파드와 통신 시 어느곳으로 보내야 될지 확인할 수 있다
c0 map get cilium_ipcache
c0 map get cilium_ipcache | grep $WEBPOD1IP
 

c0bpf net show
 
last_lxc_line=$(c0bpf net show | grep "lxc" | grep -v "lxc_health" | grep "cil_from" | tail -n 1)
 
# 필요한 값 파싱
LXC=$(echo "$last_lxc_line" | awk '{print $1}' | sed 's/(.*//')      # LXC 이름에서 ( 숫자 ) 제거
PGID=$(echo "$last_lxc_line" | awk '{print $5}')                     # prog_id 값
 
# Use bpftool prog show id to view additional information about a program, including a list of attached eBPF maps:
c0bpf prog show id $PGID
 
# 파드와 veth pair 에 IP가 없다! proxy_arp 도 없다! 하지만 GW MAC 요청 시 lxc(veth)의 MAC 으로 응답이 온다! >> eBPF Magic!
# Cilium hijacks ARP table of POD1, forces the next hop to be the peer end (host side) of the veth pair.
ip -c addr show dev $LXC# list of eBPF programs
c0bpf net show
c0bpf net show | grep $LXC
 
c0bpf prog show id $PGID
 
c0bpf map list

cat <<EOF | kubectl create -f -
apiVersion: v1
kind: Service
metadata:
  name: svc
spec:
  ports:
    - name: svc-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: ClusterIP
EOF

# 서비스 생성 확인
kubectl get svc,ep svc
 
# 노드에 iptables 더이상 KUBE-SVC rule 이 생성되지 않는다!
iptables-save | grep KUBE-SVC
iptables-save | grep CILIUM
 
# 서비스IP를 변수에 지정
SVCIP=$(kubectl get svc svc -o jsonpath='{.spec.clusterIP}')
 
# Pod1 에서 Service(ClusterIP) 접속 트래픽 발생
kubectl exec netpod -- curl -s $SVCIP
kubectl exec netpod -- curl -s $SVCIP | grep Hostname
 
# 지속적으로 접속 트래픽 발생
SVCIP=$(kubectl get svc svc -o jsonpath='{.spec.clusterIP}')
while true; do kubectl exec netpod -- curl -s $SVCIP | grep Hostname;echo "-----";sleep 1;done
 
# 파드에서 SVC(ClusterIP) 접속 시 tcpdump 로 확인 >> 파드 내부 캡쳐인데, SVC(10.108.12.195)는 보이지 않고, DNAT 된 web-pod 의 IP가 확인! Magic!
kubectl exec netpod -- tcpdump -enni any -q
 

 
kubectl exec netpod -- sh -c "ngrep -tW byline -d eth0 '' 'tcp port 80'"
 
# 서비스 정보 확인
c0 service list
ID   Frontend              Service Type   Backend
16   10.108.12.195:80      ClusterIP      1 => 172.16.2.157:80
                                          2 => 172.16.1.234:80
c0 bpf lb list
SERVICE ADDRESS       BACKEND ADDRESS
10.108.12.195:80      0.0.0.0:0 (16) [ClusterIP, non-routable]
                      172.16.1.234:80 (16)
                      172.16.2.157:80 (16)
# BPF maps
c0 map list --verbose
c0 map list --verbose | grep lb
c0 map get cilium_lb4_services_v2
c0 map get cilium_lb4_backends_v3
c0 map get cilium_lb4_reverse_nat
c0 map get cilium_lb4_reverse_sk
c0 map get cilium_lxc
c0 map get cilium_ipcache

# Socket-Based LoadBalancing 관련 설정들 확인
c0 status --verbose
 

 
# cgroup root 경로 확인
tree /run/cilium/cgroupv2 -L 1
tree /run/cilium/cgroupv2 -L 2
cilium config view | grep cgroup
 

 
# eBPF cgroup 확인 : Socket based LB 와 관련
c0bpf cgroup tree
CgroupPath
 

 
# cilium 파드의 Init Containers 에서 cgroup 마운트!
Init Containers:
  mount-cgroup:
    Container ID:  containerd://72e9d2ee9731e3536c893f9daaa7674809638e3d137f9eb0f46fe916c2aa2839
    Image:         quay.io/cilium/cilium:v1.16.3@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28
    Image ID:      quay.io/cilium/cilium@sha256:62d2a09bbef840a46099ac4c69421c90f84f28d018d479749049011329aa7f28
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
      -ec
      cp /usr/bin/cilium-mount /hostbin/cilium-mount;
      nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT;
      rm /hostbin/cilium-mount
      
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Sun, 20 Oct 2024 15:45:34 +0900
      Finished:     Sun, 20 Oct 2024 15:45:34 +0900
    Ready:          True
    Restart Count:  0
    Environment:
      CGROUP_ROOT:  /run/cilium/cgroupv2
      BIN_PATH:     /opt/cni/bin
    Mounts:
      /hostbin from cni-path (rw)
      /hostproc from hostproc (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-p6bcr (ro)
 
# mount-cgroup 로그 확인
kubetail -n kube-system -c mount-cgroup --since 12h
...
[cilium-lnwcr] time="2024-10-20T15:45:52+09:00" level=info msg="Mounted cgroupv2 filesystem at /run/cilium/cgroupv2" subsys=cgroups 
[cilium-jmr7d] time="2024-10-20T15:45:33+09:00" level=info msg="Mounted cgroupv2 filesystem at /run/cilium/cgroupv2" subsys=cgroups 
...

# syacall 호출 확인
kubectl exec netpod -- strace -c curl -s $SVCIP
 

# 배포
kubectl apply -f https://raw.githubusercontent.com/cilium/cilium/1.16.3/examples/kubernetes/addons/prometheus/monitoring-example.yaml
kubectl get all -n cilium-monitoring
 
# 파드와 서비스 확인
kubectl get pod,svc,ep -o wide -n cilium-monitoring
 
# NodePort 설정
kubectl patch svc grafana -n cilium-monitoring -p '{"spec": {"type": "NodePort"}}'
kubectl patch svc prometheus -n cilium-monitoring -p '{"spec": {"type": "NodePort"}}'
 
# Grafana 웹 접속
GPT=$(kubectl get svc -n cilium-monitoring grafana -o jsonpath={.spec.ports[0].nodePort})
echo -e "Grafana URL = http://$(curl -s ipinfo.io/ip):$GPT"
 
# Prometheus 웹 접속 정보 확인
PPT=$(kubectl get svc -n cilium-monitoring prometheus -o jsonpath={.spec.ports[0].nodePort})
echo -e "Prometheus URL = http://$(curl -s ipinfo.io/ip):$PPT"

# 설정 : 미리 설정 되어 있음
VERSION=1.11.2
helm upgrade cilium cilium/cilium --version $VERSION --namespace kube-system --reuse-values --set loadBalancer.mode=dsr
 
# cilium 파드 재시작 >> 단축키 재설정
kubectl -n kube-system rollout restart ds/cilium
 
# DSR 모드 확인 
cilium config view | grep dsr
bpf-lb-mode                                    dsr
 
c0 status --verbose | grep 'KubeProxyReplacement Details:' -A7
KubeProxyReplacement Details:
  Status:                Strict
  Socket LB Protocols:   TCP, UDP
  Devices:               enp0s3 10.0.2.15, enp0s8 192.168.10.10 (Direct Routing)
  Mode:                  DSR
  Backend Selection:     Random
  Session Affinity:      Enabled
  XDP Acceleration:      Disabled 

cat <<EOF | kubectl create -f -
apiVersion: v1
kind: Pod
metadata:
  name: netpod
  labels:
    app: netpod
spec:
  nodeName: k8s-m
  containers:
  - name: netshoot-pod
    image: nicolaka/netshoot
    command: ["tail"]
    args: ["-f", "/dev/null"]
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod1
  labels:
    app: webpod
spec:
  nodeName: k8s-w1
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Service
metadata:
  name: svc1
spec:
  ports:
    - name: svc1-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: NodePort
EOF

# 파드 서비스 정보
kubectl get pod -o wide
kubectl get svc svc1
 
# map list 확인
c0 map list --verbose
c1 map list --verbose

# 서비스(NodePort)를 확인
kubectl get svc svc1 -o jsonpath='{.spec.ports[0].nodePort}';echo
32465
 
------------------------------------------------------------------------------
# k8s-pc 나 자신의 PC 에서 Service(NodePort) 접속 트래픽 발생
SVCNPORT=$(kubectl get svc svc1 -o jsonpath='{.spec.ports[0].nodePort}')
curl -s k8s-m:$SVCNPORT
curl -s k8s-m:$SVCNPORT | grep Hostname
 
# k8s-pc 나 자신의 PC 에서 지속적으로 접속 트래픽 발생
while true; do curl -s k8s-m:$SVCNPORT | grep Hostname;echo "-----";sleep 1;done
------------------------------------------------------------------------------
 

 
# 모니터링 : 워커노드에 최초 TCP SYN 에 IP옵션 헤더값 포함된 패킷 확인! >> 아래 처럼 출력된 IPv4Option 에 맨 뒤부터 읽으면 된다!
c0 monitor # 없다!
c1 monitor -vv
 

# 패킷 덤프 - 마스터노드
NODEPORT=$(kubectl get svc svc1 -o jsonpath='{.spec.ports[0].nodePort}')
echo $NODEPORT
tcpdump -i any tcp port 80 or tcp port $NODEPORT -w /tmp/dsr1.pcap

# 패킷 덤프 - 워커노드
NODEPORT=$(kubectl get svc svc1 -o jsonpath='{.spec.ports[0].nodePort}')
tcpdump -eni any tcp port 80 or tcp port $NODEPORT -q -v -X                        ....

# 실습 완료 후 삭제
kubectl delete svc svc1 && kubectl delete pod webpod1 netpod

# 배포
kubectl create -f https://raw.githubusercontent.com/cilium/cilium/1.16.3/examples/minikube/http-sw-app.yaml
kubectl get all
 
# 파드 라벨 확인
kubectl get pod --show-labels
NAME                         READY   STATUS    RESTARTS   AGE    LABELS
deathstar-689f66b57d-4rwkf   1/1     Running   0          113s   app.kubernetes.io/name=deathstar,class=deathstar,org=empire,pod-template-hash=689f66b57d
deathstar-689f66b57d-8p2l5   1/1     Running   0          113s   app.kubernetes.io/name=deathstar,class=deathstar,org=empire,pod-template-hash=689f66b57d
tiefighter                   1/1     Running   0          113s   app.kubernetes.io/name=tiefighter,class=tiefighter,org=empire
xwing                        1/1     Running   0          113s   app.kubernetes.io/name=xwing,class=xwing,org=alliance
 
# cilium endpoint 확인
kubectl get ciliumendpoints
c1 endpoint list
c2 endpoint list
 
# 데스스타 SVC(ClusterIP) 접속하여 웹 파드 연결 확인 >> Hubble UI 에서 실시간 확인해보자!
kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed
 
kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed
 
# 확인
hubble observe

# L3/L4 정책 생성
cat <<EOF | kubectl apply -f - 
apiVersion: "cilium.io/v2"
kind: CiliumNetworkPolicy
metadata:
  name: "rule1"
spec:
  description: "L3-L4 policy to restrict deathstar access to empire ships only"
  endpointSelector:
    matchLabels:
      org: empire
      class: deathstar
  ingress:
  - fromEndpoints:
    - matchLabels:
        org: empire
    toPorts:
    - ports:
      - port: "80"
        protocol: TCP
EOF
 
# 정책 확인
kubectl get cnp
kc describe cnp rule1
c0 policy get
 

 
# 파드 curl 접속 시도 시 파드 sh 접속 후 curl 시도하자!
# 데스스타 SVC(ClusterIP) 접속하여 웹 파드 연결 확인 >> Hubble UI 에서 drop 확인!
kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed
 
kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
drop
 
# hubble cli 모니터링 
hubble observe --pod xwing
hubble observe --pod tiefighter
hubble observe --pod deathstar
hubble observe --pod deathstar --verdict DROPPED
 

 
Inspecting the Policy
# If we run cilium endpoint list again we will see that the pods with the label org=empire and class=deathstar
# now have ingress policy enforcement enabled as per the policy above.
 
# endpoint list 에서 정책 적용 확인
c1 endpoint list | grep deathstar
c2 endpoint list | grep deathstar

# 데스스타 SVC(ClusterIP) 접속
kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port

 
# POST /v1/request-landing API 호출만 허용 정책으로 기존 정책 내용을 업데이트(configured)!
cat <<EOF | kubectl apply -f - 
apiVersion: "cilium.io/v2"
kind: CiliumNetworkPolicy
metadata:
  name: "rule1"
spec:
  description: "L7 policy to restrict access to specific HTTP call"
  endpointSelector:
    matchLabels:
      org: empire
      class: deathstar
  ingress:
  - fromEndpoints:
    - matchLabels:
        org: empire
    toPorts:
    - ports:
      - port: "80"
        protocol: TCP
      rules:
        http:
        - method: "POST"
          path: "/v1/request-landing"
EOF
 
# 정책 확인
kc describe ciliumnetworkpolicies
c0 policy get
 
# 모니터링
c1 monitor -v --type l7
c2 monitor -v --type l7
<- Request http from 0 ([k8s:io.cilium.k8s.policy.cluster=default k8s:io.cilium.k8s.policy.serviceaccount=default k8s:io.kubernetes.pod.namespace=default k8s:org=empire k8s:class=tiefighter k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name=default]) to 1972 ([k8s:class=deathstar k8s:org=empire k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name=default k8s:io.kubernetes.pod.namespace=default k8s:io.cilium.k8s.policy.serviceaccount=default k8s:io.cilium.k8s.policy.cluster=default]), identity 42720->21144, verdict Denied PUT http://deathstar.default.svc.cluster.local/v1/exhaust-port => 403
 => 403
hubble observe --pod deathstar
hubble observe --pod deathstar --verdict DROPPED
 

# 접근 테스트
kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed
 
kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port
Access denied
 
## hubble cli 에 차단 로그 확인
hubble observe --pod deathstar --verdict DROPPED
 
hubble observe --pod deathstar --protocol http
# 삭제
kubectl delete -f https://raw.githubusercontent.com/cilium/cilium/1.16.3/examples/minikube/http-sw-app.yaml
kubectl delete cnp rule1

# 인터페이스 tc qdisc 확인
tc qdisc show dev ens5
 
# 설정
helm upgrade cilium cilium/cilium --namespace kube-system --reuse-values --set bandwidthManager.enabled=true
 
# 적용 확인
cilium config view | grep bandwidth
enable-bandwidth-manager                       true
 
# egress bandwidth limitation 동작하는 인터페이스 확인
c0 status | grep  BandwidthManager
BandwidthManager:        EDT with BPF [CUBIC] [ens5]
 
# 인터페이스 tc qdisc 확인 : 설정 전후 옵션값들이 상당히 추가된다
tc qdisc
tc qdisc show dev ens5

# 테스트를 위한 트래픽 발생 서버/클라이언트 파드 생성
cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Pod
metadata:
  annotations:
    # Limits egress bandwidth to 10Mbit/s.
    kubernetes.io/egress-bandwidth: "10M"
  labels:
    # This pod will act as server.
    app.kubernetes.io/name: netperf-server
  name: netperf-server
spec:
  containers:
  - name: netperf
    image: cilium/netperf
    ports:
    - containerPort: 12865
---
apiVersion: v1
kind: Pod
metadata:
  # This Pod will act as client.
  name: netperf-client
spec:
  affinity:
    # Prevents the client from being scheduled to the
    # same node as the server.
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: app.kubernetes.io/name
            operator: In
            values:
            - netperf-server
        topologyKey: kubernetes.io/hostname
  containers:
  - name: netperf
    args:
    - sleep
    - infinity
    image: cilium/netperf
EOF
 

 
# egress BW 제한 정보 확인
kubectl describe pod netperf-server | grep Annotations:
Annotations:  kubernetes.io/egress-bandwidth: 10M
 
# egress BW 제한이 설정된 파드가 있는 cilium pod 에서 제한 정보 확인
c1 bpf bandwidth list
c2 bpf bandwidth list
# 트래픽 발생 >> Hubble UI 에서 확인
# egress traffic of the netperf-server Pod has been limited to 10Mbit per second. 
NETPERF_SERVER_IP=$(kubectl get pod netperf-server -o jsonpath='{.status.podIP}')
kubectl exec netperf-client -- netperf -t TCP_MAERTS -H "${NETPERF_SERVER_IP}"
 
 
# 5M 제한 설정 후 테스트
kubectl get pod netperf-server -o json | sed -e 's|10M|5M|g' | kubectl apply -f -
c1 bpf bandwidth list
c2 bpf bandwidth list
kubectl exec netperf-client -- netperf -t TCP_MAERTS -H "${NETPERF_SERVER_IP}"
 
# 20M 제한 설정 후 테스트
kubectl get pod netperf-server -o json | sed -e 's|5M|20M|g' | kubectl apply -f -
kubectl exec netperf-client -- netperf -t TCP_MAERTS -H "${NETPERF_SERVER_IP}"
 
# 삭제
kubectl delete pod netperf-client netperf-server

#
helm upgrade cilium cilium/cilium --namespace kube-system --reuse-values \
--set l2announcements.enabled=true --set externalIPs.enabled=true \
--set l2announcements.leaseDuration=3s --set l2announcements.leaseRenewDeadline=1s --set l2announcements.leaseRetryPeriod=200ms
 
#
c0 config --all  |grep L2
EnableL2Announcements             : true
EnableL2NeighDiscovery            : true
 
# CiliumL2AnnouncementPolicy 생성
cat <<EOF | kubectl apply -f - 
apiVersion: "cilium.io/v2alpha1"
kind: ciliumL2AnnouncementPolicy
metadata:
  name: policy1
spec:
  serviceSelector:
    matchLabels:
      color: blue
  nodeSelector:
    matchExpressions:
      - key: node-role.kubernetes.io/control-plane
        operator: DoesNotExist
  interfaces:
  - ^ens[0-9]+
  externalIPs: true
  loadBalancerIPs: true
EOF
 
# 확인
kubectl get ciliuml2announcementpolicy
kc describe l2announcement
 
#
cat <<EOF | kubectl apply -f - 
apiVersion: "cilium.io/v2alpha1"
kind: ciliumLoadBalancerIPPool
metadata:
  name: "cilium-pool"
spec:
  allowFirstLastIPs: "No"
  blocks:
  - cidr: "10.10.200.0/29"
EOF
 
# cilium ip pool 조회
kubectl get CiliumLoadBalancerIPPool

#
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: webpod1
  labels:
    app: webpod
spec:
  nodeName: k8s-w1
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod2
  labels:
    app: webpod
spec:
  nodeName: k8s-w2
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Service
metadata:
  name: svc1
spec:
  ports:
    - name: svc1-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
  name: svc2
spec:
  ports:
    - name: svc2-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
  name: svc3
spec:
  ports:
    - name: svc3-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
EOF

#
kubectl get svc,ep
 
#
curl -s 10.10.200.1
curl -s 10.10.200.2
curl -s 10.10.200.3
 
# 삭제
~