[K8S] Encrypting Secret Data at Rest

참고 문서 : Encrypting Secret Data at Rest

Encrypting Confidential Data at Rest

 

root@master:~# kubectl create secret generic my-secret --from-literal=key1=supersecret
secret/my-secret created

root@master:~# kubectl get secret
NAME           TYPE     DATA   AGE
my-secret      Opaque   1      12s


root@master:~# kubectl describe secrets my-secret
Name:         my-secret
Namespace:    default
Labels:       <none>
Annotations:  <none>

Type:  Opaque

Data
====
key1:  11 bytes
root@master:~# kubectl get secret my-secret -o yaml
apiVersion: v1
data:
  key1: c3VwZXJzZWNyZXQ=       # 이렇게 encoded된 것 확인 가능하다.
kind: Secret
metadata:
  creationTimestamp: "2024-04-27T13:57:07Z"
  name: my-secret
  namespace: default
  resourceVersion: "111688"
  uid: f0b03651-b1ba-4b8f-9578-063b4c1dca34
type: Opaque

root@master:~# echo "c3VwZXJzZWNyZXQ=" | base64 -d
supersecret

 

• 데이터가 etcd server에 어떻게 저장되는지 확인한다.
• default로 API server는  plain-text 로 etcd에 데이터를 저장한다.

ETCDCTL_API=3 etcdctl \
   --cacert=/etc/kubernetes/pki/etcd/ca.crt   \
   --cert=/etc/kubernetes/pki/etcd/server.crt \
   --key=/etc/kubernetes/pki/etcd/server.key  \
   get /registry/secrets/default/secret1 | hexdump -C

• etcdctl을 사용하기 위해서는 install을 해줘야한다.

$ apt-get install etcd-client

root@master:~# ls /etc/kubernetes/pki/etcd/ca.crt
/etc/kubernetes/pki/etcd/ca.crt
root@master:~# ls /etc/kubernetes/pki/etcd/
ca.crt  ca.key  healthcheck-client.crt  healthcheck-client.key  peer.crt  peer.key  server.crt  server.key


root@master:~# ETCDCTL_API=3 etcdctl \
>    --cacert=/etc/kubernetes/pki/etcd/ca.crt   \
>    --cert=/etc/kubernetes/pki/etcd/server.crt \
>    --key=/etc/kubernetes/pki/etcd/server.key  \
>    get /registry/secrets/default/my-secret
/registry/secrets/default/my-secret
k8s

v1Secret▒
▒
        my-secretdefault"*$f0b03651-b1ba-4b8f-9578-063b4c1dca342▒▒▒▒▒a
kubectl-createUpdatev▒▒▒▒FieldsV1:-
+{"f:data":{".":{},"f:key1":{}},"f:type":{}}B
key1
    supersecretOpaque"
root@master:~#
root@master:~#
root@master:~#
root@master:~#
root@master:~#
root@master:~#
root@master:~#
root@master:~# ETCDCTL_API=3 etcdctl    --cacert=/etc/kubernetes/pki/etcd/ca.crt      --cert=/etc/kubernetes/pki/etcd/server.crt    --key=/etc/kubernetes/pki/etcd/server.key     get /registry/secrets/default/my-secret | hexdump -C
00000000  2f 72 65 67 69 73 74 72  79 2f 73 65 63 72 65 74  |/registry/secret|
00000010  73 2f 64 65 66 61 75 6c  74 2f 6d 79 2d 73 65 63  |s/default/my-sec|
00000020  72 65 74 0a 6b 38 73 00  0a 0c 0a 02 76 31 12 06  |ret.k8s.....v1..|
00000030  53 65 63 72 65 74 12 d0  01 0a b0 01 0a 09 6d 79  |Secret........my|
00000040  2d 73 65 63 72 65 74 12  00 1a 07 64 65 66 61 75  |-secret....defau|
00000050  6c 74 22 00 2a 24 66 30  62 30 33 36 35 31 2d 62  |lt".*$f0b03651-b|
00000060  31 62 61 2d 34 62 38 66  2d 39 35 37 38 2d 30 36  |1ba-4b8f-9578-06|
00000070  33 62 34 63 31 64 63 61  33 34 32 00 38 00 42 08  |3b4c1dca342.8.B.|
00000080  08 b3 88 b4 b1 06 10 00  8a 01 61 0a 0e 6b 75 62  |..........a..kub|
00000090  65 63 74 6c 2d 63 72 65  61 74 65 12 06 55 70 64  |ectl-create..Upd|
000000a0  61 74 65 1a 02 76 31 22  08 08 b3 88 b4 b1 06 10  |ate..v1"........|
000000b0  00 32 08 46 69 65 6c 64  73 56 31 3a 2d 0a 2b 7b  |.2.FieldsV1:-.+{|
000000c0  22 66 3a 64 61 74 61 22  3a 7b 22 2e 22 3a 7b 7d  |"f:data":{".":{}|
000000d0  2c 22 66 3a 6b 65 79 31  22 3a 7b 7d 7d 2c 22 66  |,"f:key1":{}},"f|
000000e0  3a 74 79 70 65 22 3a 7b  7d 7d 42 00 12 13 0a 04  |:type":{}}B.....|
000000f0  6b 65 79 31 12 0b 73 75  70 65 72 73 65 63 72 65  |key1..supersecre|
00000100  74 1a 06 4f 70 61 71 75  65 1a 00 22 00 0a        |t..Opaque.."..|
# 이게 etcd에 저장된 데이터다.

 

 

• kube-apiserver는 `--encryption-provider-config`를 제공한다.

root@master:~# ps -aux | grep kube-api
root      1781  3.1  6.7 1488280 270804 ?      Ssl  22:55   0:18 kube-apiserver --advertise-address=10.100.0.104 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
root      5384  0.0  0.0  15724  1044 pts/0    S+   23:04   0:00 grep --color=auto kube-api
# configured되어있지 않은 것 확인한다.
root@master:~# ps -aux | grep kube-api | grep "encryption-provider-config"
root@master:~#

$ head -c 32 /dev/urandom | base64
JFTx6KIIe5hGTJmDvSNvU3DAwJZ/plGGTkLBLdpiVtE=

$ vi enc.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
      - configmaps
      - pandas.awesome.bears.example
    providers:
      - aescbc:
          keys:
            - name: key1
              # See the following text for more details about the secret value
              secret: JFTx6KIIe5hGTJmDvSNvU3DAwJZ/plGGTkLBLdpiVtE=
      - identity: {} # this fallback allows reading unencrypted secrets;
                     # for example, during initial migration

# 테스트용으로 잠시 폴더 생성
$ mkdir -p /etc/kubernetes/enc/
$ mv enc.yaml /etc/kubernetes/enc/
$ ls /etc/kubernetes/enc/
enc.yaml
# kube-apiserver의 yaml파일을 수정한다.
$ vi /etc/kubenetes/manifests/kube-apiserver.yaml
...
--encryption-provider-config=/etc/kubernetes/enc/enc.yaml  # add this line
	...
    volumeMounts:
    ...
    - name: enc                           # add this line
      mountPath: /etc/kubernetes/enc      # add this line
      readOnly: true                      # add this line
    ...
  volumes:
  ...
  - name: enc                             # add this line
    hostPath:                             # add this line
      path: /etc/kubernetes/enc           # add this line
      type: DirectoryOrCreate             # add this line
  ...
  
$ ps -aux | grep kube-api
root      1781  3.1  6.7 1488280 270804 ?      Ssl  22:55   0:18 kube-apiserver --advertise-address=10.100.0.104 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key --encryption-provider-config=/etc/kubernetes/enc/enc.yaml
root      5384  0.0  0.0  15724  1044 pts/0    S+   23:04   0:00 grep --color=auto kube-api

 

• 모든 Secret 파일을 encrypt 하기

$ kubectl get secrets --all-namespaces -o json | kubectl replace -f -
secret/dashboard-token replaced
secret/db-secret replaced
secret/my-secret replaced
secret/chart-values-traefik replaced
secret/chart-values-traefik-crd replaced
secret/controlplane.node-password.k3s replaced
secret/k3s-serving replaced
secret/sh.helm.release.v1.traefik-crd.v1 replaced
secret/sh.helm.release.v1.traefik.v1 replaced