kubectl Error from server (Forbidden) 403 원인 5가지와 RBAC 해결법

Error from server (Forbidden): pods is forbidden: User "system:serviceaccount:dev:ci-deployer" cannot list resource "pods" in API group "" in the namespace "prod"

kubectl / SDK
   │
   ├─ ① 인증  : 이 요청의 주체는 누구인가        → 실패 시 401
   ├─ ② 인가  : 그 주체가 이 동작을 해도 되는가   → 실패 시 403 (이 글)
   └─ ③ 어드미션: 이 오브젝트 스펙을 허용할 것인가 → PSA 위반 시 create 거부

Error from server (Forbidden): pods is forbidden: User "system:serviceaccount:dev:ci-deployer" cannot list resource "pods" in API group "" in the namespace "prod"

Error from server (Forbidden): nodes is forbidden: User "system:anonymous" cannot get path "/api/v1/nodes"

Error from server (Forbidden): deployments.apps is forbidden: User "arn:aws:iam::123456789012:role/eks-dev-role" cannot create resource "deployments" in API group "apps" at the cluster scope

kubectl api-resources -o wide | head -20

NAME          SHORTNAMES   APIVERSION   NAMESPACED   KIND         VERBS
configmaps    cm           v1           true         ConfigMap    create,delete,get,list,patch,update,watch
pods          po           v1           true         Pod          create,delete,get,list,patch,update,watch
deployments   deploy       apps/v1      true         Deployment   create,delete,get,list,patch,update,watch

# 잘못된 예 — 이 YAML은 apply가 성공하지만 절대 매칭되지 않습니다
subjects:
  - kind: ServiceAccount
    name: system:serviceaccount:dev:ci-deployer   # ← 풀네임을 넣으면 안 됨

# 올바른 예
subjects:
  - kind: ServiceAccount
    name: ci-deployer      # SA 이름만
    namespace: dev         # ServiceAccount subject는 namespace 필수

kubectl config current-context
kubectl config view --minify -o jsonpath='{.contexts[0].context.user}{"\n"}'

# 특정 ServiceAccount가 prod 네임스페이스에서 가진 권한 전량
kubectl auth can-i --list \
  --as=system:serviceaccount:dev:ci-deployer \
  -n prod

Resources                                       Non-Resource URLs   Resource Names   Verbs
selfsubjectaccessreviews.authorization.k8s.io   []                  []               [create]
selfsubjectrulesreviews.authorization.k8s.io    []                  []               [create]
                                                [/api/*]            []               [get]
                                                [/healthz]          []               [get]

kubectl auth can-i list pods --as=system:serviceaccount:dev:ci-deployer -n prod
# → no

kubectl auth can-i create deployments.apps --as=system:serviceaccount:dev:ci-deployer -n prod
# → yes

# 하위 리소스도 직접 확인 가능
kubectl auth can-i get pods/log --as=system:serviceaccount:dev:ci-deployer -n prod
# → no

# 그룹 기준 확인
kubectl auth can-i list secrets [email protected] --as-group=platform-team -n prod

SUBJECT="ci-deployer"
SUBJECT_NS="dev"

kubectl get clusterrolebinding,rolebinding -A -o json \
| jq -r --arg n "$SUBJECT" --arg ns "$SUBJECT_NS" '
  .items[]
  | select(
      (.subjects // [])
      | any(
          (.name == $n and (.kind == "ServiceAccount") and (.namespace == $ns))
          or (.name == ("system:serviceaccount:" + $ns + ":" + $n))
        )
    )
  | "\(.kind)\t\(.metadata.namespace // "-")/\(.metadata.name)\t-> \(.roleRef.kind)/\(.roleRef.name)"
' | column -t

RoleBinding         prod/ci-deployer-deploy   -> Role/ci-deployer
ClusterRoleBinding  -/metrics-reader          -> ClusterRole/view

# ClusterRole의 rules를 사람이 읽기 좋게
kubectl describe clusterrole view | head -30

# 네임스페이스 Role 원문
kubectl get role ci-deployer -n prod -o yaml

kubectl -n kube-system get configmap aws-auth -o yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: aws-auth
  namespace: kube-system
data:
  mapRoles: |
    - rolearn: arn:aws:iam::123456789012:role/eks-node-role
      username: system:node:{{EC2PrivateDNSName}}
      groups:
        - system:bootstrappers
        - system:nodes

aws eks list-access-entries --cluster-name my-cluster
aws eks list-associated-access-policies --cluster-name my-cluster \
  --principal-arn arn:aws:iam::123456789012:role/eks-dev-role

# 감사 로그 파일이 있는 컨트롤 플레인 노드에서
jq -c 'select(.annotations["authorization.k8s.io/decision"]=="forbid")
       | {user: .user.username, verb, uri: .requestURI,
          reason: .annotations["authorization.k8s.io/reason"]}' \
  /var/log/kubernetes/audit.log | tail -20

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ns-readonly
  namespace: prod
rules:
  - apiGroups: [""]                       # core 그룹
    resources: ["pods", "services", "configmaps", "events",
                "persistentvolumeclaims"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["deployments", "replicasets", "statefulsets", "daemonsets"]
    verbs: ["get", "list", "watch"]
  # secrets는 의도적으로 제외 — get 하나로 DB 비밀번호·토큰이 전부 노출됩니다.
  # pods/log도 제외 — 로그에 개인정보·토큰이 찍히는 경우가 흔합니다.
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ns-readonly-dev-team
  namespace: prod
subjects:
  - kind: Group
    name: dev-team                        # User/Group은 namespace 필드 없음
    apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: ns-readonly
  apiGroup: rbac.authorization.k8s.io

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ns-debugger
  namespace: prod
  annotations:
    security.internal/expires-at: "2026-08-05T00:00:00Z"   # 회수 기한 명시
    security.internal/ticket: "OPS-1234"
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get"]
  - apiGroups: [""]
    resources: ["pods/exec", "pods/portforward"]
    verbs: ["create"]                     # exec/portforward는 create 동사입니다
  # delete는 제외 — 디버깅 중 파드 삭제로 장애를 키우는 사고 방지
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ns-debugger-oncall
  namespace: prod
subjects:
  - kind: User
    name: [email protected]
    apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: ns-debugger
  apiGroup: rbac.authorization.k8s.io

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ci-deployer
  namespace: prod
rules:
  - apiGroups: ["apps"]
    resources: ["deployments", "replicasets"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: ["apps"]
    resources: ["deployments/scale"]      # kubectl scale / HPA 연동
    verbs: ["get", "update", "patch"]
  - apiGroups: [""]
    resources: ["services", "configmaps"]
    verbs: ["get", "list", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["pods", "pods/log"]
    verbs: ["get", "list"]                # 롤아웃 실패 원인 확인용
  # create secrets 제외 — SA 토큰 Secret을 만들어 다른 SA 권한을 탈취하는
  # 권한 상승 경로가 열립니다. 시크릿은 External Secrets 등 별도 경로로.
  # delete deployments 제외 — 롤백은 update/patch로 충분합니다.
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: ci-deployer
  namespace: dev
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ci-deployer-prod
  namespace: prod                          # 권한이 적용될 ns
subjects:
  - kind: ServiceAccount
    name: ci-deployer
    namespace: dev                         # SA가 실제 존재하는 ns (필수)
roleRef:
  kind: Role
  name: ci-deployer
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f rbac.yaml

kubectl auth can-i --list \
  --as=system:serviceaccount:dev:ci-deployer -n prod

kubectl auth can-i update deployments.apps/scale \
  --as=system:serviceaccount:dev:ci-deployer -n prod

kubectl exec -it <pod> -n dev -- sh -c '
  TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token);
  curl -s -o /dev/null -w "%{http_code}\n" \
    --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
    -H "Authorization: Bearer $TOKEN" \
    https://kubernetes.default.svc/api/v1/namespaces/prod/pods
'

# 절대 하지 마세요
kubectl create clusterrolebinding temp-fix \
  --clusterrole=cluster-admin \
  --serviceaccount=dev:ci-deployer

kubectl get clusterrolebinding -o json \
| jq -r '.items[]
    | select(.roleRef.name=="cluster-admin")
    | "\(.metadata.name)\t\((.subjects // []) | map(.kind + ":" + .name) | join(","))"' \
| column -t