CKSの勉強をしていて、kube-apiserverの特にセキュリティまわりの設定ポイントを学んだので、EKSの場合はどうなのかを確認したメモ。
フラグの確認方法
kuber-apiserverの起動フラグを確認するには、コントロールプレーンのロギングを有効化して、ログを見ればよい。
CLUSTER_NAME="mycluster"
eksctl utils update-cluster-logging \
--enable-types=all \
--cluster=${CLUSTER_NAME} \
--approve
$ aws logs get-log-events --log-group-name /aws/eks/mycluster/cluster --log-stream-name kube-apiserver-1d24dba23787aec360e79bd7464607f7 | jq -r '.events[].message' | grep FLAG I1228 00:21:20.694581 1 flags.go:33] FLAG: --add-dir-header="false" I1228 00:21:20.694646 1 flags.go:33] FLAG: --address="0.0.0.0" I1228 00:21:20.694653 1 flags.go:33] FLAG: --admission-control="[]" I1228 00:21:20.694663 1 flags.go:33] FLAG: --admission-control-config-file="" I1228 00:21:20.694668 1 flags.go:33] FLAG: --advertise-address="192.168.93.53" I1228 00:21:20.694673 1 flags.go:33] FLAG: --allow-privileged="true" I1228 00:21:20.694678 1 flags.go:33] FLAG: --alsologtostderr="false" I1228 00:21:20.694682 1 flags.go:33] FLAG: --anonymous-auth="true" I1228 00:21:20.694685 1 flags.go:33] FLAG: --api-audiences="[https://kubernetes.default.svc]" I1228 00:21:20.694693 1 flags.go:33] FLAG: --apiserver-count="1" I1228 00:21:20.694700 1 flags.go:33] FLAG: --audit-dynamic-configuration="false" I1228 00:21:20.694705 1 flags.go:33] FLAG: --audit-log-batch-buffer-size="10000" I1228 00:21:20.694713 1 flags.go:33] FLAG: --audit-log-batch-max-size="1" I1228 00:21:20.694720 1 flags.go:33] FLAG: --audit-log-batch-max-wait="0s" I1228 00:21:20.694726 1 flags.go:33] FLAG: --audit-log-batch-throttle-burst="0" I1228 00:21:20.694732 1 flags.go:33] FLAG: --audit-log-batch-throttle-enable="false" I1228 00:21:20.694737 1 flags.go:33] FLAG: --audit-log-batch-throttle-qps="0" I1228 00:21:20.694744 1 flags.go:33] FLAG: --audit-log-format="json" I1228 00:21:20.694749 1 flags.go:33] FLAG: --audit-log-maxage="0" I1228 00:21:20.694752 1 flags.go:33] FLAG: --audit-log-maxbackup="1" I1228 00:21:20.694756 1 flags.go:33] FLAG: --audit-log-maxsize="500" I1228 00:21:20.694760 1 flags.go:33] FLAG: --audit-log-mode="blocking" I1228 00:21:20.694764 1 flags.go:33] FLAG: --audit-log-path="/var/log/kube-audit/kube-apiserver-audit.log" I1228 00:21:20.694768 1 flags.go:33] FLAG: --audit-log-truncate-enabled="false" I1228 00:21:20.694772 1 flags.go:33] FLAG: --audit-log-truncate-max-batch-size="10485760" I1228 00:21:20.694778 1 flags.go:33] FLAG: --audit-log-truncate-max-event-size="102400" I1228 00:21:20.694782 1 flags.go:33] FLAG: --audit-log-version="audit.k8s.io/v1" I1228 00:21:20.694786 1 flags.go:33] FLAG: --audit-policy-file="/etc/audit_policy.config" I1228 00:21:20.694790 1 flags.go:33] FLAG: --audit-webhook-batch-buffer-size="10000" I1228 00:21:20.694794 1 flags.go:33] FLAG: --audit-webhook-batch-initial-backoff="10s" I1228 00:21:20.694798 1 flags.go:33] FLAG: --audit-webhook-batch-max-size="400" I1228 00:21:20.694805 1 flags.go:33] FLAG: --audit-webhook-batch-max-wait="30s" I1228 00:21:20.694809 1 flags.go:33] FLAG: --audit-webhook-batch-throttle-burst="15" I1228 00:21:20.694813 1 flags.go:33] FLAG: --audit-webhook-batch-throttle-enable="true" I1228 00:21:20.694817 1 flags.go:33] FLAG: --audit-webhook-batch-throttle-qps="10" I1228 00:21:20.694821 1 flags.go:33] FLAG: --audit-webhook-config-file="" I1228 00:21:20.694824 1 flags.go:33] FLAG: --audit-webhook-initial-backoff="10s" I1228 00:21:20.694828 1 flags.go:33] FLAG: --audit-webhook-mode="batch" I1228 00:21:20.694832 1 flags.go:33] FLAG: --audit-webhook-truncate-enabled="false" I1228 00:21:20.694836 1 flags.go:33] FLAG: --audit-webhook-truncate-max-batch-size="10485760" I1228 00:21:20.694840 1 flags.go:33] FLAG: --audit-webhook-truncate-max-event-size="102400" I1228 00:21:20.694844 1 flags.go:33] FLAG: --audit-webhook-version="audit.k8s.io/v1" I1228 00:21:20.694848 1 flags.go:33] FLAG: --authentication-token-webhook-cache-ttl="7m0s" I1228 00:21:20.694851 1 flags.go:33] FLAG: --authentication-token-webhook-config-file="/etc/kubernetes/authenticator/apiserver-webhook-kubeconfig.yaml" I1228 00:21:20.694863 1 flags.go:33] FLAG: --authentication-token-webhook-version="v1beta1" I1228 00:21:20.694867 1 flags.go:33] FLAG: --authorization-mode="[Node,RBAC]" I1228 00:21:20.694876 1 flags.go:33] FLAG: --authorization-policy-file="" I1228 00:21:20.694880 1 flags.go:33] FLAG: --authorization-webhook-cache-authorized-ttl="5m0s" I1228 00:21:20.694884 1 flags.go:33] FLAG: --authorization-webhook-cache-unauthorized-ttl="30s" I1228 00:21:20.694888 1 flags.go:33] FLAG: --authorization-webhook-config-file="" I1228 00:21:20.694891 1 flags.go:33] FLAG: --authorization-webhook-version="v1beta1" I1228 00:21:20.694924 1 flags.go:33] FLAG: --basic-auth-file="" I1228 00:21:20.694942 1 flags.go:33] FLAG: --bind-address="0.0.0.0" I1228 00:21:20.694948 1 flags.go:33] FLAG: --cert-dir="/var/run/kubernetes" I1228 00:21:20.694954 1 flags.go:33] FLAG: --client-ca-file="/etc/kubernetes/pki/internal-auth/ca.crt" I1228 00:21:20.694960 1 flags.go:33] FLAG: --cloud-config="/var/lib/kubernetes/aws.config" I1228 00:21:20.694965 1 flags.go:33] FLAG: --cloud-provider="aws" I1228 00:21:20.694970 1 flags.go:33] FLAG: --cloud-provider-gce-l7lb-src-cidrs="130.211.0.0/22,35.191.0.0/16" I1228 00:21:20.694979 1 flags.go:33] FLAG: --cloud-provider-gce-lb-src-cidrs="130.211.0.0/22,209.85.152.0/22,209.85.204.0/22,35.191.0.0/16" I1228 00:21:20.694985 1 flags.go:33] FLAG: --contention-profiling="false" I1228 00:21:20.694988 1 flags.go:33] FLAG: --cors-allowed-origins="[]" I1228 00:21:20.694995 1 flags.go:33] FLAG: --default-not-ready-toleration-seconds="300" I1228 00:21:20.694999 1 flags.go:33] FLAG: --default-unreachable-toleration-seconds="300" I1228 00:21:20.695003 1 flags.go:33] FLAG: --default-watch-cache-size="100" I1228 00:21:20.695007 1 flags.go:33] FLAG: --delete-collection-workers="1" I1228 00:21:20.695010 1 flags.go:33] FLAG: --deserialization-cache-size="0" I1228 00:21:20.695014 1 flags.go:33] FLAG: --disable-admission-plugins="[]" I1228 00:21:20.695022 1 flags.go:33] FLAG: --egress-selector-config-file="" I1228 00:21:20.695026 1 flags.go:33] FLAG: --enable-admission-plugins="[NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,PodSecurityPolicy]" I1228 00:21:20.695038 1 flags.go:33] FLAG: --enable-aggregator-routing="true" I1228 00:21:20.695044 1 flags.go:33] FLAG: --enable-bootstrap-token-auth="false" I1228 00:21:20.695047 1 flags.go:33] FLAG: --enable-garbage-collector="true" I1228 00:21:20.695051 1 flags.go:33] FLAG: --enable-logs-handler="false" I1228 00:21:20.695055 1 flags.go:33] FLAG: --enable-priority-and-fairness="true" I1228 00:21:20.695058 1 flags.go:33] FLAG: --enable-swagger-ui="false" I1228 00:21:20.695062 1 flags.go:33] FLAG: --encryption-provider-config="/etc/kubernetes/aws-encryption-provider/encryption-configuration.yaml" I1228 00:21:20.695067 1 flags.go:33] FLAG: --endpoint-reconciler-type="lease" I1228 00:21:20.695071 1 flags.go:33] FLAG: --etcd-cafile="" I1228 00:21:20.695074 1 flags.go:33] FLAG: --etcd-certfile="" I1228 00:21:20.695078 1 flags.go:33] FLAG: --etcd-compaction-interval="5m0s" I1228 00:21:20.695082 1 flags.go:33] FLAG: --etcd-count-metric-poll-period="1m0s" I1228 00:21:20.695086 1 flags.go:33] FLAG: --etcd-keyfile="" I1228 00:21:20.695089 1 flags.go:33] FLAG: --etcd-prefix="/registry" I1228 00:21:20.695093 1 flags.go:33] FLAG: --etcd-servers="[http://internal-eks-e7302-EtcdLoad-SKNFU7URACDP-889187838.ap-northeast-1.elb.amazonaws.com:2379]" I1228 00:21:20.695102 1 flags.go:33] FLAG: --etcd-servers-overrides="[]" I1228 00:21:20.695107 1 flags.go:33] FLAG: --event-ttl="1h0m0s" I1228 00:21:20.695111 1 flags.go:33] FLAG: --experimental-encryption-provider-config="/etc/kubernetes/aws-encryption-provider/encryption-configuration.yaml" I1228 00:21:20.695121 1 flags.go:33] FLAG: --external-hostname="ip-10-0-52-192.ap-northeast-1.compute.internal" I1228 00:21:20.695125 1 flags.go:33] FLAG: --feature-gates="ExternalKeyService=true" I1228 00:21:20.695137 1 flags.go:33] FLAG: --goaway-chance="0" I1228 00:21:20.695142 1 flags.go:33] FLAG: --help="false" I1228 00:21:20.695146 1 flags.go:33] FLAG: --http2-max-streams-per-connection="0" I1228 00:21:20.695150 1 flags.go:33] FLAG: --insecure-bind-address="0.0.0.0" I1228 00:21:20.695154 1 flags.go:33] FLAG: --insecure-port="8080" I1228 00:21:20.695158 1 flags.go:33] FLAG: --key-service-url="/etc/kubernetes/key-server/pipe.sock" I1228 00:21:20.695162 1 flags.go:33] FLAG: --kubelet-certificate-authority="/etc/kubernetes/pki/ca.crt" I1228 00:21:20.695166 1 flags.go:33] FLAG: --kubelet-client-certificate="/etc/kubernetes/pki/apiserver-kubelet-client.crt" I1228 00:21:20.695171 1 flags.go:33] FLAG: --kubelet-client-key="/etc/kubernetes/pki/apiserver-kubelet-client.key" I1228 00:21:20.695175 1 flags.go:33] FLAG: --kubelet-https="true" I1228 00:21:20.695179 1 flags.go:33] FLAG: --kubelet-port="10250" I1228 00:21:20.695184 1 flags.go:33] FLAG: --kubelet-preferred-address-types="[InternalIP,InternalDNS]" I1228 00:21:20.695192 1 flags.go:33] FLAG: --kubelet-read-only-port="10255" I1228 00:21:20.695196 1 flags.go:33] FLAG: --kubelet-timeout="5s" I1228 00:21:20.695200 1 flags.go:33] FLAG: --kubernetes-service-node-port="0" I1228 00:21:20.695203 1 flags.go:33] FLAG: --livez-grace-period="0s" I1228 00:21:20.695207 1 flags.go:33] FLAG: --log-backtrace-at=":0" I1228 00:21:20.695212 1 flags.go:33] FLAG: --log-dir="" I1228 00:21:20.695215 1 flags.go:33] FLAG: --log-file="" I1228 00:21:20.695221 1 flags.go:33] FLAG: --log-file-max-size="1800" I1228 00:21:20.695225 1 flags.go:33] FLAG: --log-flush-frequency="5s" I1228 00:21:20.695229 1 flags.go:33] FLAG: --logtostderr="true" I1228 00:21:20.695232 1 flags.go:33] FLAG: --master-service-namespace="default" I1228 00:21:20.695236 1 flags.go:33] FLAG: --max-connection-bytes-per-sec="0" I1228 00:21:20.695240 1 flags.go:33] FLAG: --max-mutating-requests-inflight="200" I1228 00:21:20.695244 1 flags.go:33] FLAG: --max-requests-inflight="400" I1228 00:21:20.695247 1 flags.go:33] FLAG: --min-request-timeout="1800" I1228 00:21:20.695251 1 flags.go:33] FLAG: --oidc-ca-file="" I1228 00:21:20.695255 1 flags.go:33] FLAG: --oidc-client-id="" I1228 00:21:20.695258 1 flags.go:33] FLAG: --oidc-groups-claim="" I1228 00:21:20.695262 1 flags.go:33] FLAG: --oidc-groups-prefix="" I1228 00:21:20.695265 1 flags.go:33] FLAG: --oidc-issuer-url="" I1228 00:21:20.695268 1 flags.go:33] FLAG: --oidc-required-claim="" I1228 00:21:20.695274 1 flags.go:33] FLAG: --oidc-signing-algs="[RS256]" I1228 00:21:20.695282 1 flags.go:33] FLAG: --oidc-username-claim="sub" I1228 00:21:20.695286 1 flags.go:33] FLAG: --oidc-username-prefix="" I1228 00:21:20.695289 1 flags.go:33] FLAG: --port="8080" I1228 00:21:20.695293 1 flags.go:33] FLAG: --profiling="false" I1228 00:21:20.695297 1 flags.go:33] FLAG: --proxy-cidr-whitelist="192.168.0.0/16" I1228 00:21:20.695305 1 flags.go:33] FLAG: --proxy-client-cert-file="/etc/kubernetes/pki/front-proxy-client.crt" I1228 00:21:20.695309 1 flags.go:33] FLAG: --proxy-client-key-file="/etc/kubernetes/pki/front-proxy-client.key" I1228 00:21:20.695313 1 flags.go:33] FLAG: --request-timeout="1m0s" I1228 00:21:20.695317 1 flags.go:33] FLAG: --requestheader-allowed-names="[front-proxy-client]" I1228 00:21:20.695324 1 flags.go:33] FLAG: --requestheader-client-ca-file="/etc/kubernetes/pki/front-proxy-ca.crt" I1228 00:21:20.695329 1 flags.go:33] FLAG: --requestheader-extra-headers-prefix="[X-Remote-Extra-]" I1228 00:21:20.695337 1 flags.go:33] FLAG: --requestheader-group-headers="[X-Remote-Group]" I1228 00:21:20.695344 1 flags.go:33] FLAG: --requestheader-username-headers="[X-Remote-User]" I1228 00:21:20.695353 1 flags.go:33] FLAG: --runtime-config="" I1228 00:21:20.695358 1 flags.go:33] FLAG: --secure-port="443" I1228 00:21:20.695362 1 flags.go:33] FLAG: --service-account-api-audiences="[https://kubernetes.default.svc]" I1228 00:21:20.695370 1 flags.go:33] FLAG: --service-account-issuer="https://oidc.eks.ap-northeast-1.amazonaws.com/id/26BA04AD024EF5657DA104B8C1F00FEF" I1228 00:21:20.695375 1 flags.go:33] FLAG: --service-account-jwks-uri="" I1228 00:21:20.695378 1 flags.go:33] FLAG: --service-account-key-file="[/etc/kubernetes/pki/sa.pub]" I1228 00:21:20.695386 1 flags.go:33] FLAG: --service-account-lookup="true" I1228 00:21:20.695413 1 flags.go:33] FLAG: --service-account-max-token-expiration="24h0m0s" I1228 00:21:20.695418 1 flags.go:33] FLAG: --service-account-signing-key-file="" I1228 00:21:20.695422 1 flags.go:33] FLAG: --service-cluster-ip-range="10.100.0.0/16" I1228 00:21:20.695426 1 flags.go:33] FLAG: --service-node-port-range="30000-32767" I1228 00:21:20.695436 1 flags.go:33] FLAG: --show-hidden-metrics-for-version="" I1228 00:21:20.695440 1 flags.go:33] FLAG: --shutdown-delay-duration="0s" I1228 00:21:20.695444 1 flags.go:33] FLAG: --skip-headers="false" I1228 00:21:20.695448 1 flags.go:33] FLAG: --skip-log-headers="false" I1228 00:21:20.695452 1 flags.go:33] FLAG: --ssh-keyfile="" I1228 00:21:20.695456 1 flags.go:33] FLAG: --ssh-user="" I1228 00:21:20.695460 1 flags.go:33] FLAG: --stderrthreshold="2" I1228 00:21:20.695464 1 flags.go:33] FLAG: --storage-backend="" I1228 00:21:20.695468 1 flags.go:33] FLAG: --storage-media-type="application/vnd.kubernetes.protobuf" I1228 00:21:20.695473 1 flags.go:33] FLAG: --target-ram-mb="0" I1228 00:21:20.695477 1 flags.go:33] FLAG: --tls-cert-file="/etc/kubernetes/pki/apiserver.crt" I1228 00:21:20.695482 1 flags.go:33] FLAG: --tls-cipher-suites="[TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]" I1228 00:21:20.695492 1 flags.go:33] FLAG: --tls-min-version="" I1228 00:21:20.695496 1 flags.go:33] FLAG: --tls-private-key-file="/etc/kubernetes/pki/apiserver.key" I1228 00:21:20.695501 1 flags.go:33] FLAG: --tls-sni-cert-key="[]" I1228 00:21:20.695506 1 flags.go:33] FLAG: --token-auth-file="/dev/null" I1228 00:21:20.695511 1 flags.go:33] FLAG: --v="2" I1228 00:21:20.695515 1 flags.go:33] FLAG: --version="false" I1228 00:21:20.695521 1 flags.go:33] FLAG: --vmodule="" I1228 00:21:20.695525 1 flags.go:33] FLAG: --watch-cache="true" I1228 00:21:20.695531 1 flags.go:33] FLAG: --watch-cache-sizes="[]"
aws logs get-log-eventsのデフォルトでは1MBまたは10000イベントが取得できる。
どのログストリームに起動時のログがあるかわからない場合は、Logs Insights で以下のようなクエリを指定すればよい。
fields @timestamp, @message | filter @logStream like "kube-apiserver-" | filter @logStream not like "kube-apiserver-audit" | filter @message like "FLAG" | sort @timestamp desc | limit 1000
確認
anonymousアクセス
設定がない場合のデフォルトは有効だが、明示的に有効になっている。
I1228 00:21:20.694682 1 flags.go:33] FLAG: --anonymous-auth="true"
アクセスしてみると、確かに有効。
$ curl https://26BA04AD024EF5657DA104B8C1F00FEF.gr7.ap-northeast-1.eks.amazonaws.com -k
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "forbidden: User \"system:anonymous\" cannot get path \"/\"",
"reason": "Forbidden",
"details": {
},
"code": 403
}
Insecureポート
設定自体は有効になっているが、セキュリティグループか何かでブロックされているものと想像。
I1228 00:21:20.695154 1 flags.go:33] FLAG: --insecure-port="8080"
NodeRestriction
有効になっている。
I1228 00:21:20.695026 1 flags.go:33] FLAG: --enable-admission-plugins="[NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,PodSecurityPolicy]"
