CNI Plugin Helperが公開しているメトリクスにkubectlで直接アクセスしてみるメモ。
kubectl port-forward
メトリクスは61678ポートで公開されているのでこのポートにポートフォワードし、http://localhost:61678/metrics にアクセスする。
$ kubectl port-forward aws-node-hpd82 61678 Forwarding from 127.0.0.1:61678 -> 61678 Forwarding from [::1]:61678 -> 61678 Handling connection for 61678 Handling connection for 61678 Handling connection for 61678
kubectl get --raw
apiserverのプロキシを利用してアクセスする。ポート番号は指定しなくても大丈夫。
$ kubectl get --raw /api/v1/namespaces/kube-system/pods/aws-node-hpd82:61678/proxy/metrics
# HELP awscni_add_ip_req_count The number of add IP address requests
# TYPE awscni_add_ip_req_count counter
awscni_add_ip_req_count 14
# HELP awscni_assigned_ip_addresses The number of IP addresses assigned to pods
# TYPE awscni_assigned_ip_addresses gauge
awscni_assigned_ip_addresses 14
# HELP awscni_assigned_ip_per_ipv4cidr The total number of IP addresses assigned per cidr
# TYPE awscni_assigned_ip_per_ipv4cidr gauge
awscni_assigned_ip_per_ipv4cidr{cidr="192.168.192.64/28"} 12
awscni_assigned_ip_per_ipv4cidr{cidr="192.168.246.0/28"} 2
# HELP awscni_aws_api_latency_ms AWS API call latency in ms
# TYPE awscni_aws_api_latency_ms summary
awscni_aws_api_latency_ms_sum{api="DescribeNetworkInterfaces",error="false",status="200"} 252
awscni_aws_api_latency_ms_count{api="DescribeNetworkInterfaces",error="false",status="200"} 1
awscni_aws_api_latency_ms_sum{api="GetMetadata",error="false",status="200"} 9617
awscni_aws_api_latency_ms_count{api="GetMetadata",error="false",status="200"} 44774
awscni_aws_api_latency_ms_sum{api="ModifyNetworkInterfaceAttribute",error="false",status="200"} 478
awscni_aws_api_latency_ms_count{api="ModifyNetworkInterfaceAttribute",error="false",status="200"} 1
# HELP awscni_del_ip_req_count The number of delete IP address requests
# TYPE awscni_del_ip_req_count counter
awscni_del_ip_req_count{reason="PodDeleted"} 14
# HELP awscni_eni_allocated The number of ENIs allocated
# TYPE awscni_eni_allocated gauge
awscni_eni_allocated 1
# HELP awscni_eni_max The maximum number of ENIs that can be attached to the instance, accounting for unmanaged ENIs
# TYPE awscni_eni_max gauge
awscni_eni_max 3
# HELP awscni_force_removed_enis The number of ENIs force removed while they had assigned pods
# TYPE awscni_force_removed_enis counter
awscni_force_removed_enis 0
# HELP awscni_force_removed_ips The number of IPs force removed while they had assigned pods
# TYPE awscni_force_removed_ips counter
awscni_force_removed_ips 0
# HELP awscni_ip_max The maximum number of IP addresses that can be allocated to the instance
# TYPE awscni_ip_max gauge
awscni_ip_max 432
# HELP awscni_ipamd_action_inprogress The number of ipamd actions in progress
# TYPE awscni_ipamd_action_inprogress gauge
awscni_ipamd_action_inprogress{fn="nodeIPPoolReconcile"} 0
awscni_ipamd_action_inprogress{fn="nodeInit"} 0
# HELP awscni_reconcile_count The number of times ipamd reconciles on ENIs and IP/Prefix addresses
# TYPE awscni_reconcile_count counter
awscni_reconcile_count{fn="eniDataStorePoolReconcileAdd"} 8134
# HELP awscni_total_ip_addresses The total number of IP addresses
# TYPE awscni_total_ip_addresses gauge
awscni_total_ip_addresses 32
# HELP awscni_total_ipv4_prefixes The total number of IPv4 prefixes
# TYPE awscni_total_ipv4_prefixes gauge
awscni_total_ipv4_prefixes 2
# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 1.1927e-05
go_gc_duration_seconds{quantile="0.25"} 3.381e-05
go_gc_duration_seconds{quantile="0.5"} 4.1182e-05
go_gc_duration_seconds{quantile="0.75"} 5.5597e-05
go_gc_duration_seconds{quantile="1"} 0.002618703
go_gc_duration_seconds_sum 0.147290686
go_gc_duration_seconds_count 2040
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 35
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.14.15"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 6.42528e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 6.437947936e+09
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 1.635703e+06
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 1.7240465e+07
# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
# TYPE go_memstats_gc_cpu_fraction gauge
go_memstats_gc_cpu_fraction 9.738090810849369e-06
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 3.594504e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 6.42528e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 5.697536e+07
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 9.37984e+06
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 33536
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 5.3952512e+07
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 6.63552e+07
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.6310513580072515e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 0
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 1.7274001e+07
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 3472
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 16384
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 140352
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 163840
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 1.264744e+07
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 438913
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 753664
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 753664
# HELP go_memstats_sys_bytes Number of bytes obtained from system.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 7.2958208e+07
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 8
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 101.08
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 19
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 7.008256e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.63080734446e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 7.87320832e+08
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes -1
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 4
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0
