diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 18:42:06 +0000 |
commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
tree | 78be5963ec075d80116a932011d695dd33910b4e /config/prometheus | |
parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) | |
download | gitlab-ce-6e4e1050d9dba2b7b2523fdd1768823ab85feef4.tar.gz |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'config/prometheus')
-rw-r--r-- | config/prometheus/cluster_metrics.yml | 3 | ||||
-rw-r--r-- | config/prometheus/common_metrics.yml | 28 | ||||
-rw-r--r-- | config/prometheus/pod_metrics.yml | 106 | ||||
-rw-r--r-- | config/prometheus/self_monitoring_default.yml | 2 |
4 files changed, 82 insertions, 57 deletions
diff --git a/config/prometheus/cluster_metrics.yml b/config/prometheus/cluster_metrics.yml index 1e396f4bbbd..a76a162820a 100644 --- a/config/prometheus/cluster_metrics.yml +++ b/config/prometheus/cluster_metrics.yml @@ -2,12 +2,10 @@ dashboard: 'Cluster health' priority: 1 panel_groups: - group: Cluster Health - priority: 10 panels: - title: "CPU Usage" type: "area-chart" y_label: "CPU (cores)" - weight: 1 metrics: - id: cluster_health_cpu_usage query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' @@ -24,7 +22,6 @@ panel_groups: - title: "Memory Usage" type: "area-chart" y_label: "Memory (GiB)" - weight: 1 metrics: - id: cluster_health_memory_usage query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' diff --git a/config/prometheus/common_metrics.yml b/config/prometheus/common_metrics.yml index d9aaff12a4d..67994740ff1 100644 --- a/config/prometheus/common_metrics.yml +++ b/config/prometheus/common_metrics.yml @@ -2,12 +2,10 @@ dashboard: 'Environment metrics' priority: 1 panel_groups: - group: System metrics (Kubernetes) - priority: 15 panels: - title: "Memory Usage (Total)" type: "area-chart" y_label: "Total Memory Used (GB)" - weight: 4 metrics: - id: system_metrics_kubernetes_container_memory_total # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -18,7 +16,6 @@ panel_groups: - title: "Core Usage (Total)" type: "area-chart" y_label: "Total Cores" - weight: 3 metrics: - id: system_metrics_kubernetes_container_cores_total # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -29,7 +26,6 @@ panel_groups: - title: "Memory Usage (Pod average)" type: "line-chart" y_label: "Memory Used per Pod (MB)" - weight: 2 metrics: - id: system_metrics_kubernetes_container_memory_average # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -40,7 +36,6 @@ panel_groups: - title: "Canary: Memory Usage (Pod Average)" type: "line-chart" y_label: "Memory Used per Pod (MB)" - weight: 2 metrics: - id: system_metrics_kubernetes_container_memory_average_canary # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -52,7 +47,6 @@ panel_groups: - title: "Core Usage (Pod Average)" type: "line-chart" y_label: "Cores per Pod" - weight: 1 metrics: - id: system_metrics_kubernetes_container_core_usage # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -63,7 +57,6 @@ panel_groups: - title: "Canary: Core Usage (Pod Average)" type: "line-chart" y_label: "Cores per Pod" - weight: 1 metrics: - id: system_metrics_kubernetes_container_core_usage_canary # Remove the second metric (after OR) when we drop support for K8s 1.13 @@ -75,7 +68,6 @@ panel_groups: - title: "Knative function invocations" type: "area-chart" y_label: "Invocations" - weight: 1 metrics: - id: system_metrics_knative_function_invocation_count query_range: 'sum(ceil(rate(istio_requests_total{destination_service_namespace="{{kube_namespace}}", destination_service=~"{{function_name}}.*"}[1m])*60))' @@ -83,12 +75,10 @@ panel_groups: unit: requests # NGINX Ingress metrics for pre-0.16.0 versions - group: Response metrics (NGINX Ingress VTS) - priority: 10 panels: - title: "Throughput" type: "area-chart" y_label: "Requests / Sec" - weight: 1 metrics: - id: response_metrics_nginx_ingress_throughput_status_code query_range: 'sum(rate(nginx_upstream_responses_total{upstream=~"{{kube_namespace}}-{{ci_environment_slug}}-.*"}[2m])) by (status_code)' @@ -99,7 +89,6 @@ panel_groups: y_label: "Latency (ms)" y_axis: format: milliseconds - weight: 1 metrics: - id: response_metrics_nginx_ingress_latency_pod_average query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"{{kube_namespace}}-{{ci_environment_slug}}-.*"})' @@ -110,7 +99,6 @@ panel_groups: y_label: "HTTP Errors (%)" y_axis: format: percentHundred - weight: 1 metrics: - id: response_metrics_nginx_ingress_http_error_rate query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"{{kube_namespace}}-{{ci_environment_slug}}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"{{kube_namespace}}-{{ci_environment_slug}}-.*"}[2m])) * 100' @@ -118,12 +106,10 @@ panel_groups: unit: "%" # NGINX Ingress metrics for post-0.16.0 versions - group: Response metrics (NGINX Ingress) - priority: 10 panels: - title: "Throughput" type: "area-chart" y_label: "Requests / Sec" - weight: 1 metrics: - id: response_metrics_nginx_ingress_16_throughput_status_code query_range: 'sum(label_replace(rate(nginx_ingress_controller_requests{namespace="{{kube_namespace}}",ingress=~".*{{ci_environment_slug}}.*"}[2m]), "status_code", "${1}xx", "status", "(.)..")) by (status_code)' @@ -132,7 +118,6 @@ panel_groups: - title: "Latency" type: "area-chart" y_label: "Latency (ms)" - weight: 1 metrics: - id: response_metrics_nginx_ingress_16_latency_pod_average query_range: 'sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="{{kube_namespace}}",ingress=~".*{{ci_environment_slug}}.*"}[2m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="{{kube_namespace}}",ingress=~".*{{ci_environment_slug}}.*"}[2m])) * 1000' @@ -141,19 +126,16 @@ panel_groups: - title: "HTTP Error Rate" type: "area-chart" y_label: "HTTP Errors (%)" - weight: 1 metrics: - id: response_metrics_nginx_ingress_16_http_error_rate query_range: 'sum(rate(nginx_ingress_controller_requests{status=~"5.*",namespace="{{kube_namespace}}",ingress=~".*{{ci_environment_slug}}.*"}[2m])) / sum(rate(nginx_ingress_controller_requests{namespace="{{kube_namespace}}",ingress=~".*{{ci_environment_slug}}.*"}[2m])) * 100' label: 5xx Errors (%) unit: "%" - group: Response metrics (HA Proxy) - priority: 10 panels: - title: "Throughput" type: "area-chart" y_label: "Requests / Sec" - weight: 1 metrics: - id: response_metrics_ha_proxy_throughput_status_code query_range: 'sum(rate(haproxy_frontend_http_requests_total{ {{environment_filter}} }[2m])) by (code)' @@ -162,19 +144,16 @@ panel_groups: - title: "HTTP Error Rate" type: "area-chart" y_label: "Error Rate (%)" - weight: 1 metrics: - id: response_metrics_ha_proxy_http_error_rate query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",{{environment_filter}} }[2m])) / sum(rate(haproxy_frontend_http_responses_total{ {{environment_filter}} }[2m]))' label: HTTP Errors (%) unit: "%" - group: Response metrics (AWS ELB) - priority: 10 panels: - title: "Throughput" type: "area-chart" y_label: "Requests / Sec" - weight: 1 metrics: - id: response_metrics_aws_elb_throughput_requests query_range: 'sum(aws_elb_request_count_sum{ {{environment_filter}} }) / 60' @@ -183,7 +162,6 @@ panel_groups: - title: "Latency" type: "area-chart" y_label: "Latency (ms)" - weight: 1 metrics: - id: response_metrics_aws_elb_latency_average query_range: 'avg(aws_elb_latency_average{ {{environment_filter}} }) * 1000' @@ -192,19 +170,16 @@ panel_groups: - title: "HTTP Error Rate" type: "area-chart" y_label: "Error Rate (%)" - weight: 1 metrics: - id: response_metrics_aws_elb_http_error_rate query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{ {{environment_filter}} }) / sum(aws_elb_request_count_sum{ {{environment_filter}} })' label: HTTP Errors (%) unit: "%" - group: Response metrics (NGINX) - priority: 10 panels: - title: "Throughput" type: "area-chart" y_label: "Requests / Sec" - weight: 1 metrics: - id: response_metrics_nginx_throughput_status_code query_range: 'sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", {{environment_filter}} }[2m])) by (code)' @@ -213,7 +188,6 @@ panel_groups: - title: "Latency" type: "area-chart" y_label: "Latency (ms)" - weight: 1 metrics: - id: response_metrics_nginx_latency query_range: 'avg(nginx_server_requestMsec{ {{environment_filter}} })' @@ -224,7 +198,6 @@ panel_groups: y_label: "HTTP 500 Errors / Sec" y_axis: precision: 0 - weight: 1 metrics: - id: response_metrics_nginx_http_error_rate query_range: 'sum(rate(nginx_server_requests{code="5xx", {{environment_filter}} }[2m]))' @@ -233,7 +206,6 @@ panel_groups: - title: "HTTP Error Rate" type: "area-chart" y_label: "HTTP Errors (%)" - weight: 1 metrics: - id: response_metrics_nginx_http_error_percentage query_range: 'sum(rate(nginx_server_requests{code=~"5.*", host="*", {{environment_filter}} }[2m])) / sum(rate(nginx_server_requests{code="total", host="*", {{environment_filter}} }[2m])) * 100' diff --git a/config/prometheus/pod_metrics.yml b/config/prometheus/pod_metrics.yml index 29575ec543e..b9395124405 100644 --- a/config/prometheus/pod_metrics.yml +++ b/config/prometheus/pod_metrics.yml @@ -1,59 +1,115 @@ -dashboard: 'Pod metrics' -priority: 10 +dashboard: 'K8s pod health' +priority: 2 + +templating: + variables: + pod: + label: 'Pod name' + type: metric_label_values + options: + series_selector: 'container_memory_working_set_bytes' + label: 'pod' + panel_groups: - group: CPU metrics panels: - title: "CPU usage" type: "line-chart" - y_label: "Cores per pod" + y_label: "Cores per container" metrics: - id: pod_cpu_usage_seconds_total - query_range: 'rate(container_cpu_usage_seconds_total{pod_name="{{pod_name}}",container_name="POD"}[5m])' + query_range: >- + sum( + rate(container_cpu_usage_seconds_total{pod="{{pod}}",container!="POD"}[5m]) + ) + by (container) unit: "cores" - label: pod_name + label: container + + - title: "CPU throttling" + type: "line-chart" + y_label: "Cores per container" + metrics: + - id: pod_cpu_cfs_throttle + query_range: >- + sum( + rate(container_cpu_cfs_throttled_seconds_total{pod="{{pod}}"}[5m]) + ) + by (container) + unit: "cores" + label: container + - group: Memory metrics panels: - title: "Memory usage working set" type: "line-chart" - y_label: "Working set memory (MiB)" + y_label: "Working set memory" metrics: - id: pod_memory_working_set - query_range: 'container_memory_working_set_bytes{pod_name="{{pod_name}}",container_name="POD"}/1024/1024' - unit: "MiB" - label: pod_name + query_range: >- + sum( + container_memory_working_set_bytes{pod="{{pod}}",container!="POD"} + ) by (container) + unit: "bytes" + label: container + - group: Network metrics panels: - title: "Network Receive (In)" type: "line-chart" - y_label: "Received (KiB/sec)" + y_label: "Received (bytes/sec)" metrics: - id: pod_network_receive - query_range: 'rate(container_network_receive_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024' - unit: "KiB / sec" - label: pod_name + query_range: >- + sum( + rate( + container_network_receive_bytes_total{pod="{{pod}}"}[5m] + ) + ) by (pod) + unit: "bytes" + label: pod + - title: "Network Transmit (Out)" type: "line-chart" - y_label: "Transmitted (KiB/sec)" + y_label: "Transmitted (bytes/sec)" metrics: - id: pod_network_transmit - query_range: 'rate(container_network_transmit_bytes_total{pod_name="{{pod_name}}",container_name="POD"}[5m])/1024' - unit: "KiB / sec" - label: pod_name + query_range: >- + sum( + rate( + container_network_transmit_bytes_total{pod="{{pod}}"}[5m] + ) + ) by (pod) + unit: bytes + label: pod + - group: Disk metrics panels: - title: "Disk Reads" type: "line-chart" - y_label: "Disk reads (KiB/sec)" + y_label: "Disk reads (bytes/sec)" metrics: - id: pod_disk_reads - query_range: 'rate(container_fs_reads_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024' - unit: "KiB / sec" - label: pod_name + query_range: >- + sum( + rate( + container_fs_reads_bytes_total{pod="{{pod}}", container!="POD"}[5m] + ) + ) by (container,device) + + unit: "bytes / sec" + label: "{{container}} {{device}}" + - title: "Disk Writes" type: "line-chart" - y_label: "Disk writes (KiB/sec)" + y_label: "Disk writes (bytes/sec)" metrics: - id: pod_disk_writes - query_range: 'rate(container_fs_writes_bytes_total{container_name="POD",pod_name="{{pod_name}}"}[5m])/1024' - unit: "KiB / sec" - label: pod_name + query_range: >- + sum( + rate( + container_fs_writes_bytes_total{pod="{{pod}}", container!="POD"}[5m] + ) + ) by (container,device) + unit: "bytes / sec" + label: "{{container}} {{device}}" diff --git a/config/prometheus/self_monitoring_default.yml b/config/prometheus/self_monitoring_default.yml index 50e6f4585e4..024733bf2f0 100644 --- a/config/prometheus/self_monitoring_default.yml +++ b/config/prometheus/self_monitoring_default.yml @@ -1,4 +1,4 @@ -dashboard: 'Default dashboard' +dashboard: 'Overview' priority: 1 templating: |