diff options
author | Achilleas Pipinellis <axilleas@axilleas.me> | 2016-06-17 17:29:40 +0000 |
---|---|---|
committer | Achilleas Pipinellis <axilleas@axilleas.me> | 2016-06-17 17:29:40 +0000 |
commit | 4d9161603a17bcdb2a97a6e1982e4f97c057a769 (patch) | |
tree | a9f45fb5bab435314c48f55df5352682eff0e0ac | |
parent | 1ca1ebc09bda279dcbacfcfaf39e0410f94ca985 (diff) | |
parent | 23457cba44758090136b89e13fb83dbc52d69cda (diff) | |
download | gitlab-ce-4d9161603a17bcdb2a97a6e1982e4f97c057a769.tar.gz |
Merge branch 'sidekiq-api-metrics' into 'master'
Added API endpoint for Sidekiq Metrics.
## What does this MR do?
It adds an API endpoint to gather metrics about Sidekiq, it's jobs, queues, and processes.
## Why was this MR needed?
There was no API endpoint for Sidekiq information.
## What are the relevant issue numbers?
Fixes #7171
## Does this MR meet the acceptance criteria?
- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [x] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [x] API support added
- [x] Tests
- [x] Added for this feature/bug
- [x] All builds are passing
See merge request !4653
-rw-r--r-- | CHANGELOG | 1 | ||||
-rw-r--r-- | doc/api/sidekiq_metrics.md | 152 | ||||
-rw-r--r-- | lib/api/api.rb | 1 | ||||
-rw-r--r-- | lib/api/sidekiq_metrics.rb | 90 | ||||
-rw-r--r-- | spec/requests/api/sidekiq_metrics_spec.rb | 40 |
5 files changed, 284 insertions, 0 deletions
diff --git a/CHANGELOG b/CHANGELOG index 620a7eea378..c0bfe8db076 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -46,6 +46,7 @@ v 8.9.0 (unreleased) - Fixed alignment of download dropdown in merge requests - Upgrade to jQuery 2 - Adds selected branch name to the dropdown toggle + - Add API endpoint for Sidekiq Metrics !4653 - Use Knapsack to evenly distribute tests across multiple nodes - Add `sha` parameter to MR merge API, to ensure only reviewed changes are merged - Don't allow MRs to be merged when commits were added since the last review / page load diff --git a/doc/api/sidekiq_metrics.md b/doc/api/sidekiq_metrics.md new file mode 100644 index 00000000000..ebd131c94ca --- /dev/null +++ b/doc/api/sidekiq_metrics.md @@ -0,0 +1,152 @@ +# Sidekiq Metrics + +>**Note:** This endpoint is only available on GitLab 8.9 and above. + +This API endpoint allows you to retrieve some information about the current state +of Sidekiq, its jobs, queues, and processes. + +## Get the current Queue Metrics + +List information about all the registered queues, their backlog and their +latency. + +``` +GET /sidekiq/queue_metrics +``` + +```bash +curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/sidekiq/queue_metrics +``` + +Example response: + +```json +{ + "queues": { + "default": { + "backlog": 0, + "latency": 0 + } + } +} +``` + +## Get the current Process Metrics + +List information about all the Sidekiq workers registered to process your queues. + +``` +GET /sidekiq/process_metrics +``` + +```bash +curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/sidekiq/process_metrics +``` + +Example response: + +```json +{ + "processes": [ + { + "hostname": "gitlab.example.com", + "pid": 5649, + "tag": "gitlab", + "started_at": "2016-06-14T10:45:07.159-05:00", + "queues": [ + "post_receive", + "mailers", + "archive_repo", + "system_hook", + "project_web_hook", + "gitlab_shell", + "incoming_email", + "runner", + "common", + "default" + ], + "labels": [], + "concurrency": 25, + "busy": 0 + } + ] +} +``` + +## Get the current Job Statistics + +List information about the jobs that Sidekiq has performed. + +``` +GET /sidekiq/job_stats +``` + +```bash +curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/sidekiq/job_stats +``` + +Example response: + +```json +{ + "jobs": { + "processed": 2, + "failed": 0, + "enqueued": 0 + } +} +``` + +## Get a compound response of all the previously mentioned metrics + +List all the currently available information about Sidekiq. + +``` +GET /sidekiq/compound_metrics +``` + +```bash +curl -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/sidekiq/compound_metrics +``` + +Example response: + +```json +{ + "queues": { + "default": { + "backlog": 0, + "latency": 0 + } + }, + "processes": [ + { + "hostname": "gitlab.example.com", + "pid": 5649, + "tag": "gitlab", + "started_at": "2016-06-14T10:45:07.159-05:00", + "queues": [ + "post_receive", + "mailers", + "archive_repo", + "system_hook", + "project_web_hook", + "gitlab_shell", + "incoming_email", + "runner", + "common", + "default" + ], + "labels": [], + "concurrency": 25, + "busy": 0 + } + ], + "jobs": { + "processed": 2, + "failed": 0, + "enqueued": 0 + } +} +``` + diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..51ddd0dbfc4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -59,5 +59,6 @@ module API mount ::API::Licenses mount ::API::Subscriptions mount ::API::Gitignores + mount ::API::SidekiqMetrics end end diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end diff --git a/spec/requests/api/sidekiq_metrics_spec.rb b/spec/requests/api/sidekiq_metrics_spec.rb new file mode 100644 index 00000000000..41cbf0c6669 --- /dev/null +++ b/spec/requests/api/sidekiq_metrics_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe API::SidekiqMetrics, api: true do + include ApiHelpers + + let(:admin) { create(:user, :admin) } + + describe 'GET sidekiq/*' do + it 'defines the `queue_metrics` endpoint' do + get api('/sidekiq/queue_metrics', admin) + + expect(response.status).to eq(200) + expect(json_response).to be_a Hash + end + + it 'defines the `process_metrics` endpoint' do + get api('/sidekiq/process_metrics', admin) + + expect(response.status).to eq(200) + expect(json_response['processes']).to be_an Array + end + + it 'defines the `job_stats` endpoint' do + get api('/sidekiq/job_stats', admin) + + expect(response.status).to eq(200) + expect(json_response).to be_a Hash + end + + it 'defines the `compound_metrics` endpoint' do + get api('/sidekiq/compound_metrics', admin) + + expect(response.status).to eq(200) + expect(json_response).to be_a Hash + expect(json_response['queues']).to be_a Hash + expect(json_response['processes']).to be_an Array + expect(json_response['jobs']).to be_a Hash + end + end +end |