summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/controllers/projects/environments/prometheus_api_controller.rb36
-rw-r--r--app/services/prometheus/proxy_variable_substitution_service.rb51
-rw-r--r--doc/api/README.md59
-rw-r--r--doc/api/projects.md6
-rw-r--r--doc/ci/enable_or_disable_ci.md16
-rw-r--r--doc/user/clusters/applications.md20
-rw-r--r--doc/user/project/pipelines/settings.md3
-rw-r--r--lib/api/api.rb1
-rw-r--r--lib/api/entities.rb12
-rw-r--r--lib/api/remote_mirrors.rb30
-rw-r--r--locale/gitlab.pot3
-rw-r--r--qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb6
-rw-r--r--spec/controllers/projects/environments/prometheus_api_controller_spec.rb4
-rw-r--r--spec/fixtures/api/schemas/remote_mirror.json26
-rw-r--r--spec/fixtures/api/schemas/remote_mirrors.json4
-rw-r--r--spec/lib/gitlab/prometheus/query_variables_spec.rb2
-rw-r--r--spec/requests/api/remote_mirrors_spec.rb41
-rw-r--r--spec/services/prometheus/proxy_variable_substitution_service_spec.rb143
18 files changed, 423 insertions, 40 deletions
diff --git a/app/controllers/projects/environments/prometheus_api_controller.rb b/app/controllers/projects/environments/prometheus_api_controller.rb
index e902d218c75..98fcc594d6e 100644
--- a/app/controllers/projects/environments/prometheus_api_controller.rb
+++ b/app/controllers/projects/environments/prometheus_api_controller.rb
@@ -7,23 +7,34 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon
before_action :environment
def proxy
- result = Prometheus::ProxyService.new(
+ variable_substitution_result =
+ variable_substitution_service.new(environment, permit_params).execute
+
+ if variable_substitution_result[:status] == :error
+ return error_response(variable_substitution_result)
+ end
+
+ prometheus_result = Prometheus::ProxyService.new(
environment,
proxy_method,
proxy_path,
- proxy_params
+ variable_substitution_result[:params]
).execute
- return continue_polling_response if result.nil?
- return error_response(result) if result[:status] == :error
+ return continue_polling_response if prometheus_result.nil?
+ return error_response(prometheus_result) if prometheus_result[:status] == :error
- success_response(result)
+ success_response(prometheus_result)
end
private
- def query_context
- Gitlab::Prometheus::QueryVariables.call(environment)
+ def variable_substitution_service
+ Prometheus::ProxyVariableSubstitutionService
+ end
+
+ def permit_params
+ params.permit!
end
def environment
@@ -37,15 +48,4 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon
def proxy_path
params[:proxy_path]
end
-
- def proxy_params
- substitute_query_variables(params).permit!
- end
-
- def substitute_query_variables(params)
- query = params[:query]
- return params unless query
-
- params.merge(query: query % query_context)
- end
end
diff --git a/app/services/prometheus/proxy_variable_substitution_service.rb b/app/services/prometheus/proxy_variable_substitution_service.rb
new file mode 100644
index 00000000000..d3d56987f07
--- /dev/null
+++ b/app/services/prometheus/proxy_variable_substitution_service.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Prometheus
+ class ProxyVariableSubstitutionService < BaseService
+ include Stepable
+
+ steps :add_params_to_result, :substitute_ruby_variables
+
+ def initialize(environment, params = {})
+ @environment, @params = environment, params.deep_dup
+ end
+
+ def execute
+ execute_steps
+ end
+
+ private
+
+ def add_params_to_result(result)
+ result[:params] = params
+
+ success(result)
+ end
+
+ def substitute_ruby_variables(result)
+ return success(result) unless query
+
+ # The % operator doesn't replace variables if the hash contains string
+ # keys.
+ result[:params][:query] = query % predefined_context.symbolize_keys
+
+ success(result)
+ rescue TypeError, ArgumentError => exception
+ log_error(exception.message)
+ Gitlab::Sentry.track_acceptable_exception(exception, extra: {
+ template_string: query,
+ variables: predefined_context
+ })
+
+ error(_('Malformed string'))
+ end
+
+ def predefined_context
+ @predefined_context ||= Gitlab::Prometheus::QueryVariables.call(@environment)
+ end
+
+ def query
+ params[:query]
+ end
+ end
+end
diff --git a/doc/api/README.md b/doc/api/README.md
index 6858e5b7d56..bcf238f9442 100644
--- a/doc/api/README.md
+++ b/doc/api/README.md
@@ -313,6 +313,17 @@ The following table shows the possible return codes for API requests.
## Pagination
+We support two kinds of pagination methods:
+
+- Offset-based pagination. This is the default method and available on all endpoints.
+- Keyset-based pagination. Added to selected endpoints but being
+ [progressively rolled out](https://gitlab.com/groups/gitlab-org/-/epics/2039).
+
+For large collections, we recommend keyset pagination (when available) over offset
+pagination for performance reasons.
+
+### Offset-based pagination
+
Sometimes the returned result will span across many pages. When listing
resources you can pass the following parameters:
@@ -324,10 +335,10 @@ resources you can pass the following parameters:
In the example below, we list 50 [namespaces](namespaces.md) per page.
```bash
-curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50"
```
-### Pagination Link header
+#### Pagination Link header
[Link headers](https://www.w3.org/wiki/LinkHeader) are sent back with each
response. They have `rel` set to prev/next/first/last and contain the relevant
@@ -362,7 +373,7 @@ X-Total: 8
X-Total-Pages: 3
```
-### Other pagination headers
+#### Other pagination headers
Additional pagination headers are also sent back.
@@ -383,6 +394,48 @@ and **behind the `api_kaminari_count_with_limit`
more than 10,000, the `X-Total` and `X-Total-Pages` headers as well as the
`rel="last"` `Link` are not present in the response headers.
+### Keyset-based pagination
+
+Keyset-pagination allows for more efficient retrieval of pages and - in contrast to offset-based pagination - runtime
+is independent of the size of the collection.
+
+This method is controlled by the following parameters:
+
+| Parameter | Description |
+| ------------ | -------------------------------------- |
+| `pagination` | `keyset` (to enable keyset pagination) |
+| `per_page` | Number of items to list per page (default: `20`, max: `100`) |
+
+In the example below, we list 50 [projects](projects.md) per page, ordered by `id` ascending.
+
+```bash
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc"
+```
+
+The response header includes a link to the next page. For example:
+
+```
+HTTP/1.1 200 OK
+...
+Link: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next"
+Status: 200 OK
+...
+```
+
+The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already.
+Note the type of filter depends on the `order_by` option used and we may have more than one additional filter.
+
+The `Link` header is absent when the end of the collection has been reached and there are no additional records to retrieve.
+
+We recommend using only the given link to retrieve the next page instead of building your own URL. Apart from the headers shown,
+we don't expose additional pagination headers.
+
+Keyset-based pagination is only supported for selected resources and ordering options:
+
+| Resource | Order |
+| ------------------------- | -------------------------- |
+| [Projects](projects.md) | `order_by=id` only |
+
## Namespaced path encoding
If using namespaced API calls, make sure that the `NAMESPACE/PROJECT_PATH` is
diff --git a/doc/api/projects.md b/doc/api/projects.md
index 222ab729810..ec3a081f5a3 100644
--- a/doc/api/projects.md
+++ b/doc/api/projects.md
@@ -61,6 +61,9 @@ GET /projects
| `id_after` | integer | no | Limit results to projects with IDs greater than the specified ID |
| `id_before` | integer | no | Limit results to projects with IDs less than the specified ID |
+NOTE: **Note:**
+This endpoint supports [keyset pagination](README.md#keyset-based-pagination) for selected `order_by` options.
+
When `simple=true` or the user is unauthenticated this returns something like:
```json
@@ -309,6 +312,9 @@ GET /users/:user_id/projects
| `id_after` | integer | no | Limit results to projects with IDs greater than the specified ID |
| `id_before` | integer | no | Limit results to projects with IDs less than the specified ID |
+NOTE: **Note:**
+This endpoint supports [keyset pagination](README.md#keyset-based-pagination) for selected `order_by` options.
+
```json
[
{
diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md
index dcf4d8dde2d..ff104d4e177 100644
--- a/doc/ci/enable_or_disable_ci.md
+++ b/doc/ci/enable_or_disable_ci.md
@@ -34,13 +34,17 @@ pipelines that are run from an [external integration](../user/project/integratio
## Per-project user setting
-The setting to enable or disable GitLab CI/CD Pipelines can be found in your project in
-**Settings > General > Visibility, project features, permissions**. If the project
-visibility is set to:
+To enable or disable GitLab CI/CD Pipelines in your project:
-- **Private**, only project members can access pipelines.
-- **Internal** or **Public**, pipelines can be made accessible to either
- project members only or everyone with access.
+1. Navigate to **Settings > General > Visibility, project features, permissions**.
+1. Expand the **Repository** section
+1. Enable or disable the **Pipelines** checkbox as required.
+
+**Project visibility** will also affect pipeline visibility. If set to:
+
+- **Private**: Only project members can access pipelines.
+- **Internal** or **Public**: Pipelines can be set to either **Only Project Members**
+ or **Everyone With Access** via the drop-down box.
Press **Save changes** for the settings to take effect.
diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md
index 6498545a4f6..46b7eabaeea 100644
--- a/doc/user/clusters/applications.md
+++ b/doc/user/clusters/applications.md
@@ -267,13 +267,19 @@ This feature:
kubectl -n gitlab-managed-apps exec -it $(kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' | awk '{print $1}') -- tail -f /var/log/modsec/audit.log
```
-There is a small performance overhead by enabling `modsecurity`. However, if this is
-considered significant for your application, you can toggle the feature flag back to
-false by running the following command within the Rails console:
-
-```ruby
-Feature.disable(:ingress_modsecurity)
-```
+There is a small performance overhead by enabling `modsecurity`. If this is
+considered significant for your application, you can either:
+
+- Disable ModSecurity's rule engine for your deployed application by setting
+ [the deployment variable](../../topics/autodevops/index.md)
+ `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off`. This will prevent ModSecurity from
+ processing any requests for the given application or environment.
+- Toggle the feature flag to false by running the following command within your
+ instance's Rails console:
+
+ ```ruby
+ Feature.disable(:ingress_modsecurity)
+ ```
Once disabled, you must [uninstall](#uninstalling-applications) and reinstall your Ingress
application for the changes to take effect.
diff --git a/doc/user/project/pipelines/settings.md b/doc/user/project/pipelines/settings.md
index a63da5c442f..671a55da5e3 100644
--- a/doc/user/project/pipelines/settings.md
+++ b/doc/user/project/pipelines/settings.md
@@ -149,6 +149,9 @@ Pipeline visibility is determined by:
- Your current [user access level](../../permissions.md).
- The **Public pipelines** project setting under your project's **Settings > CI/CD > General pipelines**.
+NOTE: **Note:**
+If the project visibility is set to **Private**, the [**Public pipelines** setting will have no effect](../../../ci/enable_or_disable_ci.md#per-project-user-setting).
+
This also determines the visibility of these related features:
- Job output logs
diff --git a/lib/api/api.rb b/lib/api/api.rb
index a2bdb76b834..6949cfa8e49 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -156,6 +156,7 @@ module API
mount ::API::ProtectedTags
mount ::API::Releases
mount ::API::Release::Links
+ mount ::API::RemoteMirrors
mount ::API::Repositories
mount ::API::Runner
mount ::API::Runners
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 1297f8e87a2..f89070ea1cb 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -166,6 +166,18 @@ module API
end
end
+ class RemoteMirror < Grape::Entity
+ expose :id
+ expose :enabled
+ expose :safe_url, as: :url
+ expose :update_status
+ expose :last_update_at
+ expose :last_update_started_at
+ expose :last_successful_update_at
+ expose :last_error
+ expose :only_protected_branches
+ end
+
class ProjectImportStatus < ProjectIdentity
expose :import_status
diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb
new file mode 100644
index 00000000000..8a085517ce9
--- /dev/null
+++ b/lib/api/remote_mirrors.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module API
+ class RemoteMirrors < Grape::API
+ include PaginationParams
+
+ before do
+ # TODO: Remove flag: https://gitlab.com/gitlab-org/gitlab/issues/38121
+ not_found! unless Feature.enabled?(:remote_mirrors_api, user_project)
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc "List the project's remote mirrors" do
+ success Entities::RemoteMirror
+ end
+ params do
+ use :pagination
+ end
+ get ':id/remote_mirrors' do
+ unauthorized! unless can?(current_user, :admin_remote_mirror, user_project)
+
+ present paginate(user_project.remote_mirrors),
+ with: Entities::RemoteMirror
+ end
+ end
+ end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 6436a828df0..09071940131 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -10614,6 +10614,9 @@ msgstr ""
msgid "Makes this issue confidential."
msgstr ""
+msgid "Malformed string"
+msgstr ""
+
msgid "Manage"
msgstr ""
diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
index f1ba726ddec..89f0fc37f3f 100644
--- a/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
+++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
@@ -26,9 +26,11 @@ module QA
merge_request.visit!
- expect(page).to have_text('to be squashed')
-
Page::MergeRequest::Show.perform do |merge_request_page|
+ merge_request_page.retry_on_exception(reload: true) do
+ expect(merge_request_page).to have_text('to be squashed')
+ end
+
merge_request_page.mark_to_squash
merge_request_page.merge!
diff --git a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
index b12964f8d8b..b8cf47cb1f2 100644
--- a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
+++ b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
@@ -63,9 +63,7 @@ describe Projects::Environments::PrometheusApiController do
context 'with nil query' do
let(:params_without_query) do
- params = environment_params
- params.delete(:query)
- params
+ environment_params.except(:query)
end
before do
diff --git a/spec/fixtures/api/schemas/remote_mirror.json b/spec/fixtures/api/schemas/remote_mirror.json
new file mode 100644
index 00000000000..416b0f080d9
--- /dev/null
+++ b/spec/fixtures/api/schemas/remote_mirror.json
@@ -0,0 +1,26 @@
+{
+ "type": "object",
+ "required": [
+ "id",
+ "enabled",
+ "url",
+ "update_status",
+ "last_update_at",
+ "last_update_started_at",
+ "last_successful_update_at",
+ "last_error",
+ "only_protected_branches"
+ ],
+ "properties": {
+ "id": { "type": "integer" },
+ "enabled": { "type": "boolean" },
+ "url": { "type": "string" },
+ "update_status": { "type": "string" },
+ "last_update_at": { "type": ["string", "null"] },
+ "last_update_started_at": { "type": ["string", "null"] },
+ "last_successful_update_at": { "type": ["string", "null"] },
+ "last_error": { "type": ["string", "null"] },
+ "only_protected_branches": { "type": "boolean" }
+ },
+ "additionalProperties": false
+}
diff --git a/spec/fixtures/api/schemas/remote_mirrors.json b/spec/fixtures/api/schemas/remote_mirrors.json
new file mode 100644
index 00000000000..3c4600c6caa
--- /dev/null
+++ b/spec/fixtures/api/schemas/remote_mirrors.json
@@ -0,0 +1,4 @@
+{
+ "type": "array",
+ "items": { "$ref": "remote_mirror.json" }
+}
diff --git a/spec/lib/gitlab/prometheus/query_variables_spec.rb b/spec/lib/gitlab/prometheus/query_variables_spec.rb
index 3f9b245a3fb..849265de513 100644
--- a/spec/lib/gitlab/prometheus/query_variables_spec.rb
+++ b/spec/lib/gitlab/prometheus/query_variables_spec.rb
@@ -14,7 +14,7 @@ describe Gitlab::Prometheus::QueryVariables do
it do
is_expected.to include(environment_filter:
- %{container_name!="POD",environment="#{slug}"})
+ %Q[container_name!="POD",environment="#{slug}"])
end
context 'without deployment platform' do
diff --git a/spec/requests/api/remote_mirrors_spec.rb b/spec/requests/api/remote_mirrors_spec.rb
new file mode 100644
index 00000000000..c5ba9bd223e
--- /dev/null
+++ b/spec/requests/api/remote_mirrors_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe API::RemoteMirrors do
+ let_it_be(:user) { create(:user) }
+ let_it_be(:project) { create(:project, :repository, :remote_mirror) }
+
+ describe 'GET /projects/:id/remote_mirrors' do
+ let(:route) { "/projects/#{project.id}/remote_mirrors" }
+
+ it 'requires `admin_remote_mirror` permission' do
+ project.add_developer(user)
+
+ get api(route, user)
+
+ expect(response).to have_gitlab_http_status(:unauthorized)
+ end
+
+ it 'returns a list of remote mirrors' do
+ project.add_maintainer(user)
+
+ get api(route, user)
+
+ expect(response).to have_gitlab_http_status(:success)
+ expect(response).to match_response_schema('remote_mirrors')
+ end
+
+ context 'with the `remote_mirrors_api` feature disabled' do
+ before do
+ stub_feature_flags(remote_mirrors_api: false)
+ end
+
+ it 'responds with `not_found`' do
+ get api(route, user)
+
+ expect(response).to have_gitlab_http_status(:not_found)
+ end
+ end
+ end
+end
diff --git a/spec/services/prometheus/proxy_variable_substitution_service_spec.rb b/spec/services/prometheus/proxy_variable_substitution_service_spec.rb
new file mode 100644
index 00000000000..b1cdb8fd3ae
--- /dev/null
+++ b/spec/services/prometheus/proxy_variable_substitution_service_spec.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Prometheus::ProxyVariableSubstitutionService do
+ describe '#execute' do
+ let_it_be(:environment) { create(:environment) }
+
+ let(:params_keys) { { query: 'up{environment="%{ci_environment_slug}"}' } }
+ let(:params) { ActionController::Parameters.new(params_keys).permit! }
+ let(:result) { subject.execute }
+
+ subject { described_class.new(environment, params) }
+
+ shared_examples 'success' do
+ it 'replaces variables with values' do
+ expect(result[:status]).to eq(:success)
+ expect(result[:params][:query]).to eq(expected_query)
+ end
+ end
+
+ shared_examples 'error' do |message|
+ it 'returns error' do
+ expect(result[:status]).to eq(:error)
+ expect(result[:message]).to eq(message)
+ end
+ end
+
+ context 'does not alter params passed to the service' do
+ it do
+ subject.execute
+
+ expect(params).to eq(
+ ActionController::Parameters.new(
+ query: 'up{environment="%{ci_environment_slug}"}'
+ ).permit!
+ )
+ end
+ end
+
+ context 'with predefined variables' do
+ it_behaves_like 'success' do
+ let(:expected_query) { %Q[up{environment="#{environment.slug}"}] }
+ end
+
+ context 'with nil query' do
+ let(:params_keys) { {} }
+
+ it_behaves_like 'success' do
+ let(:expected_query) { nil }
+ end
+ end
+ end
+
+ context 'ruby template rendering' do
+ let(:params_keys) do
+ { query: 'up{env=%{ci_environment_slug},%{environment_filter}}' }
+ end
+
+ it_behaves_like 'success' do
+ let(:expected_query) do
+ "up{env=#{environment.slug},container_name!=\"POD\"," \
+ "environment=\"#{environment.slug}\"}"
+ end
+ end
+
+ context 'with multiple occurrences of variable in string' do
+ let(:params_keys) do
+ { query: 'up{env1=%{ci_environment_slug},env2=%{ci_environment_slug}}' }
+ end
+
+ it_behaves_like 'success' do
+ let(:expected_query) { "up{env1=#{environment.slug},env2=#{environment.slug}}" }
+ end
+ end
+
+ context 'with multiple variables in string' do
+ let(:params_keys) do
+ { query: 'up{env=%{ci_environment_slug},%{environment_filter}}' }
+ end
+
+ it_behaves_like 'success' do
+ let(:expected_query) do
+ "up{env=#{environment.slug}," \
+ "container_name!=\"POD\",environment=\"#{environment.slug}\"}"
+ end
+ end
+ end
+
+ context 'with unknown variables in string' do
+ let(:params_keys) { { query: 'up{env=%{env_slug}}' } }
+
+ it_behaves_like 'success' do
+ let(:expected_query) { 'up{env=%{env_slug}}' }
+ end
+ end
+
+ # This spec is needed if there are multiple keys in the context provided
+ # by `Gitlab::Prometheus::QueryVariables.call(environment)` which is
+ # passed to the Ruby `%` operator.
+ # If the number of keys in the context is one, there is no need for
+ # this spec.
+ context 'with extra variables in context' do
+ let(:params_keys) { { query: 'up{env=%{ci_environment_slug}}' } }
+
+ it_behaves_like 'success' do
+ let(:expected_query) { "up{env=#{environment.slug}}" }
+ end
+
+ it 'has more than one variable in context' do
+ expect(Gitlab::Prometheus::QueryVariables.call(environment).size).to be > 1
+ end
+ end
+
+ # The ruby % operator will not replace known variables if there are unknown
+ # variables also in the string. It doesn't raise an error
+ # (though the `sprintf` and `format` methods do).
+ context 'with unknown and known variables in string' do
+ let(:params_keys) do
+ { query: 'up{env=%{ci_environment_slug},other_env=%{env_slug}}' }
+ end
+
+ it_behaves_like 'success' do
+ let(:expected_query) { 'up{env=%{ci_environment_slug},other_env=%{env_slug}}' }
+ end
+ end
+
+ context 'when rendering raises error' do
+ context 'when TypeError is raised' do
+ let(:params_keys) { { query: '{% a %}' } }
+
+ it_behaves_like 'error', 'Malformed string'
+ end
+
+ context 'when ArgumentError is raised' do
+ let(:params_keys) { { query: '%<' } }
+
+ it_behaves_like 'error', 'Malformed string'
+ end
+ end
+ end
+ end
+end