Add latest changes from gitlab-org/gitlab@master

18 files changed, 423 insertions, 40 deletions

diff --git a/app/controllers/projects/environments/prometheus_api_controller.rb b/app/controllers/projects/environments/prometheus_api_controller.rb
index e902d218c75..98fcc594d6e 100644
--- a/app/controllers/projects/environments/prometheus_api_controller.rb
+++ b/app/controllers/projects/environments/prometheus_api_controller.rb
@@ -7,23 +7,34 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon
   before_action :environment
 
   def proxy
-    result = Prometheus::ProxyService.new(
+    variable_substitution_result =
+      variable_substitution_service.new(environment, permit_params).execute
+
+    if variable_substitution_result[:status] == :error
+      return error_response(variable_substitution_result)
+    end
+
+    prometheus_result = Prometheus::ProxyService.new(
       environment,
       proxy_method,
       proxy_path,
-      proxy_params
+      variable_substitution_result[:params]
     ).execute
 
-    return continue_polling_response if result.nil?
-    return error_response(result) if result[:status] == :error
+    return continue_polling_response if prometheus_result.nil?
+    return error_response(prometheus_result) if prometheus_result[:status] == :error
 
-    success_response(result)
+    success_response(prometheus_result)
   end
 
   private
 
-  def query_context
-    Gitlab::Prometheus::QueryVariables.call(environment)
+  def variable_substitution_service
+    Prometheus::ProxyVariableSubstitutionService
+  end
+
+  def permit_params
+    params.permit!
   end
 
   def environment
@@ -37,15 +48,4 @@ class Projects::Environments::PrometheusApiController < Projects::ApplicationCon
   def proxy_path
     params[:proxy_path]
   end
-
-  def proxy_params
-    substitute_query_variables(params).permit!
-  end
-
-  def substitute_query_variables(params)
-    query = params[:query]
-    return params unless query
-
-    params.merge(query: query % query_context)
-  end
 end
diff --git a/app/services/prometheus/proxy_variable_substitution_service.rb b/app/services/prometheus/proxy_variable_substitution_service.rb
new file mode 100644
index 00000000000..d3d56987f07
--- /dev/null
+++ b/app/services/prometheus/proxy_variable_substitution_service.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Prometheus
+  class ProxyVariableSubstitutionService < BaseService
+    include Stepable
+
+    steps :add_params_to_result, :substitute_ruby_variables
+
+    def initialize(environment, params = {})
+      @environment, @params = environment, params.deep_dup
+    end
+
+    def execute
+      execute_steps
+    end
+
+    private
+
+    def add_params_to_result(result)
+      result[:params] = params
+
+      success(result)
+    end
+
+    def substitute_ruby_variables(result)
+      return success(result) unless query
+
+      # The % operator doesn't replace variables if the hash contains string
+      # keys.
+      result[:params][:query] = query % predefined_context.symbolize_keys
+
+      success(result)
+    rescue TypeError, ArgumentError => exception
+      log_error(exception.message)
+      Gitlab::Sentry.track_acceptable_exception(exception, extra: {
+        template_string: query,
+        variables: predefined_context
+      })
+
+      error(_('Malformed string'))
+    end
+
+    def predefined_context
+      @predefined_context ||= Gitlab::Prometheus::QueryVariables.call(@environment)
+    end
+
+    def query
+      params[:query]
+    end
+  end
+end
diff --git a/doc/api/README.md b/doc/api/README.md
index 6858e5b7d56..bcf238f9442 100644
--- a/doc/api/README.md
+++ b/doc/api/README.md
@@ -313,6 +313,17 @@ The following table shows the possible return codes for API requests.
 
 ## Pagination
 
+We support two kinds of pagination methods:
+
+- Offset-based pagination. This is the default method and available on all endpoints.
+- Keyset-based pagination. Added to selected endpoints but being
+  [progressively rolled out](https://gitlab.com/groups/gitlab-org/-/epics/2039).
+
+For large collections, we recommend keyset pagination (when available) over offset
+pagination for performance reasons.
+
+### Offset-based pagination
+
 Sometimes the returned result will span across many pages. When listing
 resources you can pass the following parameters:
 
@@ -324,10 +335,10 @@ resources you can pass the following parameters:
 In the example below, we list 50 [namespaces](namespaces.md) per page.
 
 ```bash
-curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50"
 ```
 
-### Pagination Link header
+#### Pagination Link header
 
 [Link headers](https://www.w3.org/wiki/LinkHeader) are sent back with each
 response. They have `rel` set to prev/next/first/last and contain the relevant
@@ -362,7 +373,7 @@ X-Total: 8
 X-Total-Pages: 3
 ```
 
-### Other pagination headers
+#### Other pagination headers
 
 Additional pagination headers are also sent back.
 
@@ -383,6 +394,48 @@ and **behind the `api_kaminari_count_with_limit`
 more than 10,000, the `X-Total` and `X-Total-Pages` headers as well as the
 `rel="last"` `Link` are not present in the response headers.
 
+### Keyset-based pagination
+
+Keyset-pagination allows for more efficient retrieval of pages and - in contrast to offset-based pagination - runtime
+is independent of the size of the collection.
+
+This method is controlled by the following parameters:
+
+| Parameter    | Description                            |
+| ------------ | -------------------------------------- |
+| `pagination` | `keyset` (to enable keyset pagination) |
+| `per_page`   | Number of items to list per page (default: `20`, max: `100`) |
+
+In the example below, we list 50 [projects](projects.md) per page, ordered by `id` ascending.
+
+```bash
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc"
+```
+
+The response header includes a link to the next page. For example:
+
+```
+HTTP/1.1 200 OK
+...
+Link: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next"
+Status: 200 OK
+...
+```
+
+The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already.
+Note the type of filter depends on the `order_by` option used and we may have more than one additional filter.
+
+The `Link` header is absent when the end of the collection has been reached and there are no additional records to retrieve.
+
+We recommend using only the given link to retrieve the next page instead of building your own URL. Apart from the headers shown,
+we don't expose additional pagination headers.
+
+Keyset-based pagination is only supported for selected resources and ordering options:
+
+| Resource                  | Order                      |
+| ------------------------- | -------------------------- |
+| [Projects](projects.md)   | `order_by=id` only         |
+
 ## Namespaced path encoding
 
 If using namespaced API calls, make sure that the `NAMESPACE/PROJECT_PATH` is
diff --git a/doc/api/projects.md b/doc/api/projects.md
index 222ab729810..ec3a081f5a3 100644
--- a/doc/api/projects.md
+++ b/doc/api/projects.md
@@ -61,6 +61,9 @@ GET /projects
 | `id_after`                    | integer | no | Limit results to projects with IDs greater than the specified ID |
 | `id_before`                   | integer | no | Limit results to projects with IDs less than the specified ID |
 
+NOTE: **Note:**
+This endpoint supports [keyset pagination](README.md#keyset-based-pagination) for selected `order_by` options.
+
 When `simple=true` or the user is unauthenticated this returns something like:
 
 ```json
@@ -309,6 +312,9 @@ GET /users/:user_id/projects
 | `id_after`                    | integer | no | Limit results to projects with IDs greater than the specified ID |
 | `id_before`                   | integer | no | Limit results to projects with IDs less than the specified ID |
 
+NOTE: **Note:**
+This endpoint supports [keyset pagination](README.md#keyset-based-pagination) for selected `order_by` options.
+
 ```json
 [
   {
diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md
index dcf4d8dde2d..ff104d4e177 100644
--- a/doc/ci/enable_or_disable_ci.md
+++ b/doc/ci/enable_or_disable_ci.md
@@ -34,13 +34,17 @@ pipelines that are run from an [external integration](../user/project/integratio
 
 ## Per-project user setting
 
-The setting to enable or disable GitLab CI/CD Pipelines can be found in your project in
-**Settings > General > Visibility, project features, permissions**. If the project
-visibility is set to:
+To enable or disable GitLab CI/CD Pipelines in your project:
 
-- **Private**, only project members can access pipelines.
-- **Internal** or **Public**, pipelines can be made accessible to either
-  project members only or everyone with access.
+1. Navigate to **Settings > General > Visibility, project features, permissions**.
+1. Expand the **Repository** section
+1. Enable or disable the **Pipelines** checkbox as required.
+
+**Project visibility** will also affect pipeline visibility. If set to:
+
+- **Private**: Only project members can access pipelines.
+- **Internal** or **Public**: Pipelines can be set to either **Only Project Members**
+  or **Everyone With Access** via the drop-down box.
 
 Press **Save changes** for the settings to take effect.
 
diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md
index 6498545a4f6..46b7eabaeea 100644
--- a/doc/user/clusters/applications.md
+++ b/doc/user/clusters/applications.md
@@ -267,13 +267,19 @@ This feature:
   kubectl -n gitlab-managed-apps exec -it $(kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' | awk '{print $1}') -- tail -f /var/log/modsec/audit.log
   ```
 
-There is a small performance overhead by enabling `modsecurity`. However, if this is
-considered significant for your application, you can toggle the feature flag back to
-false by running the following command within the Rails console:
-
-```ruby
-Feature.disable(:ingress_modsecurity)
-```
+There is a small performance overhead by enabling `modsecurity`. If this is
+considered significant for your application, you can either:
+
+- Disable ModSecurity's rule engine for your deployed application by setting
+  [the deployment variable](../../topics/autodevops/index.md)
+  `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` to `Off`. This will prevent ModSecurity from
+  processing any requests for the given application or environment.
+- Toggle the feature flag to false by running the following command within your
+  instance's Rails console:
+
+  ```ruby
+  Feature.disable(:ingress_modsecurity)
+  ```
 
 Once disabled, you must [uninstall](#uninstalling-applications) and reinstall your Ingress
 application for the changes to take effect.
diff --git a/doc/user/project/pipelines/settings.md b/doc/user/project/pipelines/settings.md
index a63da5c442f..671a55da5e3 100644
--- a/doc/user/project/pipelines/settings.md
+++ b/doc/user/project/pipelines/settings.md
@@ -149,6 +149,9 @@ Pipeline visibility is determined by:
 - Your current [user access level](../../permissions.md).
 - The **Public pipelines** project setting under your project's **Settings > CI/CD > General pipelines**.
 
+NOTE: **Note:**
+If the project visibility is set to **Private**, the [**Public pipelines** setting will have no effect](../../../ci/enable_or_disable_ci.md#per-project-user-setting).
+
 This also determines the visibility of these related features:
 
 - Job output logs
diff --git a/lib/api/api.rb b/lib/api/api.rb
index a2bdb76b834..6949cfa8e49 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -156,6 +156,7 @@ module API
     mount ::API::ProtectedTags
     mount ::API::Releases
     mount ::API::Release::Links
+    mount ::API::RemoteMirrors
     mount ::API::Repositories
     mount ::API::Runner
     mount ::API::Runners
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 1297f8e87a2..f89070ea1cb 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -166,6 +166,18 @@ module API
       end
     end
 
+    class RemoteMirror < Grape::Entity
+      expose :id
+      expose :enabled
+      expose :safe_url, as: :url
+      expose :update_status
+      expose :last_update_at
+      expose :last_update_started_at
+      expose :last_successful_update_at
+      expose :last_error
+      expose :only_protected_branches
+    end
+
     class ProjectImportStatus < ProjectIdentity
       expose :import_status
 
diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb
new file mode 100644
index 00000000000..8a085517ce9
--- /dev/null
+++ b/lib/api/remote_mirrors.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module API
+  class RemoteMirrors < Grape::API
+    include PaginationParams
+
+    before do
+      # TODO: Remove flag: https://gitlab.com/gitlab-org/gitlab/issues/38121
+      not_found! unless Feature.enabled?(:remote_mirrors_api, user_project)
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc "List the project's remote mirrors" do
+        success Entities::RemoteMirror
+      end
+      params do
+        use :pagination
+      end
+      get ':id/remote_mirrors' do
+        unauthorized! unless can?(current_user, :admin_remote_mirror, user_project)
+
+        present paginate(user_project.remote_mirrors),
+          with: Entities::RemoteMirror
+      end
+    end
+  end
+end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 6436a828df0..09071940131 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -10614,6 +10614,9 @@ msgstr ""
 msgid "Makes this issue confidential."
 msgstr ""
 
+msgid "Malformed string"
+msgstr ""
+
 msgid "Manage"
 msgstr ""
 
diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
index f1ba726ddec..89f0fc37f3f 100644
--- a/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
+++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb
@@ -26,9 +26,11 @@ module QA
 
         merge_request.visit!
 
-        expect(page).to have_text('to be squashed')
-
         Page::MergeRequest::Show.perform do |merge_request_page|
+          merge_request_page.retry_on_exception(reload: true) do
+            expect(merge_request_page).to have_text('to be squashed')
+          end
+
           merge_request_page.mark_to_squash
           merge_request_page.merge!
 
diff --git a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
index b12964f8d8b..b8cf47cb1f2 100644
--- a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
+++ b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb
@@ -63,9 +63,7 @@ describe Projects::Environments::PrometheusApiController do
 
           context 'with nil query' do
             let(:params_without_query) do
-              params = environment_params
-              params.delete(:query)
-              params
+              environment_params.except(:query)
             end
 
             before do
diff --git a/spec/fixtures/api/schemas/remote_mirror.json b/spec/fixtures/api/schemas/remote_mirror.json
new file mode 100644
index 00000000000..416b0f080d9
--- /dev/null
+++ b/spec/fixtures/api/schemas/remote_mirror.json
@@ -0,0 +1,26 @@
+{
+  "type": "object",
+  "required": [
+    "id",
+    "enabled",
+    "url",
+    "update_status",
+    "last_update_at",
+    "last_update_started_at",
+    "last_successful_update_at",
+    "last_error",
+    "only_protected_branches"
+    ],
+  "properties": {
+    "id": { "type": "integer" },
+    "enabled": { "type": "boolean" },
+    "url": { "type": "string" },
+    "update_status": { "type": "string" },
+    "last_update_at": { "type": ["string", "null"] },
+    "last_update_started_at": { "type": ["string", "null"] },
+    "last_successful_update_at": { "type": ["string", "null"] },
+    "last_error": { "type": ["string", "null"] },
+    "only_protected_branches": { "type": "boolean" }
+  },
+  "additionalProperties": false
+}
diff --git a/spec/fixtures/api/schemas/remote_mirrors.json b/spec/fixtures/api/schemas/remote_mirrors.json
new file mode 100644
index 00000000000..3c4600c6caa
--- /dev/null
+++ b/spec/fixtures/api/schemas/remote_mirrors.json
@@ -0,0 +1,4 @@
+{
+  "type": "array",
+  "items": { "$ref": "remote_mirror.json" }
+}
diff --git a/spec/lib/gitlab/prometheus/query_variables_spec.rb b/spec/lib/gitlab/prometheus/query_variables_spec.rb
index 3f9b245a3fb..849265de513 100644
--- a/spec/lib/gitlab/prometheus/query_variables_spec.rb
+++ b/spec/lib/gitlab/prometheus/query_variables_spec.rb
@@ -14,7 +14,7 @@ describe Gitlab::Prometheus::QueryVariables do
 
     it do
       is_expected.to include(environment_filter:
-                             %{container_name!="POD",environment="#{slug}"})
+                             %Q[container_name!="POD",environment="#{slug}"])
     end
 
     context 'without deployment platform' do
diff --git a/spec/requests/api/remote_mirrors_spec.rb b/spec/requests/api/remote_mirrors_spec.rb
new file mode 100644
index 00000000000..c5ba9bd223e
--- /dev/null
+++ b/spec/requests/api/remote_mirrors_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe API::RemoteMirrors do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository, :remote_mirror) }
+
+  describe 'GET /projects/:id/remote_mirrors' do
+    let(:route) { "/projects/#{project.id}/remote_mirrors" }
+
+    it 'requires `admin_remote_mirror` permission' do
+      project.add_developer(user)
+
+      get api(route, user)
+
+      expect(response).to have_gitlab_http_status(:unauthorized)
+    end
+
+    it 'returns a list of remote mirrors' do
+      project.add_maintainer(user)
+
+      get api(route, user)
+
+      expect(response).to have_gitlab_http_status(:success)
+      expect(response).to match_response_schema('remote_mirrors')
+    end
+
+    context 'with the `remote_mirrors_api` feature disabled' do
+      before do
+        stub_feature_flags(remote_mirrors_api: false)
+      end
+
+      it 'responds with `not_found`' do
+        get api(route, user)
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
+end
diff --git a/spec/services/prometheus/proxy_variable_substitution_service_spec.rb b/spec/services/prometheus/proxy_variable_substitution_service_spec.rb
new file mode 100644
index 00000000000..b1cdb8fd3ae
--- /dev/null
+++ b/spec/services/prometheus/proxy_variable_substitution_service_spec.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Prometheus::ProxyVariableSubstitutionService do
+  describe '#execute' do
+    let_it_be(:environment) { create(:environment) }
+
+    let(:params_keys) { { query: 'up{environment="%{ci_environment_slug}"}' } }
+    let(:params) { ActionController::Parameters.new(params_keys).permit! }
+    let(:result) { subject.execute }
+
+    subject { described_class.new(environment, params) }
+
+    shared_examples 'success' do
+      it 'replaces variables with values' do
+        expect(result[:status]).to eq(:success)
+        expect(result[:params][:query]).to eq(expected_query)
+      end
+    end
+
+    shared_examples 'error' do |message|
+      it 'returns error' do
+        expect(result[:status]).to eq(:error)
+        expect(result[:message]).to eq(message)
+      end
+    end
+
+    context 'does not alter params passed to the service' do
+      it do
+        subject.execute
+
+        expect(params).to eq(
+          ActionController::Parameters.new(
+            query: 'up{environment="%{ci_environment_slug}"}'
+          ).permit!
+        )
+      end
+    end
+
+    context 'with predefined variables' do
+      it_behaves_like 'success' do
+        let(:expected_query) { %Q[up{environment="#{environment.slug}"}] }
+      end
+
+      context 'with nil query' do
+        let(:params_keys) { {} }
+
+        it_behaves_like 'success' do
+          let(:expected_query) { nil }
+        end
+      end
+    end
+
+    context 'ruby template rendering' do
+      let(:params_keys) do
+        { query: 'up{env=%{ci_environment_slug},%{environment_filter}}' }
+      end
+
+      it_behaves_like 'success' do
+        let(:expected_query) do
+          "up{env=#{environment.slug},container_name!=\"POD\"," \
+          "environment=\"#{environment.slug}\"}"
+        end
+      end
+
+      context 'with multiple occurrences of variable in string' do
+        let(:params_keys) do
+          { query: 'up{env1=%{ci_environment_slug},env2=%{ci_environment_slug}}' }
+        end
+
+        it_behaves_like 'success' do
+          let(:expected_query) { "up{env1=#{environment.slug},env2=#{environment.slug}}" }
+        end
+      end
+
+      context 'with multiple variables in string' do
+        let(:params_keys) do
+          { query: 'up{env=%{ci_environment_slug},%{environment_filter}}' }
+        end
+
+        it_behaves_like 'success' do
+          let(:expected_query) do
+            "up{env=#{environment.slug}," \
+            "container_name!=\"POD\",environment=\"#{environment.slug}\"}"
+          end
+        end
+      end
+
+      context 'with unknown variables in string' do
+        let(:params_keys) { { query: 'up{env=%{env_slug}}' } }
+
+        it_behaves_like 'success' do
+          let(:expected_query) { 'up{env=%{env_slug}}' }
+        end
+      end
+
+      # This spec is needed if there are multiple keys in the context provided
+      # by `Gitlab::Prometheus::QueryVariables.call(environment)` which is
+      # passed to the Ruby `%` operator.
+      # If the number of keys in the context is one, there is no need for
+      # this spec.
+      context 'with extra variables in context' do
+        let(:params_keys) { { query: 'up{env=%{ci_environment_slug}}' } }
+
+        it_behaves_like 'success' do
+          let(:expected_query) { "up{env=#{environment.slug}}" }
+        end
+
+        it 'has more than one variable in context' do
+          expect(Gitlab::Prometheus::QueryVariables.call(environment).size).to be > 1
+        end
+      end
+
+      # The ruby % operator will not replace known variables if there are unknown
+      # variables also in the string. It doesn't raise an error
+      # (though the `sprintf` and `format` methods do).
+      context 'with unknown and known variables in string' do
+        let(:params_keys) do
+          { query: 'up{env=%{ci_environment_slug},other_env=%{env_slug}}' }
+        end
+
+        it_behaves_like 'success' do
+          let(:expected_query) { 'up{env=%{ci_environment_slug},other_env=%{env_slug}}' }
+        end
+      end
+
+      context 'when rendering raises error' do
+        context 'when TypeError is raised' do
+          let(:params_keys) { { query: '{% a %}' } }
+
+          it_behaves_like 'error', 'Malformed string'
+        end
+
+        context 'when ArgumentError is raised' do
+          let(:params_keys) { { query: '%<' } }
+
+          it_behaves_like 'error', 'Malformed string'
+        end
+      end
+    end
+  end
+end