Add latest changes from gitlab-org/gitlab@13-8-stable-ee

15 files changed, 244 insertions, 23 deletions

diff --git a/app/controllers/repositories/lfs_storage_controller.rb b/app/controllers/repositories/lfs_storage_controller.rb
index 48784842d48..36912948b77 100644
--- a/app/controllers/repositories/lfs_storage_controller.rb
+++ b/app/controllers/repositories/lfs_storage_controller.rb
@@ -21,7 +21,17 @@ module Repositories
     def upload_authorize
       set_workhorse_internal_api_content_type
 
-      authorized = LfsObjectUploader.workhorse_authorize(has_length: true)
+      # We don't actually know whether Workhorse received an LFS upload
+      # request with a Content-Length header or `Transfer-Encoding:
+      # chunked`.  Since we don't know, we need to be pessimistic and
+      # set `has_length` to `false` so that multipart uploads will be
+      # used for AWS. Otherwise, AWS will respond with `501 NOT IMPLEMENTED`
+      # error because a PutObject request with `Transfer-Encoding: chunked`
+      # is not supported.
+      #
+      # This is only an issue with object storage-specific settings, not
+      # with consolidated object storage settings.
+      authorized = LfsObjectUploader.workhorse_authorize(has_length: false, maximum_size: size)
       authorized.merge!(LfsOid: oid, LfsSize: size)
 
       render json: authorized
diff --git a/changelogs/unreleased/dblessing_fix_ldap_server_config.yml b/changelogs/unreleased/dblessing_fix_ldap_server_config.yml
new file mode 100644
index 00000000000..619d83c98f6
--- /dev/null
+++ b/changelogs/unreleased/dblessing_fix_ldap_server_config.yml
@@ -0,0 +1,5 @@
+---
+title: Fix missing setting LDAP servers
+merge_request: 52512
+author:
+type: fixed
diff --git a/changelogs/unreleased/mc-backstage-cleanup-artifact-expiry-migration-problem.yml b/changelogs/unreleased/mc-backstage-cleanup-artifact-expiry-migration-problem.yml
new file mode 100644
index 00000000000..bbe9a5bebfe
--- /dev/null
+++ b/changelogs/unreleased/mc-backstage-cleanup-artifact-expiry-migration-problem.yml
@@ -0,0 +1,5 @@
+---
+title: Cancel artifact expiry backfill background jobs.
+merge_request: 51821
+author:
+type: fixed
diff --git a/changelogs/unreleased/sh-fix-lfs-aws-specific-storage.yml b/changelogs/unreleased/sh-fix-lfs-aws-specific-storage.yml
new file mode 100644
index 00000000000..064755d9116
--- /dev/null
+++ b/changelogs/unreleased/sh-fix-lfs-aws-specific-storage.yml
@@ -0,0 +1,5 @@
+---
+title: Fix LFS not working with S3 specific-storage settings
+merge_request: 52296
+author:
+type: fixed
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index b3ea72fc7e5..bbed08f5044 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -12,6 +12,7 @@ Settings.encrypted_settings['path'] = Settings.absolute(Settings.encrypted_setti
 
 Settings['ldap'] ||= Settingslogic.new({})
 Settings.ldap['enabled'] = false if Settings.ldap['enabled'].nil?
+Settings.ldap['servers'] ||= Settingslogic.new({})
 Settings.ldap['prevent_ldap_sign_in'] = false if Settings.ldap['prevent_ldap_sign_in'].blank?
 Settings.ldap['secret_file'] = Settings.absolute(Settings.ldap['secret_file'] || File.join(Settings.encrypted_settings['path'], "ldap.yaml.enc"))
 
diff --git a/data/whats_new/202101140001_13_08.yml b/data/whats_new/202101140001_13_08.yml
new file mode 100644
index 00000000000..826259fe7be
--- /dev/null
+++ b/data/whats_new/202101140001_13_08.yml
@@ -0,0 +1,83 @@
+---
+- title: Pipeline editor
+  body: |
+    GitLab CI/CD provides you with flexible options to support a variety of advanced pipeline use cases. Pipeline syntax can be verbose and sometimes complicated, especially for those who are new to GitLab CI/CD. In this release, we are proud to introduce our first iteration of the Pipeline Editor.
+
+    The editor makes the CI configuration authoring experience much easier for both novice and advanced users alike. The pipeline editor is a single solution that groups all the existing CI authoring features (and future ones) in a single location. The pipeline editor is the best place to go when configuring your pipeline.
+  stage: Verify
+  self-managed: true
+  gitlab-com: true
+  packages: [Core, Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/ee/ci/pipeline_editor/index.html
+  image_url: https://img.youtube.com/vi/MQpSyvMpsHA/hqdefault.jpg
+  published_at: 2021-01-22
+  release: 13.8
+- title: Visualization of pipeline configuration
+  body: |
+    A complex CI configuration can be difficult to understand as a developer, especially when trying to predict how your pipeline might behave (or misbehave). Without a visual aid, it is challenging to form a mental image of the relationships between all of the jobs and determine how they are interconnected. In our first iteration of a pipeline visualization, you can now see a graphic representation of your `.gitlab-ci.yml` configuration to better understand and predict how your pipelines will perform.
+  stage: verify
+  self-managed: true
+  gitlab-com: true
+  packages: [Core, Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/ee/ci/yaml/visualization.html
+  image_url: https://about.gitlab.com/images/13_8/pipeline_visual_builder.png
+  published_at: 2021-01-22
+  release: 13.8
+- title: Deployment frequency charts
+  packages: [Ultimate]
+  self-managed: true
+  gitlab-com: true
+  image_url: https://about.gitlab.com/images/13_8/deployment_graph.png
+  url: https://docs.gitlab.com/ee/user/analytics/ci_cd_analytics.html#deployment-frequency-charts
+  stage: Release
+  body: |
+    Knowing and monitoring deployment frequency is a starting point for organizations adopting DevOps. We are proud to introduce the deployment frequency charts at the project level so that you and your development teams can monitor the efficiency of deployments over time, find bottlenecks, and make improvements when necessary. This is the first of the DORA metrics that we are making available within GitLab out of the box.
+  published_at: 2021-01-22
+  release: 13.8
+- title: Scope a board to the current iteration
+  body: |
+    Many teams use boards to manage issues during an iteration. In 13.6, we added support for [filtering issues on a board to a specific Iteration](https://gitlab.com/gitlab-org/gitlab/-/issues/118742), but it is cumbersome to remember to apply that filter every time you go to your board. In this release, we've added the ability to scope your board to the currently active iteration.
+  stage: plan
+  self-managed: true
+  gitlab-com: true
+  packages: [Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/ee/user/project/issue_board.html#configurable-issue-boards
+  image_url: https://about.gitlab.com/images/13_8/scope_board_to_current_iteration.png
+  published_at: 2021-01-22
+  release: 13.8
+- title: Download artifacts directly from the merge request widget
+  body: |
+    We added the ability to download build artifacts directly from the MR widget. This is especially useful for mobile development. An example of this is where users want to test an Android package of a specific build created on a physical device or an emulator. You can now access these artifacts directly from the merge request widget without having to find the artifacts buried in the pipeline view.
+  stage: Release
+  self-managed: true
+  gitlab-com: true
+  packages: [Core, Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html#downloading-artifacts
+  image_url: https://about.gitlab.com/images/13_8/artifact_mr.png
+  published_at: 2021-01-22
+  release: 13.8
+- title: Rebase quick action for merge requests
+  body: |
+    Rebase is a Git command used to reapply commits on top of a new commit. In practice, this means reapplying commits from a feature branch on top of the latest version of the target branch (e.g. `main`). In this way, it is possible to bring the feature branch up to date and resolve any conflicts without using a merge commit with the benefit of a simpler linear Git history.
+
+    GitLab 13.8 brings the ability to execute the rebase quick action in merge requests, allowing you to quickly invoke the rebase Git utility.
+  stage: create
+  self-managed: true
+  gitlab-com: true
+  packages: [Core, Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/ee/user/project/quick_actions.html#quick-actions-for-issues-merge-requests-and-epics
+  image_url: https://about.gitlab.com/images/13_8/rebase_comment.png
+  published_at: 2021-01-22
+  release: 13.8
+- title: GitLab Pages is now available for Kubernetes deployments of GitLab
+  body: |
+    GitLab Pages is a popular static-site hosting service built into GitLab, and we are excited to announce that it is now available for GitLab instances running on Kubernetes. Pages was one of the last remaining feature gaps compared to an Omnibus deployment.
+  stage: enablement
+  self-managed: true
+  gitlab-com: false
+  packages: [Core, Starter, Premium, Ultimate]
+  url: https://docs.gitlab.com/charts/charts/gitlab/gitlab-pages/index.html
+  image_url: https://about.gitlab.com/images/home/kubernetes.png
+  published_at: 2021-01-22
+  release: 13.8
+
diff --git a/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb
index f11c0bbe33a..1ffe9abbc58 100644
--- a/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb
+++ b/db/post_migrate/20201208175117_schedule_backfilling_artifact_expiry_migration.rb
@@ -24,12 +24,18 @@ class ScheduleBackfillingArtifactExpiryMigration < ActiveRecord::Migration[6.0]
     # Needs to be removed in a later migration
     add_concurrent_index(:ci_job_artifacts, %i(id created_at), where: INDEX_CONDITION, name: INDEX_NAME)
 
-    queue_background_migration_jobs_by_range_at_intervals(
-      JobArtifact.without_expiry_date.before_switch,
-      ::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate,
-      2.minutes,
-      batch_size: 200_000
-    )
+    # queue_background_migration_jobs_by_range_at_intervals(
+    #   JobArtifact.without_expiry_date.before_switch,
+    #   ::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate,
+    #   2.minutes,
+    #   batch_size: 200_000
+    # )
+    # The scheduling code was using the full class symbol
+    # (`::Gitlab::BackgroundMigration::BackfillArtifactExpiryDate`) instead of a
+    # string with the class name (`BackfillArtifactExpiryDate`) by mistake,
+    # which resulted in an error. It is commented out so it's a no-op to prevent
+    # errors and will be reintroduced with
+    # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51822.
   end
 
   def down
diff --git a/db/post_migrate/20210115215854_cancel_artifact_expiry_backfill.rb b/db/post_migrate/20210115215854_cancel_artifact_expiry_backfill.rb
new file mode 100644
index 00000000000..8a03a90a1c5
--- /dev/null
+++ b/db/post_migrate/20210115215854_cancel_artifact_expiry_backfill.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class CancelArtifactExpiryBackfill < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'BackfillArtifactExpiryDate'
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration.steal(MIGRATION) do |job|
+      job.delete
+
+      false
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/schema_migrations/20210115215854 b/db/schema_migrations/20210115215854
new file mode 100644
index 00000000000..c27de41ad0a
--- /dev/null
+++ b/db/schema_migrations/20210115215854
@@ -0,0 +1 @@
+cb846ce5f6270cfdc543c3d4ad3e861b2a92445b952ee8f0a02f4171b9792411
\ No newline at end of file
diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md
index 6e2ddfb812c..95f67b20ab5 100644
--- a/doc/administration/geo/setup/database.md
+++ b/doc/administration/geo/setup/database.md
@@ -634,6 +634,20 @@ For each Patroni instance on the secondary site:
 to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your Geo secondary. This will ensure that Patroni recognizes the replication slot as permanent and will not drop it upon restarting.
 1. If database replication to the secondary was paused before migration, resume replication once Patroni is confirmed working on the primary.
 
+## Migrating a single PostgreSQL node to Patroni
+
+Before the introduction of Patroni, Geo had no Omnibus support for HA setups on the secondary node.
+
+With Patroni it's now possible to support that. In order to migrate the existing PostgreSQL to Patroni:
+
+1. Make sure you have a Consul cluster setup on the secondary (similar to how you set it up on the primary).
+1. [Configure a permanent replication slot](#step-1-configure-patroni-permanent-replication-slot-on-the-primary-site).
+1. [Configure a Standby Cluster](#step-2-configure-a-standby-cluster-on-the-secondary-site)
+   on that single node machine.
+   
+You will end up with a "Standby Cluster" with a single node. That allows you to later on add additional patroni nodes
+by following the same instructions above.
+
 ## Troubleshooting
 
 Read the [troubleshooting document](../replication/troubleshooting.md).
diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md
index 38ef01b7537..ae672d8414f 100644
--- a/doc/user/profile/notifications.md
+++ b/doc/user/profile/notifications.md
@@ -149,6 +149,7 @@ Users are notified of the following events:
 | Password changed by administrator | User           | Security email, always sent when an administrator changes the password of another user |
 | Two-factor authentication disabled | User          | Security email, always sent. |
 | New user created             | User                | Sent on user creation, except for OmniAuth (LDAP)|
+| New SAML/SCIM user provisioned. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276018) in GitLab 13.8  | User            | Sent when a user is provisioned through SAML/SCIM |
 | User added to project        | User                | Sent when user is added to project |
 | Project access level changed | User                | Sent when user project access level is changed |
 | User added to group          | User                | Sent when user is added to group |
diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md
index bc718ae867f..dc5e1f81a63 100644
--- a/doc/user/project/merge_requests/getting_started.md
+++ b/doc/user/project/merge_requests/getting_started.md
@@ -161,7 +161,7 @@ Feature.disable(:merge_request_reviewers)
 Feature.disable(:merge_request_reviewers, Project.find(<project id>))
 ```
 
-#### Reviewer approval rules
+#### Approval Rule information for Reviewers **(STARTER)**
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233736) in GitLab 13.8.
 > - It was [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default.
@@ -169,11 +169,15 @@ Feature.disable(:merge_request_reviewers, Project.find(<project id>))
 > - It's enabled on GitLab.com.
 > - It's recommended for production use.
 > - It can be enabled or disabled for a single project.
-> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-reviewer-approval-rules). **(CORE ONLY)**
+> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-approval-rule-information-for-reviewers). **(STARTER ONLY)**
 
-When editing the **Reviewers** field in a new or existing merge request, this feature
+WARNING:
+This feature might not be available to you. Check the **version history** note above for details.
+
+When editing the **Reviewers** field in a new or existing merge request, GitLab
 displays the name of the matching [approval rule](merge_request_approvals.md#approval-rules)
-below the name of each suggested reviewer. [Code Owners](../code_owners.md) are displayed as `Codeowner` without group detail. We intend to iterate on this feature in future releases.
+below the name of each suggested reviewer. [Code Owners](../code_owners.md) are displayed as **Code Owner** without group detail.
+We intend to iterate on this feature in future releases.
 
 This example shows reviewers and approval rules when creating a new merge request:
 
@@ -183,7 +187,7 @@ This example shows reviewers and approval rules in a merge request sidebar:
 
 ![Reviewer approval rules in sidebar](img/reviewer_approval_rules_sidebar_v13_8.png)
 
-##### Enable or disable Reviewer Approval Rules **(CORE ONLY)**
+##### Enable or disable Approval Rule information for Reviewers **(STARTER ONLY)**
 
 Merge Request Reviewers is under development and ready for production use.
 It is deployed behind a feature flag that is **enabled by default**.
diff --git a/doc/user/project/merge_requests/merge_request_approvals.md b/doc/user/project/merge_requests/merge_request_approvals.md
index 01de98edeac..887f563be51 100644
--- a/doc/user/project/merge_requests/merge_request_approvals.md
+++ b/doc/user/project/merge_requests/merge_request_approvals.md
@@ -164,8 +164,15 @@ To add or edit the default merge request approval rule:
      the rule.
 1. Click **Add approval rule** or **Update approval rule**.
 
-Any merge requests that were created before changing the rules will not be changed.
-They will keep the original approval rules, unless manually [overridden](#editing--overriding-approval-rules-per-merge-request).
+When [approval rule overrides](#prevent-overriding-default-approvals) are allowed,
+changes to these default rules will **not** be applied to existing merge
+requests, except for changes to the [target branch](#scoped-to-protected-branch) of
+the rule.
+
+When approval rule overrides are not allowed, all changes to these default rules
+will be applied to existing merge requests. Any approval rules that had previously been
+manually [overridden](#editing--overriding-approval-rules-per-merge-request) during a
+period when approval rule overrides where allowed, will not be modified.
 
 NOTE:
 If a merge request targets a different project, such as from a fork to the upstream project,
@@ -252,6 +259,11 @@ one of the following is possible:
 
   ![Remove approval](img/remove_approval.png)
 
+When [approval rule overrides](#prevent-overriding-default-approvals) are allowed,
+changes to default approval rules will **not** be applied to existing
+merge requests, except for changes to the [target branch](#scoped-to-protected-branch)
+of the rule.
+
 NOTE:
 The merge request author is not allowed to approve their own merge request if
 [**Prevent author approval**](#allowing-merge-request-authors-to-approve-their-own-merge-requests)
diff --git a/spec/controllers/repositories/lfs_storage_controller_spec.rb b/spec/controllers/repositories/lfs_storage_controller_spec.rb
index e361a7442bb..7ddc5723e2e 100644
--- a/spec/controllers/repositories/lfs_storage_controller_spec.rb
+++ b/spec/controllers/repositories/lfs_storage_controller_spec.rb
@@ -11,24 +11,76 @@ RSpec.describe Repositories::LfsStorageController do
   let_it_be(:pat) { create(:personal_access_token, user: user, scopes: ['write_repository']) }
 
   let(:lfs_enabled) { true }
+  let(:params) do
+    {
+      repository_path: "#{project.full_path}.git",
+      oid: '6b9765d3888aaec789e8c309eb05b05c3a87895d6ad70d2264bd7270fff665ac',
+      size: '6725030'
+    }
+  end
 
   before do
     stub_config(lfs: { enabled: lfs_enabled })
   end
 
-  describe 'PUT #upload_finalize' do
+  describe 'PUT #upload_authorize' do
     let(:headers) { workhorse_internal_api_request_header }
     let(:extra_headers) { {} }
-    let(:uploaded_file) { temp_file }
 
-    let(:params) do
-      {
-        repository_path: "#{project.full_path}.git",
-        oid: '6b9765d3888aaec789e8c309eb05b05c3a87895d6ad70d2264bd7270fff665ac',
-        size: '6725030'
-      }
+    before do
+      request.headers.merge!(extra_headers)
+      request.headers.merge!(headers)
+    end
+
+    subject do
+      put :upload_authorize, params: params
     end
 
+    context 'with unauthorized roles' do
+      where(:user_role, :expected_status) do
+        :guest     | :forbidden
+        :anonymous | :unauthorized
+      end
+
+      with_them do
+        let(:extra_headers) do
+          if user_role == :anonymous
+            {}
+          else
+            { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials(user.username, pat.token) }
+          end
+        end
+
+        before do
+          project.send("add_#{user_role}", user) unless user_role == :anonymous
+        end
+
+        it_behaves_like 'returning response status', params[:expected_status]
+      end
+    end
+
+    context 'with at least developer role' do
+      let(:extra_headers) { { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials(user.username, pat.token) } }
+
+      before do
+        project.add_developer(user)
+      end
+
+      it 'sets Workhorse with a max limit' do
+        expect(LfsObjectUploader).to receive(:workhorse_authorize).with(has_length: false, maximum_size: params[:size].to_i).and_call_original
+
+        subject
+
+        expect(response).to have_gitlab_http_status(:ok)
+      end
+    end
+  end
+
+  describe 'PUT #upload_finalize' do
+    let(:headers) { workhorse_internal_api_request_header }
+    let(:extra_headers) { {} }
+    let(:uploaded_file) { temp_file }
+
     before do
       request.headers.merge!(extra_headers)
       request.headers.merge!(headers)
diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb
index aeec18aee2b..4e18c9cb4ca 100644
--- a/spec/requests/lfs_http_spec.rb
+++ b/spec/requests/lfs_http_spec.rb
@@ -750,7 +750,7 @@ RSpec.describe 'Git LFS API and storage' do
                         expect(json_response['RemoteObject']).to have_key('GetURL')
                         expect(json_response['RemoteObject']).to have_key('StoreURL')
                         expect(json_response['RemoteObject']).to have_key('DeleteURL')
-                        expect(json_response['RemoteObject']).not_to have_key('MultipartUpload')
+                        expect(json_response['RemoteObject']).to have_key('MultipartUpload')
                         expect(json_response['LfsOid']).to eq(sample_oid)
                         expect(json_response['LfsSize']).to eq(sample_size)
                       end