Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

4 files changed, 470 insertions, 195 deletions

diff --git a/spec/support/shared_examples/services/boards/issues_move_service_shared_examples.rb b/spec/support/shared_examples/services/boards/issues_move_service_shared_examples.rb
index 7d4fbeea0dc..d9b837258ce 100644
--- a/spec/support/shared_examples/services/boards/issues_move_service_shared_examples.rb
+++ b/spec/support/shared_examples/services/boards/issues_move_service_shared_examples.rb
@@ -100,8 +100,8 @@ RSpec.shared_examples 'issues move service' do |group|
       create(:labeled_issue, project: project, labels: [bug, development], assignees: [assignee])
     end
 
-    it 'returns false' do
-      expect(described_class.new(parent, user, params).execute(issue)).to eq false
+    it 'returns nil' do
+      expect(described_class.new(parent, user, params).execute(issue)).to be_nil
     end
 
     it 'keeps issues labels' do
diff --git a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb
index eafcbd77040..f6e25ee6647 100644
--- a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb
+++ b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb
@@ -69,6 +69,10 @@ RSpec.shared_examples 'a browsable' do
 end
 
 RSpec.shared_examples 'an accessible' do
+  before do
+    stub_feature_flags(container_registry_migration_phase1: false)
+  end
+
   let(:access) do
     [{ 'type' => 'repository',
        'name' => project.full_path,
@@ -203,9 +207,7 @@ RSpec.shared_examples 'a container registry auth service' do
       end
     end
 
-    context 'for private project' do
-      let_it_be(:project) { create(:project) }
-
+    shared_examples 'private project' do
       context 'allow to use scope-less authentication' do
         it_behaves_like 'a valid token'
       end
@@ -345,8 +347,20 @@ RSpec.shared_examples 'a container registry auth service' do
       end
     end
 
-    context 'for public project' do
-      let_it_be(:project) { create(:project, :public) }
+    context 'for private project' do
+      let_it_be_with_reload(:project) { create(:project) }
+
+      it_behaves_like 'private project'
+    end
+
+    context 'for public project with private container registry' do
+      let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
+
+      it_behaves_like 'private project'
+    end
+
+    context 'for public project with container_registry `enabled`' do
+      let_it_be(:project) { create(:project, :public, :container_registry_enabled) }
 
       context 'allow anyone to pull images' do
         let(:current_params) do
@@ -394,8 +408,8 @@ RSpec.shared_examples 'a container registry auth service' do
       end
     end
 
-    context 'for internal project' do
-      let_it_be(:project) { create(:project, :internal) }
+    context 'for internal project with container_registry `enabled`' do
+      let_it_be(:project) { create(:project, :internal, :container_registry_enabled) }
 
       context 'for internal user' do
         context 'allow anyone to pull images' do
@@ -470,6 +484,12 @@ RSpec.shared_examples 'a container registry auth service' do
         end
       end
     end
+
+    context 'for internal project with private container registry' do
+      let_it_be_with_reload(:project) { create(:project, :internal, :container_registry_private) }
+
+      it_behaves_like 'private project'
+    end
   end
 
   context 'delete authorized as maintainer' do
@@ -630,12 +650,8 @@ RSpec.shared_examples 'a container registry auth service' do
           end
         end
 
-        context 'for project with private container registry' do
-          let_it_be(:project, reload: true) { create(:project, :public) }
-
-          before do
-            project.project_feature.update!(container_registry_access_level: ProjectFeature::PRIVATE)
-          end
+        context 'for public project with private container registry' do
+          let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
 
           it_behaves_like 'pullable for being team member'
 
@@ -675,11 +691,7 @@ RSpec.shared_examples 'a container registry auth service' do
     end
 
     context 'for project without container registry' do
-      let_it_be(:project) { create(:project, :public, container_registry_enabled: false) }
-
-      before do
-        project.update!(container_registry_enabled: false)
-      end
+      let_it_be_with_reload(:project) { create(:project, :public, :container_registry_disabled) }
 
       context 'disallow when pulling' do
         let(:current_params) do
@@ -719,12 +731,16 @@ RSpec.shared_examples 'a container registry auth service' do
   context 'support for multiple scopes' do
     let_it_be(:internal_project) { create(:project, :internal) }
     let_it_be(:private_project) { create(:project, :private) }
+    let_it_be(:public_project) { create(:project, :public) }
+    let_it_be(:public_project_private_container_registry) { create(:project, :public, :container_registry_private) }
 
     let(:current_params) do
       {
         scopes: [
           "repository:#{internal_project.full_path}:pull",
-          "repository:#{private_project.full_path}:pull"
+          "repository:#{private_project.full_path}:pull",
+          "repository:#{public_project.full_path}:pull",
+          "repository:#{public_project_private_container_registry.full_path}:pull"
         ]
       }
     end
@@ -744,13 +760,19 @@ RSpec.shared_examples 'a container registry auth service' do
               'actions' => ['pull'] },
             { 'type' => 'repository',
               'name' => private_project.full_path,
+              'actions' => ['pull'] },
+            { 'type' => 'repository',
+              'name' => public_project.full_path,
+              'actions' => ['pull'] },
+            { 'type' => 'repository',
+              'name' => public_project_private_container_registry.full_path,
               'actions' => ['pull'] }
           ]
         end
       end
     end
 
-    context 'user only has access to internal project' do
+    context 'user only has access to internal and public projects' do
       let_it_be(:current_user) { create(:user) }
 
       it_behaves_like 'a browsable' do
@@ -758,16 +780,35 @@ RSpec.shared_examples 'a container registry auth service' do
           [
             { 'type' => 'repository',
               'name' => internal_project.full_path,
+              'actions' => ['pull'] },
+            { 'type' => 'repository',
+              'name' => public_project.full_path,
               'actions' => ['pull'] }
           ]
         end
       end
     end
 
-    context 'anonymous access is rejected' do
+    context 'anonymous user has access only to public project' do
       let(:current_user) { nil }
 
-      it_behaves_like 'a forbidden'
+      it_behaves_like 'a browsable' do
+        let(:access) do
+          [
+            { 'type' => 'repository',
+              'name' => public_project.full_path,
+              'actions' => ['pull'] }
+          ]
+        end
+      end
+
+      context 'with no public container registry' do
+        before do
+          public_project.project_feature.update_column(:container_registry_access_level, ProjectFeature::PRIVATE)
+        end
+
+        it_behaves_like 'a forbidden'
+      end
     end
   end
 
@@ -796,8 +837,8 @@ RSpec.shared_examples 'a container registry auth service' do
       it_behaves_like 'a forbidden'
     end
 
-    context 'for public project' do
-      let_it_be(:project) { create(:project, :public) }
+    context 'for public project with container registry `enabled`' do
+      let_it_be_with_reload(:project) { create(:project, :public, :container_registry_enabled) }
 
       context 'when pulling and pushing' do
         let(:current_params) do
@@ -818,6 +859,19 @@ RSpec.shared_examples 'a container registry auth service' do
       end
     end
 
+    context 'for public project with container registry `private`' do
+      let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
+
+      context 'when pulling and pushing' do
+        let(:current_params) do
+          { scopes: ["repository:#{project.full_path}:pull,push"] }
+        end
+
+        it_behaves_like 'a forbidden'
+        it_behaves_like 'not a container repository factory'
+      end
+    end
+
     context 'for registry catalog' do
       let(:current_params) do
         { scopes: ["registry:catalog:*"] }
@@ -830,15 +884,15 @@ RSpec.shared_examples 'a container registry auth service' do
 
   context 'for deploy tokens' do
     let(:current_params) do
-      { scopes: ["repository:#{project.full_path}:pull"] }
+      { scopes: ["repository:#{project.full_path}:pull"], deploy_token: deploy_token }
     end
 
     context 'when deploy token has read and write registry as scopes' do
-      let(:current_user) { create(:deploy_token, write_registry: true, projects: [project]) }
+      let(:deploy_token) { create(:deploy_token, write_registry: true, projects: [project]) }
 
       shared_examples 'able to login' do
         context 'registry provides read_container_image authentication_abilities' do
-          let(:current_params) { {} }
+          let(:current_params) { { deploy_token: deploy_token } }
           let(:authentication_abilities) { [:read_container_image] }
 
           it_behaves_like 'an authenticated'
@@ -854,7 +908,7 @@ RSpec.shared_examples 'a container registry auth service' do
 
         context 'when pushing' do
           let(:current_params) do
-            { scopes: ["repository:#{project.full_path}:push"] }
+            { scopes: ["repository:#{project.full_path}:push"], deploy_token: deploy_token }
           end
 
           it_behaves_like 'a pushable'
@@ -872,7 +926,7 @@ RSpec.shared_examples 'a container registry auth service' do
 
         context 'when pushing' do
           let(:current_params) do
-            { scopes: ["repository:#{project.full_path}:push"] }
+            { scopes: ["repository:#{project.full_path}:push"], deploy_token: deploy_token }
           end
 
           it_behaves_like 'a pushable'
@@ -890,7 +944,25 @@ RSpec.shared_examples 'a container registry auth service' do
 
         context 'when pushing' do
           let(:current_params) do
-            { scopes: ["repository:#{project.full_path}:push"] }
+            { scopes: ["repository:#{project.full_path}:push"], deploy_token: deploy_token }
+          end
+
+          it_behaves_like 'a pushable'
+        end
+
+        it_behaves_like 'able to login'
+      end
+
+      context 'for public project with private container registry' do
+        let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
+
+        context 'when pulling' do
+          it_behaves_like 'a pullable'
+        end
+
+        context 'when pushing' do
+          let(:current_params) do
+            { scopes: ["repository:#{project.full_path}:push"], deploy_token: deploy_token }
           end
 
           it_behaves_like 'a pushable'
@@ -901,26 +973,26 @@ RSpec.shared_examples 'a container registry auth service' do
     end
 
     context 'when deploy token does not have read_registry scope' do
-      let(:current_user) { create(:deploy_token, projects: [project], read_registry: false) }
+      let(:deploy_token) do
+        create(:deploy_token, projects: [project], read_registry: false)
+      end
 
       shared_examples 'unable to login' do
         context 'registry provides no container authentication_abilities' do
-          let(:current_params) { {} }
           let(:authentication_abilities) { [] }
 
           it_behaves_like 'a forbidden'
         end
 
         context 'registry provides inapplicable container authentication_abilities' do
-          let(:current_params) { {} }
           let(:authentication_abilities) { [:download_code] }
 
           it_behaves_like 'a forbidden'
         end
       end
 
-      context 'for public project' do
-        let_it_be(:project) { create(:project, :public) }
+      context 'for public project with container registry `enabled`' do
+        let_it_be_with_reload(:project) { create(:project, :public, :container_registry_enabled) }
 
         context 'when pulling' do
           it_behaves_like 'a pullable'
@@ -929,6 +1001,16 @@ RSpec.shared_examples 'a container registry auth service' do
         it_behaves_like 'unable to login'
       end
 
+      context 'for public project with container registry `private`' do
+        let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
+
+        context 'when pulling' do
+          it_behaves_like 'an inaccessible'
+        end
+
+        it_behaves_like 'unable to login'
+      end
+
       context 'for internal project' do
         let_it_be(:project) { create(:project, :internal) }
 
@@ -958,16 +1040,24 @@ RSpec.shared_examples 'a container registry auth service' do
     end
 
     context 'when deploy token is not related to the project' do
-      let_it_be(:current_user) { create(:deploy_token, read_registry: false) }
+      let_it_be(:deploy_token) { create(:deploy_token, read_registry: false) }
 
-      context 'for public project' do
-        let_it_be(:project) { create(:project, :public) }
+      context 'for public project with container registry `enabled`' do
+        let_it_be_with_reload(:project) { create(:project, :public, :container_registry_enabled) }
 
         context 'when pulling' do
           it_behaves_like 'a pullable'
         end
       end
 
+      context 'for public project with container registry `private`' do
+        let_it_be_with_reload(:project) { create(:project, :public, :container_registry_private) }
+
+        context 'when pulling' do
+          it_behaves_like 'an inaccessible'
+        end
+      end
+
       context 'for internal project' do
         let_it_be(:project) { create(:project, :internal) }
 
@@ -986,14 +1076,20 @@ RSpec.shared_examples 'a container registry auth service' do
     end
 
     context 'when deploy token has been revoked' do
-      let(:current_user) { create(:deploy_token, :revoked, projects: [project]) }
+      let(:deploy_token) { create(:deploy_token, :revoked, projects: [project]) }
 
-      context 'for public project' do
-        let_it_be(:project) { create(:project, :public) }
+      context 'for public project with container registry `enabled`' do
+        let_it_be(:project) { create(:project, :public, :container_registry_enabled) }
 
         it_behaves_like 'a pullable'
       end
 
+      context 'for public project with container registry `private`' do
+        let_it_be(:project) { create(:project, :public, :container_registry_private) }
+
+        it_behaves_like 'an inaccessible'
+      end
+
       context 'for internal project' do
         let_it_be(:project) { create(:project, :internal) }
 
diff --git a/spec/support/shared_examples/services/jira/requests/base_shared_examples.rb b/spec/support/shared_examples/services/jira/requests/base_shared_examples.rb
new file mode 100644
index 00000000000..56a6d24d557
--- /dev/null
+++ b/spec/support/shared_examples/services/jira/requests/base_shared_examples.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples 'a service that handles Jira API errors' do
+  include AfterNextHelpers
+  using RSpec::Parameterized::TableSyntax
+
+  where(:exception_class, :exception_message, :expected_message) do
+    Errno::ECONNRESET | ''    | 'A connection error occurred'
+    Errno::ECONNREFUSED | ''  | 'A connection error occurred'
+    Errno::ETIMEDOUT | ''     | 'A timeout error occurred'
+    Timeout::Error | ''       | 'A timeout error occurred'
+    URI::InvalidURIError | '' | 'The Jira API URL'
+    SocketError | ''          | 'The Jira API URL'
+    OpenSSL::SSL::SSLError | 'foo'   | 'An SSL error occurred while connecting to Jira: foo'
+    JIRA::HTTPError | 'Unauthorized' | 'The credentials for accessing Jira are not valid'
+    JIRA::HTTPError | 'Forbidden'    | 'The credentials for accessing Jira are not allowed'
+    JIRA::HTTPError | 'Bad Request'  | 'An error occurred while requesting data from Jira'
+    JIRA::HTTPError | 'Foo'          | 'An error occurred while requesting data from Jira.'
+    JIRA::HTTPError | '{"errorMessages":["foo","bar"]}' | 'An error occurred while requesting data from Jira: foo and bar'
+    JIRA::HTTPError | '{"errorMessages":[""]}'          | 'An error occurred while requesting data from Jira.'
+  end
+
+  with_them do
+    it 'handles the error' do
+      stub_client_and_raise(exception_class, exception_message)
+
+      expect(subject).to be_a(ServiceResponse)
+      expect(subject).to be_error
+      expect(subject.message).to include(expected_message)
+    end
+  end
+
+  context 'when the JSON in JIRA::HTTPError is unsafe' do
+    before do
+      stub_client_and_raise(JIRA::HTTPError, error)
+    end
+
+    context 'when JSON is malformed' do
+      let(:error) { '{"errorMessages":' }
+
+      it 'returns the default error message' do
+        expect(subject.message).to eq('An error occurred while requesting data from Jira. Check your Jira integration configuration and try again.')
+      end
+    end
+
+    context 'when JSON contains tags' do
+      let(:error) { '{"errorMessages":["<script>alert(true)</script>foo"]}' }
+
+      it 'sanitizes it' do
+        expect(subject.message).to eq('An error occurred while requesting data from Jira: foo. Check your Jira integration configuration and try again.')
+      end
+    end
+  end
+
+  it 'allows unknown exception classes to bubble' do
+    stub_client_and_raise(StandardError)
+
+    expect { subject }.to raise_exception(StandardError)
+  end
+
+  it 'logs the error' do
+    stub_client_and_raise(Timeout::Error, 'foo')
+
+    expect(Gitlab::ProjectServiceLogger).to receive(:error).with(
+      hash_including(
+        client_url: be_present,
+        message: 'Error sending message',
+        service_class: described_class.name,
+        error: hash_including(
+          exception_class: Timeout::Error.name,
+          exception_message: 'foo',
+          exception_backtrace: be_present
+        )
+      )
+    )
+    expect(subject).to be_error
+  end
+
+  def stub_client_and_raise(exception_class, message = '')
+    # `JIRA::HTTPError` classes take a response from the JIRA API, rather than a `String`.
+    message = double(body: message) if exception_class == JIRA::HTTPError
+
+    allow_next(JIRA::Client).to receive(:get).and_raise(exception_class, message)
+  end
+end
diff --git a/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb b/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb
index 9ffeba1b1d0..c979fdc2bb0 100644
--- a/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb
+++ b/spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb
@@ -1,165 +1,259 @@
 # frozen_string_literal: true
 
 RSpec.shared_examples 'Generate Debian Distribution and component files' do
-  let_it_be(:component_main) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
-  let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
-
-  let_it_be(:architecture_all) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
-  let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
-  let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
-
-  let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated
-  let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed
-  let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation
-  let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation
-  let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation
-  let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago
-
-  def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content)
-    component_file = distribution
-      .component_files
-      .with_component_name(component_name)
-      .with_file_type(component_file_type)
-      .with_architecture_name(architecture_name)
-      .order_updated_asc
-      .last
-
-    expect(component_file).not_to be_nil
-    expect(component_file.updated_at).to eq(release_date)
-
-    unless expected_content.nil?
-      component_file.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_content)
-      end
+  def check_release_files(expected_release_content)
+    distribution.reload
+
+    distribution.file.use_file do |file_path|
+      expect(File.read(file_path)).to eq(expected_release_content)
+    end
+
+    expect(distribution.file_signature).to start_with("-----BEGIN PGP SIGNATURE-----\n")
+    expect(distribution.file_signature).to end_with("\n-----END PGP SIGNATURE-----\n")
+
+    distribution.signed_file.use_file do |file_path|
+      expect(File.read(file_path)).to start_with("-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n#{expected_release_content}-----BEGIN PGP SIGNATURE-----\n")
+      expect(File.read(file_path)).to end_with("\n-----END PGP SIGNATURE-----\n")
     end
   end
 
-  it 'generates Debian distribution and component files', :aggregate_failures do
-    current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456)
-
-    travel_to(current_time) do
-      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-      initial_count = 6
-      destroyed_count = 2
-      # updated_count = 1
-      created_count = 5
-
-      expect { subject }
-        .to not_change { Packages::Package.count }
-        .and not_change { Packages::PackageFile.count }
-        .and change { distribution.reload.updated_at }.to(current_time.round)
-        .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
-        .and change { component_file1.reload.updated_at }.to(current_time.round)
-
-      debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
-      pool_prefix = "pool/unstable/#{project.id}/p/#{package.name}"
-      expected_main_amd64_content = <<~EOF
-      Package: libsample0
-      Source: #{package.name}
-      Version: #{package.version}
-      Installed-Size: 7
-      Maintainer: #{debs[0].debian_fields['Maintainer']}
-      Architecture: amd64
-      Description: Some mostly empty lib
-       Used in GitLab tests.
-       .
-       Testing another paragraph.
-      Multi-Arch: same
-      Homepage: #{debs[0].debian_fields['Homepage']}
-      Section: libs
-      Priority: optional
-      Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
-      Size: 409600
-      MD5sum: #{debs[0].file_md5}
-      SHA256: #{debs[0].file_sha256}
-
-      Package: sample-dev
-      Source: #{package.name} (#{package.version})
-      Version: 1.2.3~binary
-      Installed-Size: 7
-      Maintainer: #{debs[1].debian_fields['Maintainer']}
-      Architecture: amd64
-      Depends: libsample0 (= 1.2.3~binary)
-      Description: Some mostly empty development files
-       Used in GitLab tests.
-       .
-       Testing another paragraph.
-      Multi-Arch: same
-      Homepage: #{debs[1].debian_fields['Homepage']}
-      Section: libdevel
-      Priority: optional
-      Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
-      Size: 409600
-      MD5sum: #{debs[1].file_md5}
-      SHA256: #{debs[1].file_sha256}
-      EOF
-
-      check_component_file(current_time.round, 'main', :packages, 'all', nil)
-      check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
-      check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
-
-      check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
-      check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
-      check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
-
-      main_amd64_size = expected_main_amd64_content.length
-      main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
-      main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
-
-      contrib_all_size = component_file1.size
-      contrib_all_md5sum = component_file1.file_md5
-      contrib_all_sha256 = component_file1.file_sha256
-
-      expected_release_content = <<~EOF
-      Codename: unstable
-      Date: Sat, 25 Jan 2020 15:17:18 +0000
-      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-      Architectures: all amd64 arm64
-      Components: contrib main
-      MD5Sum:
-       #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
-       #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
-      SHA256:
-       #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
-       #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
-      EOF
-
-      distribution.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_release_content)
+  context 'with Debian components and architectures' do
+    let_it_be(:component_main) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
+    let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
+
+    let_it_be(:architecture_all) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
+    let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
+    let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
+
+    let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated
+    let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed
+    let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation
+    let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation
+    let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation
+    let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago
+
+    def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content)
+      component_file = distribution
+        .component_files
+        .with_component_name(component_name)
+        .with_file_type(component_file_type)
+        .with_architecture_name(architecture_name)
+        .order_updated_asc
+        .last
+
+      expect(component_file).not_to be_nil
+      expect(component_file.updated_at).to eq(release_date)
+
+      unless expected_content.nil?
+        component_file.file.use_file do |file_path|
+          expect(File.read(file_path)).to eq(expected_content)
+        end
       end
     end
+
+    it 'generates Debian distribution and component files', :aggregate_failures do
+      current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456)
+
+      travel_to(current_time) do
+        expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+        components_count = 2
+        architectures_count = 3
+
+        initial_count = 6
+        destroyed_count = 2
+        updated_count = 1
+        created_count = components_count * (architectures_count * 2 + 1) - updated_count
+
+        expect { subject }
+          .to not_change { Packages::Package.count }
+          .and not_change { Packages::PackageFile.count }
+          .and change { distribution.reload.updated_at }.to(current_time.round)
+          .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
+          .and change { component_file1.reload.updated_at }.to(current_time.round)
+
+        package_files = package.package_files.order(id: :asc).preload_debian_file_metadata.to_a
+        pool_prefix = 'pool/unstable'
+        pool_prefix += "/#{project.id}" if container_type == :group
+        pool_prefix += "/p/#{package.name}/#{package.version}"
+        expected_main_amd64_content = <<~EOF
+        Package: libsample0
+        Source: #{package.name}
+        Version: #{package.version}
+        Installed-Size: 7
+        Maintainer: #{package_files[2].debian_fields['Maintainer']}
+        Architecture: amd64
+        Description: Some mostly empty lib
+         Used in GitLab tests.
+         .
+         Testing another paragraph.
+        Multi-Arch: same
+        Homepage: #{package_files[2].debian_fields['Homepage']}
+        Section: libs
+        Priority: optional
+        Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
+        Size: 409600
+        MD5sum: #{package_files[2].file_md5}
+        SHA256: #{package_files[2].file_sha256}
+
+        Package: sample-dev
+        Source: #{package.name} (#{package.version})
+        Version: 1.2.3~binary
+        Installed-Size: 7
+        Maintainer: #{package_files[3].debian_fields['Maintainer']}
+        Architecture: amd64
+        Depends: libsample0 (= 1.2.3~binary)
+        Description: Some mostly empty development files
+         Used in GitLab tests.
+         .
+         Testing another paragraph.
+        Multi-Arch: same
+        Homepage: #{package_files[3].debian_fields['Homepage']}
+        Section: libdevel
+        Priority: optional
+        Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
+        Size: 409600
+        MD5sum: #{package_files[3].file_md5}
+        SHA256: #{package_files[3].file_sha256}
+        EOF
+
+        expected_main_amd64_di_content = <<~EOF
+        Section: misc
+        Priority: extra
+        Filename: #{pool_prefix}/sample-udeb_1.2.3~alpha2_amd64.udeb
+        Size: 409600
+        MD5sum: #{package_files[4].file_md5}
+        SHA256: #{package_files[4].file_sha256}
+        EOF
+
+        expected_main_source_content = <<~EOF
+        Package: #{package.name}
+        Binary: sample-dev, libsample0, sample-udeb
+        Version: #{package.version}
+        Maintainer: #{package_files[1].debian_fields['Maintainer']}
+        Build-Depends: debhelper-compat (= 13)
+        Architecture: any
+        Standards-Version: 4.5.0
+        Format: 3.0 (native)
+        Files:
+         #{package_files[1].file_md5} #{package_files[1].size} #{package_files[1].file_name}
+         d5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz
+        Checksums-Sha256:
+         #{package_files[1].file_sha256} #{package_files[1].size} #{package_files[1].file_name}
+         40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz
+        Checksums-Sha1:
+         #{package_files[1].file_sha1} #{package_files[1].size} #{package_files[1].file_name}
+         c5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz
+        Homepage: #{package_files[1].debian_fields['Homepage']}
+        Section: misc
+        Priority: extra
+        Directory: #{pool_prefix}
+        EOF
+
+        check_component_file(current_time.round, 'main', :packages, 'all', nil)
+        check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
+        check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'main', :di_packages, 'all', nil)
+        check_component_file(current_time.round, 'main', :di_packages, 'amd64', expected_main_amd64_di_content)
+        check_component_file(current_time.round, 'main', :di_packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'main', :source, nil, expected_main_source_content)
+
+        check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
+        check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
+        check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'contrib', :di_packages, 'all', nil)
+        check_component_file(current_time.round, 'contrib', :di_packages, 'amd64', nil)
+        check_component_file(current_time.round, 'contrib', :di_packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'contrib', :source, nil, nil)
+
+        main_amd64_size = expected_main_amd64_content.length
+        main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
+        main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
+
+        contrib_all_size = component_file1.size
+        contrib_all_md5sum = component_file1.file_md5
+        contrib_all_sha256 = component_file1.file_sha256
+
+        main_amd64_di_size = expected_main_amd64_di_content.length
+        main_amd64_di_md5sum = Digest::MD5.hexdigest(expected_main_amd64_di_content)
+        main_amd64_di_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_di_content)
+
+        main_source_size = expected_main_source_content.length
+        main_source_md5sum = Digest::MD5.hexdigest(expected_main_source_content)
+        main_source_sha256 = Digest::SHA256.hexdigest(expected_main_source_content)
+
+        expected_release_content = <<~EOF
+        Codename: unstable
+        Date: Sat, 25 Jan 2020 15:17:18 +0000
+        Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+        Architectures: all amd64 arm64
+        Components: contrib main
+        MD5Sum:
+         #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/source/Source
+         d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/debian-installer/binary-all/Packages
+         #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
+         #{main_amd64_di_md5sum}      #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/debian-installer/binary-arm64/Packages
+         #{main_source_md5sum}      #{main_source_size} main/source/Source
+        SHA256:
+         #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/source/Source
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/debian-installer/binary-all/Packages
+         #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
+         #{main_amd64_di_sha256}      #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/debian-installer/binary-arm64/Packages
+         #{main_source_sha256}      #{main_source_size} main/source/Source
+        EOF
+
+        check_release_files(expected_release_content)
+      end
+
+      create_list(:debian_package, 10, project: project, published_in: project_distribution)
+      control_count = ActiveRecord::QueryRecorder.new { subject2 }.count
+
+      create_list(:debian_package, 10, project: project, published_in: project_distribution)
+      expect { subject3 }.not_to exceed_query_limit(control_count)
+    end
   end
-end
 
-RSpec.shared_examples 'Generate minimal Debian Distribution' do
-  it 'generates minimal distribution', :aggregate_failures do
-    travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
-      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-      expect { subject }
-        .to not_change { Packages::Package.count }
-        .and not_change { Packages::PackageFile.count }
-        .and not_change { distribution.component_files.reset.count }
-
-      expected_release_content = <<~EOF
-      Codename: unstable
-      Date: Sat, 25 Jan 2020 15:17:18 +0000
-      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-      MD5Sum:
-      SHA256:
-      EOF
-
-      distribution.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_release_content)
+  context 'without components and architectures' do
+    it 'generates minimal distribution', :aggregate_failures do
+      travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
+        expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+        expect { subject }
+          .to not_change { Packages::Package.count }
+          .and not_change { Packages::PackageFile.count }
+          .and not_change { distribution.component_files.reset.count }
+
+        expected_release_content = <<~EOF
+        Codename: unstable
+        Date: Sat, 25 Jan 2020 15:17:18 +0000
+        Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+        MD5Sum:
+        SHA256:
+        EOF
+
+        check_release_files(expected_release_content)
       end
     end
   end