Add latest changes from gitlab-org/gitlab@master

18 files changed, 396 insertions, 7 deletions

diff --git a/app/assets/javascripts/diffs/components/diff_line_note_form.vue b/app/assets/javascripts/diffs/components/diff_line_note_form.vue
index 2f09f2e24b2..3f249567daa 100644
--- a/app/assets/javascripts/diffs/components/diff_line_note_form.vue
+++ b/app/assets/javascripts/diffs/components/diff_line_note_form.vue
@@ -10,7 +10,12 @@ import {
 } from '../../notes/components/multiline_comment_utils';
 import noteForm from '../../notes/components/note_form.vue';
 import autosave from '../../notes/mixins/autosave';
-import { DIFF_NOTE_TYPE, INLINE_DIFF_LINES_KEY, PARALLEL_DIFF_VIEW_TYPE } from '../constants';
+import {
+  DIFF_NOTE_TYPE,
+  INLINE_DIFF_LINES_KEY,
+  PARALLEL_DIFF_VIEW_TYPE,
+  OLD_LINE_TYPE,
+} from '../constants';
 
 export default {
   components: {
@@ -113,6 +118,34 @@ export default {
       const lines = getDiffLines();
       return commentLineOptions(lines, this.line, this.line.line_code, side);
     },
+    commentLines() {
+      if (!this.selectedCommentPosition) return [];
+
+      const lines = [];
+      const { start, end } = this.selectedCommentPosition;
+      const diffLines = this.diffFile[INLINE_DIFF_LINES_KEY];
+      let isAdding = false;
+
+      for (let i = 0, diffLinesLength = diffLines.length - 1; i < diffLinesLength; i += 1) {
+        const line = diffLines[i];
+
+        if (start.line_code === line.line_code) {
+          isAdding = true;
+        }
+
+        if (isAdding) {
+          if (line.type !== OLD_LINE_TYPE) {
+            lines.push(line);
+          }
+
+          if (end.line_code === line.line_code) {
+            break;
+          }
+        }
+      }
+
+      return lines;
+    },
   },
   mounted() {
     if (this.isLoggedIn) {
@@ -177,6 +210,7 @@ export default {
       :is-editing="true"
       :line-code="line.line_code"
       :line="line"
+      :lines="commentLines"
       :help-page-path="helpPagePath"
       :diff-file="diffFile"
       :show-suggest-popover="showSuggestPopover"
diff --git a/app/assets/javascripts/lib/utils/text_markdown.js b/app/assets/javascripts/lib/utils/text_markdown.js
index 345dfaf895b..1593a363dd1 100644
--- a/app/assets/javascripts/lib/utils/text_markdown.js
+++ b/app/assets/javascripts/lib/utils/text_markdown.js
@@ -232,7 +232,7 @@ export function insertMarkdownText({
         .join('\n');
     }
   } else if (tag.indexOf(textPlaceholder) > -1) {
-    textToInsert = tag.replace(textPlaceholder, selected);
+    textToInsert = tag.replace(textPlaceholder, selected.replace(/\\n/g, '\n'));
   } else {
     textToInsert = String(startChar) + tag + selected + (wrap ? tag : '');
   }
@@ -322,7 +322,7 @@ export function updateTextForToolbarBtn($toolbarBtn) {
     blockTag: $toolbarBtn.data('mdBlock'),
     wrap: !$toolbarBtn.data('mdPrepend'),
     select: $toolbarBtn.data('mdSelect'),
-    tagContent: $toolbarBtn.data('mdTagContent'),
+    tagContent: $toolbarBtn.attr('data-md-tag-content'),
   });
 }
 
diff --git a/app/assets/javascripts/notes/components/note_form.vue b/app/assets/javascripts/notes/components/note_form.vue
index d74ade15de1..6a60e4de518 100644
--- a/app/assets/javascripts/notes/components/note_form.vue
+++ b/app/assets/javascripts/notes/components/note_form.vue
@@ -60,6 +60,11 @@ export default {
       required: false,
       default: null,
     },
+    lines: {
+      type: Array,
+      required: false,
+      default: () => [],
+    },
     note: {
       type: Object,
       required: false,
@@ -333,6 +338,7 @@ export default {
           :help-page-path="helpPagePath"
           :show-suggest-popover="showSuggestPopover"
           :textarea-value="updatedNoteBody"
+          :lines="lines"
           @handleSuggestDismissed="() => $emit('handleSuggestDismissed')"
         >
           <template #textarea>
diff --git a/app/assets/javascripts/packages/list/constants.js b/app/assets/javascripts/packages/list/constants.js
index 25a55200df2..b4fe3c70dea 100644
--- a/app/assets/javascripts/packages/list/constants.js
+++ b/app/assets/javascripts/packages/list/constants.js
@@ -82,6 +82,10 @@ export const PACKAGE_TYPES = [
     title: s__('PackageRegistry|PyPI'),
     type: PackageType.PYPI,
   },
+  {
+    title: s__('PackageRegistry|RubyGems'),
+    type: PackageType.RUBYGEMS,
+  },
 ];
 
 export const LIST_TITLE_TEXT = s__('PackageRegistry|Package Registry');
diff --git a/app/assets/javascripts/packages/shared/constants.js b/app/assets/javascripts/packages/shared/constants.js
index c0f7f150337..f7de31c2c86 100644
--- a/app/assets/javascripts/packages/shared/constants.js
+++ b/app/assets/javascripts/packages/shared/constants.js
@@ -7,6 +7,7 @@ export const PackageType = {
   NUGET: 'nuget',
   PYPI: 'pypi',
   COMPOSER: 'composer',
+  RUBYGEMS: 'rubygems',
   GENERIC: 'generic',
 };
 
diff --git a/app/assets/javascripts/packages/shared/utils.js b/app/assets/javascripts/packages/shared/utils.js
index d34372e89b6..1f9cb8bf477 100644
--- a/app/assets/javascripts/packages/shared/utils.js
+++ b/app/assets/javascripts/packages/shared/utils.js
@@ -19,6 +19,8 @@ export const getPackageTypeLabel = (packageType) => {
       return s__('PackageType|NuGet');
     case PackageType.PYPI:
       return s__('PackageType|PyPI');
+    case PackageType.RUBYGEMS:
+      return s__('PackageType|RubyGems');
     case PackageType.COMPOSER:
       return s__('PackageType|Composer');
     case PackageType.GENERIC:
diff --git a/app/assets/javascripts/repository/components/upload_blob_modal.vue b/app/assets/javascripts/repository/components/upload_blob_modal.vue
index ec7ba469ca0..d2ff01e7fc1 100644
--- a/app/assets/javascripts/repository/components/upload_blob_modal.vue
+++ b/app/assets/javascripts/repository/components/upload_blob_modal.vue
@@ -168,6 +168,7 @@ export default {
         });
     },
   },
+  validFileMimetypes: [],
 };
 </script>
 <template>
@@ -179,7 +180,12 @@ export default {
       :action-cancel="cancelOptions"
       @primary.prevent="uploadFile"
     >
-      <upload-dropzone class="gl-h-200! gl-mb-4" single-file-selection @change="setFile">
+      <upload-dropzone
+        class="gl-h-200! gl-mb-4"
+        single-file-selection
+        :valid-file-mimetypes="$options.validFileMimetypes"
+        @change="setFile"
+      >
         <div
           v-if="file"
           class="card upload-dropzone-card upload-dropzone-border gl-w-full gl-h-full gl-align-items-center gl-justify-content-center gl-p-3"
diff --git a/app/assets/javascripts/vue_shared/components/markdown/field.vue b/app/assets/javascripts/vue_shared/components/markdown/field.vue
index 220570cd8a0..8f6d94a172a 100644
--- a/app/assets/javascripts/vue_shared/components/markdown/field.vue
+++ b/app/assets/javascripts/vue_shared/components/markdown/field.vue
@@ -77,6 +77,11 @@ export default {
       required: false,
       default: null,
     },
+    lines: {
+      type: Array,
+      required: false,
+      default: () => [],
+    },
     note: {
       type: Object,
       required: false,
@@ -115,6 +120,20 @@ export default {
       return this.referencedUsers.length >= referencedUsersThreshold;
     },
     lineContent() {
+      if (this.lines.length) {
+        return this.lines
+          .map((line) => {
+            const { rich_text: richText, text } = line;
+
+            if (text) {
+              return text;
+            }
+
+            return unescape(stripHtml(richText).replace(/\n/g, ''));
+          })
+          .join('\\n');
+      }
+
       if (this.line) {
         const { rich_text: richText, text } = this.line;
 
@@ -241,6 +260,7 @@ export default {
       :line-content="lineContent"
       :can-suggest="canSuggest"
       :show-suggest-popover="showSuggestPopover"
+      :suggestion-start-index="lines.length"
       @preview-markdown="showPreviewTab"
       @write-markdown="showWriteTab"
       @handleSuggestDismissed="() => $emit('handleSuggestDismissed')"
diff --git a/app/assets/javascripts/vue_shared/components/markdown/header.vue b/app/assets/javascripts/vue_shared/components/markdown/header.vue
index f52dc43aaff..01cf0beea3a 100644
--- a/app/assets/javascripts/vue_shared/components/markdown/header.vue
+++ b/app/assets/javascripts/vue_shared/components/markdown/header.vue
@@ -37,6 +37,11 @@ export default {
       required: false,
       default: false,
     },
+    suggestionStartIndex: {
+      type: Number,
+      required: false,
+      default: 0,
+    },
   },
   data() {
     return {
@@ -54,7 +59,9 @@ export default {
       ].join('\n');
     },
     mdSuggestion() {
-      return ['```suggestion:-0+0', `{text}`, '```'].join('\n');
+      return [['```', `suggestion:-${this.suggestionStartIndex}+0`].join(''), `{text}`, '```'].join(
+        '\n',
+      );
     },
     isMac() {
       // Accessing properties using ?. to allow tests to use
diff --git a/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql b/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql
index 310d8d88904..4ce13827da2 100644
--- a/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql
+++ b/app/assets/javascripts/vue_shared/security_reports/queries/security_report_download_paths.query.graphql
@@ -6,6 +6,7 @@ query securityReportDownloadPaths(
   project(fullPath: $projectPath) {
     mergeRequest(iid: $iid) {
       headPipeline {
+        id
         jobs(securityReportTypes: $reportTypes) {
           nodes {
             name
diff --git a/app/serializers/discussion_entity.rb b/app/serializers/discussion_entity.rb
index bcf6b331192..0dbfe0f0772 100644
--- a/app/serializers/discussion_entity.rb
+++ b/app/serializers/discussion_entity.rb
@@ -2,7 +2,13 @@
 
 class DiscussionEntity < BaseDiscussionEntity
   expose :notes do |discussion, opts|
-    request.note_entity.represent(discussion.notes, opts.merge(with_base_discussion: false))
+    request.note_entity.represent(
+      discussion.notes,
+      opts.merge(
+        with_base_discussion: false,
+        discussion: discussion
+      )
+    )
   end
 
   expose :positions, if: -> (d, _) { display_merge_ref_discussions?(d) } do |discussion|
diff --git a/app/serializers/note_entity.rb b/app/serializers/note_entity.rb
index 9a96778786b..d44958bc0c4 100644
--- a/app/serializers/note_entity.rb
+++ b/app/serializers/note_entity.rb
@@ -36,7 +36,8 @@ class NoteEntity < API::Entities::Note
     end
 
     expose :can_resolve_discussion do |note|
-      note.discussion.resolvable? && note.discussion.can_resolve?(current_user)
+      discussion = options.fetch(:discussion, nil) || note.discussion
+      discussion.resolvable? && discussion.can_resolve?(current_user)
     end
   end
 
diff --git a/app/services/packages/rubygems/create_dependencies_service.rb b/app/services/packages/rubygems/create_dependencies_service.rb
new file mode 100644
index 00000000000..dea429148cf
--- /dev/null
+++ b/app/services/packages/rubygems/create_dependencies_service.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Packages
+  module Rubygems
+    class CreateDependenciesService
+      include BulkInsertSafe
+
+      def initialize(package, gemspec)
+        @package = package
+        @gemspec = gemspec
+      end
+
+      def execute
+        set_dependencies
+      end
+
+      private
+
+      attr_reader :package, :gemspec
+
+      def set_dependencies
+        Packages::Dependency.transaction do
+          dependency_type_rows = gemspec.dependencies.map do |dependency|
+            dependency = Packages::Dependency.safe_find_or_create_by!(
+              name: dependency.name,
+              version_pattern: dependency.requirement.to_s
+            )
+
+            {
+              dependency_id: dependency.id,
+              package_id: package.id,
+              dependency_type: :dependencies
+            }
+          end
+
+          package.dependency_links.upsert_all(
+            dependency_type_rows,
+            unique_by: %i[package_id dependency_id dependency_type]
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/rubygems/create_gemspec_service.rb b/app/services/packages/rubygems/create_gemspec_service.rb
new file mode 100644
index 00000000000..22533264480
--- /dev/null
+++ b/app/services/packages/rubygems/create_gemspec_service.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Packages
+  module Rubygems
+    class CreateGemspecService
+      def initialize(package, gemspec)
+        @package = package
+        @gemspec = gemspec
+      end
+
+      def execute
+        write_gemspec_to_file
+      end
+
+      private
+
+      attr_reader :package, :gemspec
+
+      def write_gemspec_to_file
+        file = Tempfile.new
+
+        begin
+          content = gemspec.to_ruby
+          file.write(content)
+          file.flush
+
+          package.package_files.create!(
+            file: file,
+            size: file.size,
+            file_name: "#{gemspec.name}.gemspec",
+            file_sha1: Digest::SHA1.hexdigest(content),
+            file_md5: Digest::MD5.hexdigest(content),
+            file_sha256: Digest::SHA256.hexdigest(content)
+          )
+        ensure
+          file.close
+          file.unlink
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/rubygems/metadata_extraction_service.rb b/app/services/packages/rubygems/metadata_extraction_service.rb
new file mode 100644
index 00000000000..b3bac1854d7
--- /dev/null
+++ b/app/services/packages/rubygems/metadata_extraction_service.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Packages
+  module Rubygems
+    class MetadataExtractionService
+      def initialize(package, gemspec)
+        @package = package
+        @gemspec = gemspec
+      end
+
+      def execute
+        write_metadata
+      end
+
+      private
+
+      attr_reader :package, :gemspec
+
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def write_metadata
+        metadatum.update!(
+          authors: gemspec&.authors,
+          files: gemspec&.files&.to_json,
+          summary: gemspec&.summary,
+          description: gemspec&.description,
+          email: gemspec&.email,
+          homepage: gemspec&.homepage,
+          licenses: gemspec&.licenses&.to_json,
+          metadata: gemspec&.metadata&.to_json,
+          author: gemspec&.author,
+          bindir: gemspec&.bindir,
+          executables: gemspec&.executables&.to_json,
+          extensions: gemspec&.extensions&.to_json,
+          extra_rdoc_files: gemspec&.extra_rdoc_files&.to_json,
+          platform: gemspec&.platform,
+          post_install_message: gemspec&.post_install_message,
+          rdoc_options: gemspec&.rdoc_options&.to_json,
+          require_paths: gemspec&.require_paths&.to_json,
+          required_ruby_version: gemspec&.required_ruby_version&.to_s,
+          required_rubygems_version: gemspec&.required_rubygems_version&.to_s,
+          requirements: gemspec&.requirements&.to_json,
+          rubygems_version: gemspec&.rubygems_version
+        )
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/PerceivedComplexity
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      def metadatum
+        Packages::Rubygems::Metadatum.safe_find_or_create_by!(package: package)
+      end
+    end
+  end
+end
diff --git a/app/services/packages/rubygems/process_gem_service.rb b/app/services/packages/rubygems/process_gem_service.rb
new file mode 100644
index 00000000000..59bf2a1ec28
--- /dev/null
+++ b/app/services/packages/rubygems/process_gem_service.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+
+module Packages
+  module Rubygems
+    class ProcessGemService
+      include Gitlab::Utils::StrongMemoize
+      include ExclusiveLeaseGuard
+
+      ExtractionError = Class.new(StandardError)
+      DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
+
+      def initialize(package_file)
+        @package_file = package_file
+      end
+
+      def execute
+        return success if process_gem
+
+        error('Gem was not processed')
+      end
+
+      private
+
+      attr_reader :package_file
+
+      def process_gem
+        return false unless package_file
+
+        try_obtain_lease do
+          package.transaction do
+            rename_package_and_set_version
+            rename_package_file
+            ::Packages::Rubygems::MetadataExtractionService.new(package, gemspec).execute
+            ::Packages::Rubygems::CreateGemspecService.new(package, gemspec).execute
+            ::Packages::Rubygems::CreateDependenciesService.new(package, gemspec).execute
+            cleanup_temp_package
+          end
+        end
+
+        true
+      end
+
+      def rename_package_and_set_version
+        package.update!(
+          name: gemspec.name,
+          version: gemspec.version,
+          status: :default
+        )
+      end
+
+      def rename_package_file
+        # Updating file_name updates the path where the file is stored.
+        # We must pass the file again so that CarrierWave can handle the update
+        package_file.update!(
+          file_name: "#{gemspec.name}-#{gemspec.version}.gem",
+          file: package_file.file,
+          package_id: package.id
+        )
+      end
+
+      def cleanup_temp_package
+        temp_package.destroy if package.id != temp_package.id
+      end
+
+      def gemspec
+        strong_memoize(:gemspec) do
+          gem.spec
+        end
+      end
+
+      def success
+        ServiceResponse.success(payload: { package: package })
+      end
+
+      def error(message)
+        ServiceResponse.error(message: message)
+      end
+
+      def temp_package
+        strong_memoize(:temp_package) do
+          package_file.package
+        end
+      end
+
+      def package
+        strong_memoize(:package) do
+          # if package with name/version already exists, use that package
+          package = temp_package.project
+                                .packages
+                                .rubygems
+                                .with_name(gemspec.name)
+                                .with_version(gemspec.version.to_s)
+                                .last
+          package || temp_package
+        end
+      end
+
+      def gem
+        # use_file will set an exclusive lease on the file for as long as
+        # the resulting gem object is being used. This means we are not
+        # able to rename the package_file while also using the gem object.
+        # We need to use a separate AR object to create the gem file to allow
+        # `package_file` to be free for update so we re-find the file here.
+        Packages::PackageFile.find(package_file.id).file.use_file do |file_path|
+          Gem::Package.new(File.open(file_path))
+        end
+      rescue
+        raise ExtractionError.new('Unable to read gem file')
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_key
+        "packages:rubygems:process_gem_service:package:#{package.id}"
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_timeout
+        DEFAULT_LEASE_TIMEOUT
+      end
+    end
+  end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 3dbb087d8d8..eeae5de2637 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -1107,6 +1107,14 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: package_repositories:packages_rubygems_extraction
+  :feature_category: :package_registry
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: pipeline_background:archive_trace
   :feature_category: :continuous_integration
   :has_external_dependencies:
diff --git a/app/workers/packages/rubygems/extraction_worker.rb b/app/workers/packages/rubygems/extraction_worker.rb
new file mode 100644
index 00000000000..1e5cd0b54ce
--- /dev/null
+++ b/app/workers/packages/rubygems/extraction_worker.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Packages
+  module Rubygems
+    class ExtractionWorker # rubocop:disable Scalability/IdempotentWorker
+      include ApplicationWorker
+
+      queue_namespace :package_repositories
+      feature_category :package_registry
+      deduplicate :until_executing
+
+      idempotent!
+
+      def perform(package_file_id)
+        package_file = ::Packages::PackageFile.find_by_id(package_file_id)
+
+        return unless package_file
+
+        ::Packages::Rubygems::ProcessGemService.new(package_file).execute
+
+      rescue ::Packages::Rubygems::ProcessGemService::ExtractionError => e
+        Gitlab::ErrorTracking.log_exception(e, project_id: package_file.project_id)
+        package_file.package.destroy!
+      end
+    end
+  end
+end