13 files changed, 281 insertions, 22 deletions

diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb
new file mode 100644
index 00000000000..0d49f3dd8c5
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_pages_access_level.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # corrects stored pages access level on db depending on project visibility
+    class FixPagesAccessLevel
+      # Copy routable here to avoid relying on application logic
+      module Routable
+        def build_full_path
+          if parent && path
+            parent.build_full_path + '/' + path
+          else
+            path
+          end
+        end
+      end
+
+      # Namespace
+      class Namespace < ApplicationRecord
+        self.table_name = 'namespaces'
+        self.inheritance_column = :_type_disabled
+
+        include Routable
+
+        belongs_to :parent, class_name: "Namespace"
+      end
+
+      # Project
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+        self.inheritance_column = :_type_disabled
+
+        include Routable
+
+        belongs_to :namespace
+        alias_method :parent, :namespace
+        alias_attribute :parent_id, :namespace_id
+
+        PRIVATE = 0
+        INTERNAL = 10
+        PUBLIC = 20
+
+        def pages_deployed?
+          Dir.exist?(public_pages_path)
+        end
+
+        def public_pages_path
+          File.join(pages_path, 'public')
+        end
+
+        def pages_path
+          # TODO: when we migrate Pages to work with new storage types, change here to use disk_path
+          File.join(Settings.pages.path, build_full_path)
+        end
+      end
+
+      # ProjectFeature
+      class ProjectFeature < ActiveRecord::Base
+        include ::EachBatch
+
+        self.table_name = 'project_features'
+
+        belongs_to :project
+
+        PRIVATE = 10
+        ENABLED = 20
+        PUBLIC  = 30
+      end
+
+      def perform(start_id, stop_id)
+        fix_public_access_level(start_id, stop_id)
+
+        make_internal_projects_public(start_id, stop_id)
+
+        fix_private_access_level(start_id, stop_id)
+      end
+
+      private
+
+      def access_control_is_enabled
+        @access_control_is_enabled = Gitlab.config.pages.access_control
+      end
+
+      # Public projects are allowed to have only enabled pages_access_level
+      # which is equivalent to public
+      def fix_public_access_level(start_id, stop_id)
+        project_features(start_id, stop_id, ProjectFeature::PUBLIC, Project::PUBLIC).each_batch do |features|
+          features.update_all(pages_access_level: ProjectFeature::ENABLED)
+        end
+      end
+
+      # If access control is disabled and project has pages deployed
+      # project will become unavailable when access control will become enabled
+      # we make these projects public to avoid negative surprise to user
+      def make_internal_projects_public(start_id, stop_id)
+        return if access_control_is_enabled
+
+        project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::INTERNAL).find_each do |project_feature|
+          next unless project_feature.project.pages_deployed?
+
+          project_feature.update(pages_access_level: ProjectFeature::PUBLIC)
+        end
+      end
+
+      # Private projects are not allowed to have enabled access level, only `private` and `public`
+      # If access control is enabled, these projects currently behave as if the have `private` pages_access_level
+      # if access control is disabled, these projects currently behave as if the have `public` pages_access_level
+      # so we preserve this behaviour for projects with pages already deployed
+      # for project without pages we always set `private` access_level
+      def fix_private_access_level(start_id, stop_id)
+        project_features(start_id, stop_id, ProjectFeature::ENABLED, Project::PRIVATE).find_each do |project_feature|
+          if access_control_is_enabled
+            project_feature.update!(pages_access_level: ProjectFeature::PRIVATE)
+          else
+            fixed_access_level = project_feature.project.pages_deployed? ? ProjectFeature::PUBLIC : ProjectFeature::PRIVATE
+            project_feature.update!(pages_access_level: fixed_access_level)
+          end
+        end
+      end
+
+      def project_features(start_id, stop_id, pages_access_level, project_visibility_level)
+        ProjectFeature.where(id: start_id..stop_id).joins(:project)
+          .where(pages_access_level: pages_access_level)
+          .where(projects: { visibility_level: project_visibility_level })
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index c911bfa7ff6..afad391e8e0 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -20,6 +20,12 @@ module Gitlab
             end
           end
 
+          def uses_unsupported_legacy_trigger?
+            trigger_request.present? &&
+              trigger_request.trigger.legacy? &&
+              !trigger_request.trigger.supports_legacy_tokens?
+          end
+
           def branch_exists?
             strong_memoize(:is_branch) do
               project.repository.branch_exists?(ref)
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index aaa3daddcc5..357a1d55b3b 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -14,6 +14,10 @@ module Gitlab
                 return error('Pipelines are disabled!')
               end
 
+              if @command.uses_unsupported_legacy_trigger?
+                return error('Trigger token is invalid because is not owned by any user')
+              end
+
               unless allowed_to_trigger_pipeline?
                 if can?(current_user, :create_pipeline, project)
                   return error("Insufficient permissions for protected ref '#{command.ref}'")
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index f176771775e..89eccce69f6 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -41,6 +41,8 @@ dependency_scanning:
           DS_PULL_ANALYZER_IMAGE_TIMEOUT \
           DS_RUN_ANALYZER_TIMEOUT \
           DS_PYTHON_VERSION \
+          PIP_INDEX_URL \
+          PIP_EXTRA_INDEX_URL \
         ) \
         --volume "$PWD:/code" \
         --volume /var/run/docker.sock:/var/run/docker.sock \
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index b7b7578cef9..a7d9ba51277 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -464,6 +464,18 @@ module Gitlab
         end
       end
 
+      # Returns path to url mappings for submodules
+      #
+      # Ex.
+      #   @repository.submodule_urls_for('master')
+      #   # => { 'rack' => 'git@localhost:rack.git' }
+      #
+      def submodule_urls_for(ref)
+        wrapped_gitaly_errors do
+          gitaly_submodule_urls_for(ref)
+        end
+      end
+
       # Return total commits count accessible from passed ref
       def commit_count(ref)
         wrapped_gitaly_errors do
@@ -1059,12 +1071,16 @@ module Gitlab
 
         return unless commit_object && commit_object.type == :COMMIT
 
+        urls = gitaly_submodule_urls_for(ref)
+        urls && urls[path]
+      end
+
+      def gitaly_submodule_urls_for(ref)
         gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
         return unless gitmodules
 
-        found_module = GitmodulesParser.new(gitmodules.data).parse[path]
-
-        found_module && found_module['url']
+        submodules = GitmodulesParser.new(gitmodules.data).parse
+        submodules.transform_values { |submodule| submodule['url'] }
       end
 
       # Returns true if the given ref name exists
diff --git a/lib/gitlab/graphql/representation/submodule_tree_entry.rb b/lib/gitlab/graphql/representation/submodule_tree_entry.rb
new file mode 100644
index 00000000000..65716dff75d
--- /dev/null
+++ b/lib/gitlab/graphql/representation/submodule_tree_entry.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Representation
+      class SubmoduleTreeEntry < SimpleDelegator
+        class << self
+          def decorate(submodules, tree)
+            repository = tree.repository
+            submodule_links = Gitlab::SubmoduleLinks.new(repository)
+
+            submodules.map do |submodule|
+              self.new(submodule, submodule_links.for(submodule, tree.sha))
+            end
+          end
+        end
+
+        def initialize(submodule, submodule_links)
+          @submodule_links = submodule_links
+
+          super(submodule)
+        end
+
+        def web_url
+          @submodule_links.first
+        end
+
+        def tree_url
+          @submodule_links.last
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index efd3f550a22..1b545b1d049 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -28,7 +28,7 @@ module Gitlab
                     links: 'Releases::Link',
                     metrics_setting: 'ProjectMetricsSetting' }.freeze
 
-      USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id owner_id].freeze
 
       PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
 
@@ -78,6 +78,9 @@ module Gitlab
       def create
         return if unknown_service?
 
+        # Do not import legacy triggers
+        return if !Feature.enabled?(:use_legacy_pipeline_triggers, @project) && legacy_trigger?
+
         setup_models
 
         generate_imported_object
@@ -278,6 +281,10 @@ module Gitlab
           !Object.const_defined?(parsed_relation_hash['type'])
       end
 
+      def legacy_trigger?
+        @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil?
+      end
+
       def find_or_create_object!
         return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature
 
diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb
new file mode 100644
index 00000000000..a6c0369d864
--- /dev/null
+++ b/lib/gitlab/submodule_links.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class SubmoduleLinks
+    include Gitlab::Utils::StrongMemoize
+
+    def initialize(repository)
+      @repository = repository
+    end
+
+    def for(submodule, sha)
+      submodule_url = submodule_url_for(sha)[submodule.path]
+      SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository)
+    end
+
+    private
+
+    attr_reader :repository
+
+    def submodule_url_for(sha)
+      strong_memoize(:"submodule_links_for_#{sha}") do
+        repository.submodule_urls_for(sha)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 0180fe7fa71..055e01a9399 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -130,7 +130,7 @@ module Gitlab
 
       def usage_counters
         {
-          web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count
+          web_ide_commits: Gitlab::UsageDataCounters::WebIdeCommitsCounter.total_count
         }
       end
 
diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb
new file mode 100644
index 00000000000..123b8e1bef1
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/redis_counter.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    module RedisCounter
+      def increment
+        Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
+      end
+
+      def total_count
+        Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i }
+      end
+
+      def redis_counter_key
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb b/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb
new file mode 100644
index 00000000000..62236fa07a3
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    class WebIdeCommitsCounter
+      extend RedisCounter
+
+      def self.redis_counter_key
+        'WEB_IDE_COMMITS_COUNT'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb
deleted file mode 100644
index 1cd9b5295b9..00000000000
--- a/lib/gitlab/web_ide_commits_counter.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module WebIdeCommitsCounter
-    WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze
-
-    class << self
-      def increment
-        Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) }
-      end
-
-      def total_count
-        Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/zoom_link_extractor.rb b/lib/gitlab/zoom_link_extractor.rb
new file mode 100644
index 00000000000..d9994898a08
--- /dev/null
+++ b/lib/gitlab/zoom_link_extractor.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Detect links matching the following formats:
+# Zoom Start links: https://zoom.us/s/<meeting-id>
+# Zoom Join links: https://zoom.us/j/<meeting-id>
+# Personal Zoom links: https://zoom.us/my/<meeting-id>
+# Vanity Zoom links: https://gitlab.zoom.us/j/<meeting-id> (also /s and /my)
+
+module Gitlab
+  class ZoomLinkExtractor
+    ZOOM_REGEXP = %r{https://(?:[\w-]+\.)?zoom\.us/(?:s|j|my)/\S+}.freeze
+
+    def initialize(text)
+      @text = text.to_s
+    end
+
+    def links
+      @text.scan(ZOOM_REGEXP)
+    end
+  end
+end