Add latest changes from gitlab-org/gitlab@master

14 files changed, 163 insertions, 61 deletions

diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index af7c69f857e..42e7dc751f0 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -36,6 +36,7 @@ module API
         optional :font, type: String, desc: 'Foreground color'
         optional :target_path, type: String, desc: 'Target path'
         optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast type. Defaults to banner', default: -> { 'banner' }
+        optional :dismissable, type: Boolean, desc: 'Is dismissable'
       end
       post do
         authenticated_as_admin!
@@ -75,6 +76,7 @@ module API
         optional :font, type: String, desc: 'Foreground color'
         optional :target_path, type: String, desc: 'Target path'
         optional :broadcast_type, type: String, values: BroadcastMessage.broadcast_types.keys, desc: 'Broadcast Type'
+        optional :dismissable, type: Boolean, desc: 'Is dismissable'
       end
       put ':id' do
         authenticated_as_admin!
diff --git a/lib/api/entities/broadcast_message.rb b/lib/api/entities/broadcast_message.rb
index 403677aa300..e42b110adbe 100644
--- a/lib/api/entities/broadcast_message.rb
+++ b/lib/api/entities/broadcast_message.rb
@@ -3,7 +3,7 @@
 module API
   module Entities
     class BroadcastMessage < Grape::Entity
-      expose :id, :message, :starts_at, :ends_at, :color, :font, :target_path, :broadcast_type
+      expose :id, :message, :starts_at, :ends_at, :color, :font, :target_path, :broadcast_type, :dismissable
       expose :active?, as: :active
     end
   end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index ab43096a1de..f7aabc8ce4f 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -3,7 +3,7 @@
 module API
   module Helpers
     module InternalHelpers
-      attr_reader :redirected_path
+      attr_reader :redirected_path, :container
 
       delegate :wiki?, to: :repo_type
 
@@ -22,10 +22,10 @@ module API
       end
 
       def access_checker_for(actor, protocol)
-        access_checker_klass.new(actor.key_or_user, project, protocol,
+        access_checker_klass.new(actor.key_or_user, container, protocol,
           authentication_abilities: ssh_authentication_abilities,
           namespace_path: namespace_path,
-          project_path: project_path,
+          repository_path: project_path,
           redirected_path: redirected_path)
       end
 
@@ -80,7 +80,7 @@ module API
 
       # rubocop:disable Gitlab/ModuleWithInstanceVariables
       def set_project
-        @project, @repo_type, @redirected_path =
+        @container, @project, @repo_type, @redirected_path =
           if params[:gl_repository]
             Gitlab::GlRepository.parse(params[:gl_repository])
           elsif params[:project]
@@ -92,17 +92,17 @@ module API
       # Project id to pass between components that don't share/don't have
       # access to the same filesystem mounts
       def gl_repository
-        repo_type.identifier_for_container(project)
+        repo_type.identifier_for_container(container)
       end
 
-      def gl_project_path
+      def gl_repository_path
         repository.full_path
       end
 
       # Return the repository depending on whether we want the wiki or the
       # regular repository
       def repository
-        @repository ||= repo_type.repository_for(project)
+        @repository ||= repo_type.repository_for(container)
       end
 
       # Return the Gitaly Address if it is enabled
@@ -111,8 +111,8 @@ module API
 
         {
           repository: repository.gitaly_repository,
-          address: Gitlab::GitalyClient.address(project.repository_storage),
-          token: Gitlab::GitalyClient.token(project.repository_storage),
+          address: Gitlab::GitalyClient.address(container.repository_storage),
+          token: Gitlab::GitalyClient.token(container.repository_storage),
           features: Feature::Gitaly.server_feature_flags
         }
       end
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index ba66404e406..2e1891ac656 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -67,7 +67,7 @@ module API
           when ::Gitlab::GitAccessResult::Success
             payload = {
               gl_repository: gl_repository,
-              gl_project_path: gl_project_path,
+              gl_project_path: gl_repository_path,
               gl_id: Gitlab::GlId.gl_id(actor.user),
               gl_username: actor.username,
               git_config_options: [],
@@ -216,7 +216,7 @@ module API
         post '/post_receive' do
           status 200
 
-          response = PostReceiveService.new(actor.user, project, params).execute
+          response = PostReceiveService.new(actor.user, repository, project, params).execute
 
           ee_post_receive_response_hook(response)
 
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 2b6b10cf044..915567f8106 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -12,7 +12,7 @@ module Backup
       @progress = progress
     end
 
-    def pack
+    def write_info
       # Make sure there is a connection
       ActiveRecord::Base.connection.reconnect!
 
@@ -20,7 +20,11 @@ module Backup
         File.open("#{backup_path}/backup_information.yml", "w+") do |file|
           file << backup_information.to_yaml.gsub(/^---\n/, '')
         end
+      end
+    end
 
+    def pack
+      Dir.chdir(backup_path) do
         # create archive
         progress.print "Creating backup archive: #{tar_file} ... "
         # Set file permissions on open to prevent chmod races.
@@ -31,8 +35,6 @@ module Backup
           puts "creating archive #{tar_file} failed".color(:red)
           raise Backup::Error, 'Backup failed'
         end
-
-        upload
       end
     end
 
@@ -105,8 +107,30 @@ module Backup
       end
     end
 
-    # rubocop: disable Metrics/AbcSize
+    def verify_backup_version
+      Dir.chdir(backup_path) do
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != Gitlab::VERSION
+          progress.puts(<<~HEREDOC.color(:red))
+            GitLab version mismatch:
+              Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
+              Please switch to the following version and try again:
+              version: #{settings[:gitlab_version]}
+          HEREDOC
+          progress.puts
+          progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
+          exit 1
+        end
+      end
+    end
+
     def unpack
+      if ENV['BACKUP'].blank? && non_tarred_backup?
+        progress.puts "Non tarred backup found in #{backup_path}, using that"
+
+        return false
+      end
+
       Dir.chdir(backup_path) do
         # check for existing backups in the backup dir
         if backup_file_list.empty?
@@ -141,21 +165,6 @@ module Backup
           progress.puts 'unpacking backup failed'.color(:red)
           exit 1
         end
-
-        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
-
-        # restoring mismatching backups can lead to unexpected problems
-        if settings[:gitlab_version] != Gitlab::VERSION
-          progress.puts(<<~HEREDOC.color(:red))
-            GitLab version mismatch:
-              Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!
-              Please switch to the following version and try again:
-              version: #{settings[:gitlab_version]}
-          HEREDOC
-          progress.puts
-          progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
-          exit 1
-        end
       end
     end
 
@@ -170,6 +179,10 @@ module Backup
 
     private
 
+    def non_tarred_backup?
+      File.exist?(File.join(backup_path, 'backup_information.yml'))
+    end
+
     def backup_path
       Gitlab.config.backup.path
     end
@@ -252,7 +265,7 @@ module Backup
     def create_attributes
       attrs = {
         key: remote_target,
-        body: File.open(tar_file),
+        body: File.open(File.join(backup_path, tar_file)),
         multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
         encryption: Gitlab.config.backup.upload.encryption,
         encryption_key: Gitlab.config.backup.upload.encryption_key,
diff --git a/lib/gitlab/asset_proxy.rb b/lib/gitlab/asset_proxy.rb
index fd7c58ba68f..fb4369e01d8 100644
--- a/lib/gitlab/asset_proxy.rb
+++ b/lib/gitlab/asset_proxy.rb
@@ -11,12 +11,14 @@ module Gitlab
         return url if asset_host_whitelisted?(url)
 
         "#{Gitlab.config.asset_proxy.url}/#{asset_url_hash(url)}/#{hexencode(url)}"
+      rescue Addressable::URI::InvalidURIError
+        url
       end
 
       private
 
       def asset_host_whitelisted?(url)
-        parsed_url = URI.parse(url)
+        parsed_url = Addressable::URI.parse(url)
 
         Gitlab.config.asset_proxy.domain_regexp&.match?(parsed_url.host)
       end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 45db423187c..7a13a35c096 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -43,15 +43,15 @@ module Gitlab
     PUSH_COMMANDS = %w{git-receive-pack}.freeze
     ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
 
-    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger
+    attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :repository_path, :redirected_path, :auth_result_type, :changes, :logger
 
-    def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil)
+    def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil)
       @actor    = actor
       @project  = project
       @protocol = protocol
-      @authentication_abilities = authentication_abilities
+      @authentication_abilities = Array(authentication_abilities)
       @namespace_path = namespace_path || project&.namespace&.full_path
-      @project_path = project_path || project&.path
+      @repository_path = repository_path || project&.path
       @redirected_path = redirected_path
       @auth_result_type = auth_result_type
     end
@@ -224,7 +224,7 @@ module Gitlab
       return unless user&.can?(:create_projects, namespace)
 
       project_params = {
-        path: project_path,
+        path: repository_path,
         namespace_id: namespace.id,
         visibility_level: Gitlab::VisibilityLevel::PRIVATE
       }
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index 5264bae47a1..13d991cdfbd 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -3,10 +3,10 @@
 module Gitlab
   class GitPostReceive
     include Gitlab::Identifier
-    attr_reader :project, :identifier, :changes, :push_options
+    attr_reader :container, :identifier, :changes, :push_options
 
-    def initialize(project, identifier, changes, push_options = {})
-      @project = project
+    def initialize(container, identifier, changes, push_options = {})
+      @container = container
       @identifier = identifier
       @changes = parse_changes(changes)
       @push_options = push_options
@@ -27,10 +27,10 @@ module Gitlab
     def includes_default_branch?
       # If the branch doesn't have a default branch yet, we presume the
       # first branch pushed will be the default.
-      return true unless project.default_branch.present?
+      return true unless container.default_branch.present?
 
       changes.branch_changes.any? do |change|
-        Gitlab::Git.branch_name(change[:ref]) == project.default_branch
+        Gitlab::Git.branch_name(change[:ref]) == container.default_branch
       end
     end
 
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index fcebcb463cd..26440e6f82d 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -7,19 +7,21 @@ module Gitlab
     PROJECT = RepoType.new(
       name: :project,
       access_checker_class: Gitlab::GitAccess,
-      repository_resolver: -> (project) { project.repository }
+      repository_resolver: -> (project) { project&.repository }
     ).freeze
     WIKI = RepoType.new(
       name: :wiki,
       access_checker_class: Gitlab::GitAccessWiki,
-      repository_resolver: -> (project) { project.wiki.repository },
+      repository_resolver: -> (project) { project&.wiki&.repository },
       suffix: :wiki
     ).freeze
     SNIPPET = RepoType.new(
       name: :snippet,
       access_checker_class: Gitlab::GitAccessSnippet,
-      repository_resolver: -> (snippet) { snippet.repository },
-      container_resolver: -> (id) { Snippet.find_by_id(id) }
+      repository_resolver: -> (snippet) { snippet&.repository },
+      container_resolver: -> (id) { Snippet.find_by_id(id) },
+      project_resolver: -> (snippet) { snippet&.project },
+      guest_read_ability: :read_snippet
     ).freeze
 
     TYPES = {
@@ -42,7 +44,7 @@ module Gitlab
 
       container = type.fetch_container!(gl_repository)
 
-      [container, type]
+      [container, type.project_for(container), type]
     end
 
     def self.default_type
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 9663fd7de8f..052ce578881 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -7,6 +7,8 @@ module Gitlab
                   :access_checker_class,
                   :repository_resolver,
                   :container_resolver,
+                  :project_resolver,
+                  :guest_read_ability,
                   :suffix
 
       def initialize(
@@ -14,11 +16,15 @@ module Gitlab
         access_checker_class:,
         repository_resolver:,
         container_resolver: default_container_resolver,
+        project_resolver: nil,
+        guest_read_ability: :download_code,
         suffix: nil)
         @name = name
         @access_checker_class = access_checker_class
         @repository_resolver = repository_resolver
         @container_resolver = container_resolver
+        @project_resolver = project_resolver
+        @guest_read_ability = guest_read_ability
         @suffix = suffix
       end
 
@@ -59,8 +65,18 @@ module Gitlab
         repository_resolver.call(container)
       end
 
+      def project_for(container)
+        return container unless project_resolver
+
+        project_resolver.call(container)
+      end
+
       def valid?(repository_path)
-        repository_path.end_with?(path_suffix)
+        repository_path.end_with?(path_suffix) &&
+        (
+          !snippet? ||
+          repository_path.match?(Gitlab::PathRegex.full_snippets_repository_path_regex)
+        )
       end
 
       private
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 9606e3e134c..db094bfe973 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -237,8 +237,32 @@ module Gitlab
       }x
     end
 
+    def full_snippets_repository_path_regex
+      %r{\A(#{personal_snippet_repository_path_regex}|#{project_snippet_repository_path_regex})\z}
+    end
+
+    def personal_and_project_snippets_path_regex
+      %r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}}
+    end
+
     private
 
+    def personal_snippet_path_regex
+      /snippets/
+    end
+
+    def personal_snippet_repository_path_regex
+      %r{#{personal_snippet_path_regex}/\d+}
+    end
+
+    def project_snippet_path_regex
+      %r{#{full_namespace_route_regex}/#{project_route_regex}/snippets}
+    end
+
+    def project_snippet_repository_path_regex
+      %r{#{project_snippet_path_regex}/\d+}
+    end
+
     def single_line_regexp(regex)
       # Turns a multiline extended regexp into a single line one,
       # because `rake routes` breaks on multiline regexes.
diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb
index e8c749cac14..da74c38d0ec 100644
--- a/lib/gitlab/repo_path.rb
+++ b/lib/gitlab/repo_path.rb
@@ -19,22 +19,33 @@ module Gitlab
 
         # Removing the suffix (.wiki, .design, ...) from the project path
         full_path = repo_path.chomp(type.path_suffix)
-
-        project, was_redirected = find_project(full_path)
+        container, project, was_redirected = find_container(type, full_path)
         redirected_path = repo_path if was_redirected
 
-        # If we found a matching project, then the type was matched, no need to
-        # continue looking.
-        return [project, type, redirected_path] if project
+        return [container, project, type, redirected_path] if container
       end
 
       # When a project did not exist, the parsed repo_type would be empty.
       # In that case, we want to continue with a regular project repository. As we
       # could create the project if the user pushing is allowed to do so.
-      [nil, Gitlab::GlRepository.default_type, nil]
+      [nil, nil, Gitlab::GlRepository.default_type, nil]
+    end
+
+    def self.find_container(type, full_path)
+      if type.snippet?
+        snippet, was_redirected = find_snippet(full_path)
+
+        [snippet, snippet&.project, was_redirected]
+      else
+        project, was_redirected = find_project(full_path)
+
+        [project, project, was_redirected]
+      end
     end
 
     def self.find_project(project_path)
+      return [nil, false] if project_path.blank?
+
       project = Project.find_by_full_path(project_path, follow_redirects: true)
 
       [project, redirected?(project, project_path)]
@@ -43,6 +54,27 @@ module Gitlab
     def self.redirected?(project, project_path)
       project && project.full_path.casecmp(project_path) != 0
     end
+
+    # Snippet_path can be either:
+    # - snippets/1
+    # - h5bp/html5-boilerplate/snippets/53
+    def self.find_snippet(snippet_path)
+      return [nil, false] if snippet_path.blank?
+
+      snippet_id, project_path = extract_snippet_info(snippet_path)
+      project, was_redirected = find_project(project_path)
+
+      [Snippet.find_by_id_and_project(id: snippet_id, project: project), was_redirected]
+    end
+
+    def self.extract_snippet_info(snippet_path)
+      path_segments = snippet_path.split('/')
+      snippet_id = path_segments.pop
+      path_segments.pop # Remove snippets from path
+      project_path = File.join(path_segments)
+
+      [snippet_id, project_path]
+    end
   end
 end
 
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 8696e23cbc7..7da20b49d9d 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -24,7 +24,7 @@ module Gitlab
 
         attrs = {
           GL_ID: Gitlab::GlId.gl_id(user),
-          GL_REPOSITORY: repo_type.identifier_for_container(repository.project),
+          GL_REPOSITORY: repo_type.identifier_for_container(repository.container),
           GL_USERNAME: user&.username,
           ShowAllRefs: show_all_refs,
           Repository: repository.gitaly_repository.to_h,
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 8f34101ea15..e5e2faaa7df 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -17,9 +17,16 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:registry:create'].invoke
 
       backup = Backup::Manager.new(progress)
-      backup.pack
-      backup.cleanup
-      backup.remove_old
+      backup.write_info
+
+      if ENV['SKIP'] && ENV['SKIP'].include?('tar')
+        backup.upload
+      else
+        backup.pack
+        backup.upload
+        backup.cleanup
+        backup.remove_old
+      end
 
       progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need these files to restore a backup.\n" \
@@ -33,7 +40,8 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       backup = Backup::Manager.new(progress)
-      backup.unpack
+      cleanup_required = backup.unpack
+      backup.verify_backup_version
 
       unless backup.skipped?('db')
         begin
@@ -72,7 +80,10 @@ namespace :gitlab do
       Rake::Task['gitlab:shell:setup'].invoke
       Rake::Task['cache:clear'].invoke
 
-      backup.cleanup
+      if cleanup_required
+        backup.cleanup
+      end
+
       puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need to restore these files manually.".color(:red)
       puts "Restore task is done."