Merge branch 'master' into feature/gb/manual-actions-protected-branches-permissions

* master: (274 commits)
  Update VERSION to 9.2.0-pre
  Update CHANGELOG.md for 9.1.0
  Update Auto Deploy documentation
  Disable import URL field in New project form since it's hidden by default
  Remove reference to burndown charts since they don't exist for ce.
  Use master_password for Sentinel
  Refactor Discussions docs
  Start versioning cached markdown fields
  Refactor add_users method for project and group
  Improved the spec Now correctly tests against different forms
  Refactor environments components into vue files - part 3
  Adding animation for all dropdown
  fix placeholder visibility
  submodule_links: handle urls that don't end with .git
  Add help regarding vue resource and where to include it
  Append .json to ajax endpoint to prevent browser to display raw json
  Fixed the preview keyboard shortcut focusing wrong tab
  Fix broken link
  Added new discussions docs
  Started on resolvable discussions docs
  ...

36 files changed, 299 insertions, 160 deletions

diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 66b37fd2bcc..621b9dcecd9 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -62,7 +62,7 @@ module API
       post ":id/repository/commits" do
         authorize! :push_code, user_project
 
-        attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch])
+        attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch])
 
         result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -140,7 +140,7 @@ module API
         commit_params = {
           commit: commit,
           start_branch: params[:branch],
-          target_branch: params[:branch]
+          branch_name: params[:branch]
         }
 
         result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 9919762cd82..64ab6f01eb5 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -18,6 +18,12 @@ module API
       expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
+    class UserActivity < Grape::Entity
+      expose :username
+      expose :last_activity_on
+      expose :last_activity_on, as: :last_activity_at # Back-compat
+    end
+
     class Identity < Grape::Entity
       expose :provider, :extern_uid
     end
@@ -25,6 +31,7 @@ module API
     class UserPublic < User
       expose :last_sign_in_at
       expose :confirmed_at
+      expose :last_activity_on
       expose :email
       expose :color_scheme_id, :projects_limit, :current_sign_in_at
       expose :identities, using: Entities::Identity
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 33fc970dc09..e6ea12c5ab7 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -5,7 +5,7 @@ module API
         {
           file_path: attrs[:file_path],
           start_branch: attrs[:branch],
-          target_branch: attrs[:branch],
+          branch_name: attrs[:branch],
           commit_message: attrs[:commit_message],
           file_content: attrs[:content],
           file_content_encoding: attrs[:encoding],
@@ -130,7 +130,7 @@ module API
         authorize! :push_code, user_project
 
         file_params = declared_params(include_missing: false)
-        result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute
+        result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
 
         if result[:status] != :success
           render_api_error!(result[:message], 400)
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 810e5063996..718f936a1fc 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -60,6 +60,12 @@ module API
       rescue JSON::ParserError
         {}
       end
+
+      def log_user_activity(actor)
+        commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
+
+        ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action])
+      end
     end
   end
 end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 215bc03d0e9..5b48ee8665f 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -40,6 +40,8 @@ module API
         response = { status: access_status.status, message: access_status.message }
 
         if access_status.status
+          log_user_activity(actor)
+
           # Return the repository full path so that gitlab-shell has it when
           # handling ssh commands
           response[:repository_path] =
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 05423c17449..244725bb292 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -35,7 +35,7 @@ module API
         optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
         optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
         optional :labels, type: String, desc: 'Comma-separated list of label names'
-        optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
+        optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
       end
 
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 50842370947..db4b31b55bc 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -11,7 +11,7 @@ module API
         optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
         optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
         optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
-        optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled'
+        optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled'
         optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled'
         optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
         optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
@@ -103,6 +103,7 @@ module API
       end
       post do
         attrs = declared_params(include_missing: false)
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled)
         project = ::Projects::CreateService.new(current_user, attrs).execute
 
         if project.saved?
@@ -205,7 +206,7 @@ module API
         # CE
         at_least_one_of_ce =
           [
-            :builds_enabled,
+            :jobs_enabled,
             :container_registry_enabled,
             :default_branch,
             :description,
@@ -236,6 +237,8 @@ module API
         authorize! :rename_project, user_project if attrs[:name].present?
         authorize! :change_visibility_level, user_project if attrs[:visibility].present?
 
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled)
+
         result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
 
         if result[:status] == :success
diff --git a/lib/api/users.rb b/lib/api/users.rb
index eedc59f8636..46f221f68fe 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -39,10 +39,13 @@ module API
       params do
         # CE
         optional :username, type: String, desc: 'Get a single user with a specific username'
+        optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID'
+        optional :provider, type: String, desc: 'The external provider'
         optional :search, type: String, desc: 'Search for a username'
         optional :active, type: Boolean, default: false, desc: 'Filters only active users'
         optional :external, type: Boolean, default: false, desc: 'Filters only external users'
         optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+        all_or_none_of :extern_uid, :provider
 
         use :pagination
       end
@@ -51,14 +54,17 @@ module API
           render_api_error!("Not authorized.", 403)
         end
 
-        if params[:username].present?
-          users = User.where(username: params[:username])
-        else
-          users = User.all
-          users = users.active if params[:active]
-          users = users.search(params[:search]) if params[:search].present?
-          users = users.blocked if params[:blocked]
-          users = users.external if params[:external] && current_user.admin?
+        authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
+
+        users = User.all
+        users = User.where(username: params[:username]) if params[:username]
+        users = users.active if params[:active]
+        users = users.search(params[:search]) if params[:search].present?
+        users = users.blocked if params[:blocked]
+
+        if current_user.admin?
+          users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider]
+          users = users.external if params[:external]
         end
 
         entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic
@@ -534,6 +540,21 @@ module API
         email.destroy
         current_user.update_secondary_emails!
       end
+
+      desc 'Get a list of user activities'
+      params do
+        optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
+        use :pagination
+      end
+      get "activities" do
+        authenticated_as_admin!
+
+        activities = User.
+          where(User.arel_table[:last_activity_on].gteq(params[:from])).
+          reorder(last_activity_on: :asc)
+
+        present paginate(activities), with: Entities::UserActivity
+      end
     end
   end
 end
diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb
index 3414a2883e5..674de592f0a 100644
--- a/lib/api/v3/commits.rb
+++ b/lib/api/v3/commits.rb
@@ -53,7 +53,7 @@ module API
 
           attrs = declared_params.dup
           branch = attrs.delete(:branch_name)
-          attrs.merge!(branch: branch, start_branch: branch, target_branch: branch)
+          attrs.merge!(start_branch: branch, branch_name: branch)
 
           result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -131,7 +131,7 @@ module API
           commit_params = {
             commit: commit,
             start_branch: params[:branch],
-            target_branch: params[:branch]
+            branch_name: params[:branch]
           }
 
           result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb
index 13542b0c71c..c76acc86504 100644
--- a/lib/api/v3/files.rb
+++ b/lib/api/v3/files.rb
@@ -6,7 +6,7 @@ module API
           {
             file_path: attrs[:file_path],
             start_branch: attrs[:branch],
-            target_branch: attrs[:branch],
+            branch_name: attrs[:branch],
             commit_message: attrs[:commit_message],
             file_content: attrs[:content],
             file_content_encoding: attrs[:encoding],
@@ -123,7 +123,7 @@ module API
           file_params = declared_params(include_missing: false)
           file_params[:branch] = file_params.delete(:branch_name)
 
-          result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute
+          result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
 
           if result[:status] == :success
             status(200)
diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb
index 6b78aa795b4..327ea9449a1 100644
--- a/lib/banzai/filter/issuable_state_filter.rb
+++ b/lib/banzai/filter/issuable_state_filter.rb
@@ -9,12 +9,14 @@ module Banzai
       VISIBLE_STATES = %w(closed merged).freeze
 
       def call
+        return doc unless context[:issuable_state_filter_enabled]
+
         extractor = Banzai::IssuableExtractor.new(project, current_user)
         issuables = extractor.extract([doc])
 
         issuables.each do |node, issuable|
-          if VISIBLE_STATES.include?(issuable.state)
-            node.children.last.content += " [#{issuable.state}]"
+          if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project)
+            node.content += " (#{issuable.state})"
           end
         end
 
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index b2537117558..5325819d828 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -7,14 +7,14 @@ module Banzai
     #
     class PlantumlFilter < HTML::Pipeline::Filter
       def call
-        return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled
+        return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled
 
         plantuml_setup
 
-        doc.css('pre.plantuml').each do |el|
+        doc.css('pre > code[lang="plantuml"]').each do |node|
           img_tag = Nokogiri::HTML::DocumentFragment.parse(
-            Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {}))
-          el.replace img_tag
+            Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {}))
+          node.parent.replace(img_tag)
         end
 
         doc
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index dabf71d6aeb..c2503fa2adc 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -136,7 +136,8 @@ module Banzai
 
         nodes.each_with_object({}) do |node, hash|
           if node.has_attribute?(attribute)
-            hash[node] = objects_by_id[node.attr(attribute).to_i]
+            obj = objects_by_id[node.attr(attribute).to_i]
+            hash[node] = obj if obj
           end
         end
       end
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 74663556cbb..c7801cb5baf 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,7 +1,5 @@
 module Banzai
   module Renderer
-    module_function
-
     # Convert a Markdown String into an HTML-safe String of HTML
     #
     # Note that while the returned HTML will have been sanitized of dangerous
@@ -16,7 +14,7 @@ module Banzai
     # context  - Hash of context options passed to our HTML Pipeline
     #
     # Returns an HTML-safe String
-    def render(text, context = {})
+    def self.render(text, context = {})
       cache_key = context.delete(:cache_key)
       cache_key = full_cache_key(cache_key, context[:pipeline])
 
@@ -35,24 +33,16 @@ module Banzai
     # of HTML. This method is analogous to calling render(object.field), but it
     # can cache the rendered HTML in the object, rather than Redis.
     #
-    # The context to use is learned from the passed-in object by calling
-    # #banzai_render_context(field), and cannot be changed. Use #render, passing
-    # it the field text, if a custom rendering is needed. The generated context
-    # is returned along with the HTML.
-    def render_field(object, field)
-      html_field = object.markdown_cache_field_for(field)
-
-      html = object.__send__(html_field)
-      return html if html.present?
-
-      html = cacheless_render_field(object, field)
-      update_object(object, html_field, html) unless object.new_record? || object.destroyed?
+    # The context to use is managed by the object and cannot be changed.
+    # Use #render, passing it the field text, if a custom rendering is needed.
+    def self.render_field(object, field)
+      object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field)
 
-      html
+      object.cached_html_for(field)
     end
 
     # Same as +render_field+, but without consulting or updating the cache field
-    def cacheless_render_field(object, field, options = {})
+    def self.cacheless_render_field(object, field, options = {})
       text = object.__send__(field)
       context = object.banzai_render_context(field).merge(options)
 
@@ -82,7 +72,7 @@ module Banzai
     #    texts_and_contexts
     #    => [{ text: '### Hello',
     #          context: { cache_key: [note, :note] } }]
-    def cache_collection_render(texts_and_contexts)
+    def self.cache_collection_render(texts_and_contexts)
       items_collection = texts_and_contexts.each_with_index do |item, index|
         context = item[:context]
         cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline])
@@ -111,7 +101,7 @@ module Banzai
       items_collection.map { |item| item[:rendered] }
     end
 
-    def render_result(text, context = {})
+    def self.render_result(text, context = {})
       text = Pipeline[:pre_process].to_html(text, context) if text
 
       Pipeline[context[:pipeline]].call(text, context)
@@ -130,7 +120,7 @@ module Banzai
     #            :user      - User object
     #
     # Returns an HTML-safe String
-    def post_process(html, context)
+    def self.post_process(html, context)
       context = Pipeline[context[:pipeline]].transform_context(context)
 
       pipeline = Pipeline[:post_process]
@@ -141,7 +131,7 @@ module Banzai
       end.html_safe
     end
 
-    def cacheless_render(text, context = {})
+    def self.cacheless_render(text, context = {})
       Gitlab::Metrics.measure(:banzai_cacheless_render) do
         result = render_result(text, context)
 
@@ -154,7 +144,7 @@ module Banzai
       end
     end
 
-    def full_cache_key(cache_key, pipeline_name)
+    def self.full_cache_key(cache_key, pipeline_name)
       return unless cache_key
       ["banzai", *cache_key, pipeline_name || :full]
     end
@@ -162,13 +152,14 @@ module Banzai
     # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key.
     # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key
     # method.
-    def full_cache_multi_key(cache_key, pipeline_name)
+    def self.full_cache_multi_key(cache_key, pipeline_name)
       return unless cache_key
       Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name))
     end
 
-    def update_object(object, html_field, html)
-      object.update_column(html_field, html)
+    # GitLab EE needs to disable updates on GET requests in Geo
+    def self.update_object?(object)
+      true
     end
   end
 end
diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index 1020452480a..b439b0ee29b 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -172,7 +172,7 @@ module Ci
         close_open_tags()
 
         OpenStruct.new(
-          html: @out,
+          html: @out.force_encoding(Encoding.default_external),
           state: state,
           append: append,
           truncated: truncated,
diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb
index a4b5f2aba6c..61849a40383 100644
--- a/lib/container_registry/path.rb
+++ b/lib/container_registry/path.rb
@@ -15,7 +15,7 @@ module ContainerRegistry
     LEVELS_SUPPORTED = 3
 
     def initialize(path)
-      @path = path
+      @path = path.to_s.downcase
     end
 
     def valid?
@@ -25,7 +25,7 @@ module ContainerRegistry
     end
 
     def components
-      @components ||= @path.to_s.split('/')
+      @components ||= @path.split('/')
     end
 
     def nodes
@@ -48,7 +48,7 @@ module ContainerRegistry
     end
 
     def root_repository?
-      @path == repository_project.full_path
+      @path == project_path
     end
 
     def repository_project
@@ -60,7 +60,13 @@ module ContainerRegistry
     def repository_name
       return unless has_project?
 
-      @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?))
+      @path.remove(%r(^#{Regexp.escape(project_path)}/?))
+    end
+
+    def project_path
+      return unless has_project?
+
+      repository_project.full_path.downcase
     end
 
     def to_s
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index 41dcf846fed..fa462cbe095 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -4,7 +4,7 @@ module Gitlab
       # This was inspired from: http://stackoverflow.com/a/10219411/1520132
       class Stream
         BUFFER_SIZE = 4096
-        LIMIT_SIZE = 50.kilobytes
+        LIMIT_SIZE = 500.kilobytes
 
         attr_reader :stream
 
@@ -14,6 +14,7 @@ module Gitlab
 
         def initialize
           @stream = yield
+          @stream&.binmode
         end
 
         def valid?
@@ -25,11 +26,10 @@ module Gitlab
         end
 
         def limit(last_bytes = LIMIT_SIZE)
-          stream_size = size
-          if stream_size < last_bytes
-            last_bytes = stream_size
+          if last_bytes < size
+            stream.seek(-last_bytes, IO::SEEK_END)
+            stream.readline
           end
-          stream.seek(-last_bytes, IO::SEEK_END)
         end
 
         def append(data, offset)
@@ -52,7 +52,7 @@ module Gitlab
             read_last_lines(last_lines)
           else
             stream.read
-          end
+          end.force_encoding(Encoding.default_external)
         end
 
         def html_with_state(state = nil)
@@ -61,8 +61,8 @@ module Gitlab
 
         def html(last_lines: nil)
           text = raw(last_lines: last_lines)
-          stream = StringIO.new(text)
-          ::Ci::Ansi2html.convert(stream).html
+          buffer = StringIO.new(text)
+          ::Ci::Ansi2html.convert(buffer).html
         end
 
         def extract_coverage(regex)
@@ -114,7 +114,6 @@ module Gitlab
           end
 
           chunks.join.lines.last(last_lines).join
-            .force_encoding(Encoding.default_external)
         end
       end
     end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 0e22f2189ee..c66b0435f3a 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -7,6 +7,8 @@ module Gitlab
       class CreateNoteHandler < BaseHandler
         include ReplyProcessing
 
+        delegate :project, to: :sent_notification, allow_nil: true
+
         def can_handle?
           mail_key =~ /\A\w+\z/
         end
@@ -32,10 +34,6 @@ module Gitlab
           sent_notification.recipient
         end
 
-        def project
-          sent_notification.project
-        end
-
         def sent_notification
           @sent_notification ||= SentNotification.for(mail_key)
         end
diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb
index 97d7a8d65ff..df491f060bf 100644
--- a/lib/gitlab/email/handler/unsubscribe_handler.rb
+++ b/lib/gitlab/email/handler/unsubscribe_handler.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Email
     module Handler
       class UnsubscribeHandler < BaseHandler
+        delegate :project, to: :sent_notification, allow_nil: true
+
         def can_handle?
           mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/
         end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index ec0529b5a4b..bb4fdd1f1f4 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -32,6 +32,10 @@ module Gitlab
 
         raise UnknownIncomingEmail unless handler
 
+        Gitlab::Metrics.add_event(:receive_email,
+                                  project: handler.try(:project),
+                                  handler: handler.class.name)
+
         handler.execute
       end
 
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index e56eb0d3beb..98fd4e78126 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -8,7 +8,7 @@ module Gitlab
       # the user. We load as much as we can for encoding detection
       # (Linguist) and LFS pointer parsing. All other cases where we need full
       # blob data should use load_all_data!.
-      MAX_DATA_DISPLAY_SIZE = 10485760
+      MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
       attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
 
@@ -153,7 +153,7 @@ module Gitlab
       def lfs_size
         if has_lfs_version_key?
           size = data.match(/(?<=size )([0-9]+)/)
-          return size[1] if size
+          return size[1].to_i if size
         end
 
         nil
diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb
index e57d228e688..f918074cb14 100644
--- a/lib/gitlab/git/encoding_helper.rb
+++ b/lib/gitlab/git/encoding_helper.rb
@@ -40,7 +40,13 @@ module Gitlab
       def encode_utf8(message)
         detect = CharlockHolmes::EncodingDetector.detect(message)
         if detect
-          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+          begin
+            CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+          rescue ArgumentError => e
+            Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
+
+            ''
+          end
         else
           clean(message)
         end
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index af1744c9c46..1add037fa5f 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -1,8 +1,12 @@
 module Gitlab
   module Git
     class Index
+      IndexError = Class.new(StandardError)
+
       DEFAULT_MODE = 0o100644
 
+      ACTIONS = %w(create create_dir update move delete).freeze
+
       attr_reader :repository, :raw_index
 
       def initialize(repository)
@@ -23,9 +27,8 @@ module Gitlab
       def create(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        if file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists")
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         add_blob(options)
@@ -34,13 +37,12 @@ module Gitlab
       def create_dir(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        if file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file")
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         if dir_exists?(options[:file_path])
-          raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists")
+          raise IndexError, "A directory with this name already exists"
         end
 
         options = options.dup
@@ -55,7 +57,7 @@ module Gitlab
 
         file_entry = get(options[:file_path])
         unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+          raise IndexError, "A file with this name doesn't exist"
         end
 
         add_blob(options, mode: file_entry[:mode])
@@ -66,7 +68,11 @@ module Gitlab
 
         file_entry = get(options[:previous_path])
         unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+          raise IndexError, "A file with this name doesn't exist"
+        end
+
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
         end
 
         raw_index.remove(options[:previous_path])
@@ -77,9 +83,8 @@ module Gitlab
       def delete(options)
         options = normalize_options(options)
 
-        file_entry = get(options[:file_path])
-        unless file_entry
-          raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist")
+        unless get(options[:file_path])
+          raise IndexError, "A file with this name doesn't exist"
         end
 
         raw_index.remove(options[:file_path])
@@ -95,10 +100,20 @@ module Gitlab
       end
 
       def normalize_path(path)
+        unless path
+          raise IndexError, "You must provide a file path"
+        end
+
         pathname = Gitlab::Git::PathHelper.normalize_path(path.dup)
 
-        if pathname.each_filename.include?('..')
-          raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path')
+        pathname.each_filename do |segment|
+          if segment == '..'
+            raise IndexError, 'Path cannot include directory traversal'
+          end
+
+          unless segment =~ Gitlab::Regex.file_name_regex
+            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
+          end
         end
 
         pathname.to_s
@@ -106,6 +121,10 @@ module Gitlab
 
       def add_blob(options, mode: nil)
         content = options[:content]
+        unless content
+          raise IndexError, "You must provide content"
+        end
+
         content = Base64.decode64(content) if options[:encoding] == 'base64'
 
         detect = CharlockHolmes::EncodingDetector.new.detect(content)
@@ -119,7 +138,7 @@ module Gitlab
 
         raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE)
       rescue Rugged::IndexError => e
-        raise Gitlab::Git::Repository::InvalidBlobName.new(e.message)
+        raise IndexError, e.message
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index bcdf1b1faa8..c69676a1dac 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -15,7 +15,7 @@ module Gitlab
         end
 
         unless URI(address).scheme.in?(%w(tcp unix))
-          raise "Unsupported Gitaly address: #{address.inspect}"
+          raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'"
         end
 
         @addresses[name] = address
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index dda371e6554..49285e35251 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -1,6 +1,11 @@
 module Gitlab
   module MarkupHelper
-    module_function
+    extend self
+
+    MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze
+    ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze
+    OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze
+    EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
 
     # Public: Determines if a given filename is compatible with GitHub::Markup.
     #
@@ -8,10 +13,7 @@ module Gitlab
     #
     # Returns boolean
     def markup?(filename)
-      gitlab_markdown?(filename) ||
-        asciidoc?(filename) ||
-        filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki
-                                        .mediawiki .rst))
+      EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if a given filename is compatible with
@@ -21,7 +23,7 @@ module Gitlab
     #
     # Returns boolean
     def gitlab_markdown?(filename)
-      filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown))
+      MARKDOWN_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename has AsciiDoc extension.
@@ -30,7 +32,7 @@ module Gitlab
     #
     # Returns boolean
     def asciidoc?(filename)
-      filename.downcase.end_with?(*%w(.adoc .ad .asciidoc))
+      ASCIIDOC_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename is plain text.
@@ -39,12 +41,17 @@ module Gitlab
     #
     # Returns boolean
     def plain?(filename)
-      filename.downcase.end_with?('.txt') ||
-        filename.casecmp('readme').zero?
+      extension(filename) == 'txt' || filename.casecmp('readme').zero?
     end
 
     def previewable?(filename)
       markup?(filename)
     end
+
+    private
+
+    def extension(filename)
+      File.extname(filename).downcase.delete('.')
+    end
   end
 end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 857e0abf710..c6dfa4ad9bd 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -138,6 +138,11 @@ module Gitlab
       @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
     end
 
+    # Allow access from other metrics related middlewares
+    def self.current_transaction
+      Transaction.current
+    end
+
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
@@ -149,10 +154,5 @@ module Gitlab
           new(udp: { host: host, port: port })
       end
     end
-
-    # Allow access from other metrics related middlewares
-    def self.current_transaction
-      Transaction.current
-    end
   end
 end
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index f98481c6d3a..6e42d8941fb 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -148,7 +148,7 @@ module Gitlab
 
       def build_new_user
         user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true)
-        Users::CreateService.new(nil, user_params).build
+        Users::BuildService.new(nil, user_params).execute
       end
 
       def user_attributes
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index e599dd4a656..08b061d5e31 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -73,22 +73,6 @@ module Gitlab
       "can contain only letters, digits, '_', '-', '@', '+' and '.'."
     end
 
-    def file_path_regex
-      @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze
-    end
-
-    def file_path_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'."
-    end
-
-    def directory_traversal_regex
-      @directory_traversal_regex ||= /\.{2}/.freeze
-    end
-
-    def directory_traversal_regex_message
-      "cannot include directory traversal."
-    end
-
     def archive_formats_regex
       #                           |zip|tar|    tar.gz    |         tar.bz2         |
       @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
new file mode 100644
index 00000000000..6aca6db3123
--- /dev/null
+++ b/lib/gitlab/usage_data.rb
@@ -0,0 +1,65 @@
+module Gitlab
+  class UsageData
+    include Gitlab::CurrentSettings
+
+    class << self
+      def data(force_refresh: false)
+        Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
+      end
+
+      def uncached_data
+        license_usage_data.merge(system_usage_data)
+      end
+
+      def to_json(force_refresh: false)
+        data(force_refresh: force_refresh).to_json
+      end
+
+      def system_usage_data
+        {
+          counts: {
+            boards: Board.count,
+            ci_builds: ::Ci::Build.count,
+            ci_pipelines: ::Ci::Pipeline.count,
+            ci_runners: ::Ci::Runner.count,
+            ci_triggers: ::Ci::Trigger.count,
+            deploy_keys: DeployKey.count,
+            deployments: Deployment.count,
+            environments: Environment.count,
+            groups: Group.count,
+            issues: Issue.count,
+            keys: Key.count,
+            labels: Label.count,
+            lfs_objects: LfsObject.count,
+            merge_requests: MergeRequest.count,
+            milestones: Milestone.count,
+            notes: Note.count,
+            pages_domains: PagesDomain.count,
+            projects: Project.count,
+            projects_prometheus_active: PrometheusService.active.count,
+            protected_branches: ProtectedBranch.count,
+            releases: Release.count,
+            services: Service.where(active: true).count,
+            snippets: Snippet.count,
+            todos: Todo.count,
+            uploads: Upload.count,
+            web_hooks: WebHook.count
+          }
+        }
+      end
+
+      def license_usage_data
+        usage_data = {
+          uuid: current_application_settings.uuid,
+          version: Gitlab::VERSION,
+          active_user_count: User.active.count,
+          recorded_at: Time.now,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          edition: 'CE'
+        }
+
+        usage_data
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
new file mode 100644
index 00000000000..eb36ab9fded
--- /dev/null
+++ b/lib/gitlab/user_activities.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  class UserActivities
+    include Enumerable
+
+    KEY = 'users:activities'.freeze
+    BATCH_SIZE = 500
+
+    def self.record(key, time = Time.now)
+      Gitlab::Redis.with do |redis|
+        redis.hset(KEY, key, time.to_i)
+      end
+    end
+
+    def delete(*keys)
+      Gitlab::Redis.with do |redis|
+        redis.hdel(KEY, keys)
+      end
+    end
+
+    def each
+      cursor = 0
+      loop do
+        cursor, pairs =
+          Gitlab::Redis.with do |redis|
+            redis.hscan(KEY, cursor, count: BATCH_SIZE)
+          end
+
+        Hash[pairs].each { |pair| yield pair }
+
+        break if cursor == '0'
+      end
+    end
+  end
+end
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index d55923673b1..125a3d560d6 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -21,12 +21,7 @@ namespace :cache do
       end
     end
 
-    desc "GitLab | Clear database cache (in the background)"
-    task db: :environment do
-      ClearDatabaseCacheWorker.perform_async
-    end
-
-    task all: [:db, :redis]
+    task all: [:redis]
   end
 
   task clear: 'cache:clear:redis'
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 9f6cfe3957c..8079c6e416c 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -7,10 +7,10 @@ namespace :gitlab do
         abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
       end
 
-      tag = "v#{Gitlab::GitalyClient.expected_server_version}"
+      version = Gitlab::GitalyClient.expected_server_version
       repo = 'https://gitlab.com/gitlab-org/gitaly.git'
 
-      checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir)
 
       _, status = Gitlab::Popen.popen(%w[which gmake])
       command = status.zero? ? 'gmake' : 'make'
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index dd2fda54e62..95687066819 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -1,19 +1,18 @@
 namespace :gitlab do
   namespace :shell do
     desc "GitLab | Install or upgrade gitlab-shell"
-    task :install, [:tag, :repo] => :environment do |t, args|
+    task :install, [:repo] => :environment do |t, args|
       warn_user_is_not_gitlab
 
       default_version = Gitlab::Shell.version_required
-      default_version_tag = "v#{default_version}"
-      args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
+      args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
 
       gitlab_url = Gitlab.config.gitlab.url
       # gitlab-shell requires a / at the end of the url
       gitlab_url += '/' unless gitlab_url.end_with?('/')
       target_dir = Gitlab.config.gitlab_shell.path
 
-      checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir)
+      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
 
       # Make sure we're on the right tag
       Dir.chdir(target_dir) do
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index cdba2262bc2..e3c9d3b491c 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -147,41 +147,30 @@ module Gitlab
       Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
     end
 
-    def checkout_or_clone_tag(tag:, repo:, target_dir:)
-      if Dir.exist?(target_dir)
-        checkout_tag(tag, target_dir)
-      else
-        clone_repo(repo, target_dir)
-      end
+    def checkout_or_clone_version(version:, repo:, target_dir:)
+      version =
+        if version.starts_with?("=")
+          version.sub(/\A=/, '') # tag or branch
+        else
+          "v#{version}" # tag
+        end
 
-      reset_to_tag(tag, target_dir)
+      clone_repo(repo, target_dir) unless Dir.exist?(target_dir)
+      checkout_version(version, target_dir)
+      reset_to_version(version, target_dir)
     end
 
     def clone_repo(repo, target_dir)
       run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
     end
 
-    def checkout_tag(tag, target_dir)
-      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet])
-      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}])
+    def checkout_version(version, target_dir)
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet])
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}])
     end
 
-    def reset_to_tag(tag_wanted, target_dir)
-      tag =
-        begin
-          # First try to checkout without fetching
-          # to avoid stalling tests if the Internet is down.
-          run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}])
-        rescue Gitlab::TaskFailedError
-          run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin])
-          run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}])
-        end
-
-      if tag
-        run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}])
-      else
-        raise Gitlab::TaskFailedError
-      end
+    def reset_to_version(version, target_dir)
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}])
     end
   end
 end
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index baea94bf8ca..a00b02188cf 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -7,10 +7,10 @@ namespace :gitlab do
         abort %(Please specify the directory where you want to install gitlab-workhorse:\n  rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]")
       end
 
-      tag = "v#{Gitlab::Workhorse.version}"
+      version = Gitlab::Workhorse.version
       repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git'
 
-      checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir)
+      checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir)
 
       _, status = Gitlab::Popen.popen(%w[which gmake])
       command = status.zero? ? 'gmake' : 'make'
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 15131fbf755..a9dad6a1bf0 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -52,7 +52,6 @@ class NewImporter < ::Gitlab::GithubImport::Importer
       project.repository.add_remote(project.import_type, project.import_url)
       project.repository.set_remote_as_mirror(project.import_type)
       project.repository.fetch_remote(project.import_type, forced: true)
-      project.repository.remove_remote(project.import_type)
     rescue => e
       # Expire cache to prevent scenarios such as:
       # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true