Merge remote-tracking branch 'upstream/master' into 30634-protected-pipeline

* upstream/master: (119 commits)
  Speed up operations performed by gitlab-shell
  Change the force flag to a keyword argument
  add image - issue boards - moving card
  copyedit == ee !2296
  Reset @full_path to nil when cache expires
  Replace existing runner links with icons and tooltips, move into btn-group.
  add margin between captcha and register button
  Eagerly create a milestone that is used in a feature spec
  Adjust readme repo width
  Resolve "Issue Board -> "Remove from board" button when viewing an issue gives js error and fails"
  Set force_remove_source_branch default to false.
  Fix rubocop offenses
  Make entrypoint and command keys to be array of strings
  Add issuable-list class to shared mr/issue lists to fix new responsive layout
  New navigation breadcrumbs
  Restore timeago translations in renderTimeago.
  Fix curl example paths (missing the 'files' segment)
  Automatically hide sidebar on smaller screens
  Fix typo in IssuesFinder comment
  Make Project#ensure_repository force create a repo
  ...

25 files changed, 1380 insertions, 91 deletions

diff --git a/lib/api/features.rb b/lib/api/features.rb
index cff0ba2ddff..21745916463 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -2,6 +2,29 @@ module API
   class Features < Grape::API
     before { authenticated_as_admin! }
 
+    helpers do
+      def gate_value(params)
+        case params[:value]
+        when 'true'
+          true
+        when '0', 'false'
+          false
+        else
+          params[:value].to_i
+        end
+      end
+
+      def gate_target(params)
+        if params[:feature_group]
+          Feature.group(params[:feature_group])
+        elsif params[:user]
+          User.find_by_username(params[:user])
+        else
+          gate_value(params)
+        end
+      end
+    end
+
     resource :features do
       desc 'Get a list of all features' do
         success Entities::Feature
@@ -17,16 +40,22 @@ module API
       end
       params do
         requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+        optional :feature_group, type: String, desc: 'A Feature group name'
+        optional :user, type: String, desc: 'A GitLab username'
+        mutually_exclusive :feature_group, :user
       end
       post ':name' do
         feature = Feature.get(params[:name])
+        target = gate_target(params)
+        value = gate_value(params)
 
-        if %w(0 false).include?(params[:value])
-          feature.disable
-        elsif params[:value] == 'true'
-          feature.enable
+        case value
+        when true
+          feature.enable(target)
+        when false
+          feature.disable(target)
         else
-          feature.enable_percentage_of_time(params[:value].to_i)
+          feature.enable_percentage_of_time(value)
         end
 
         present feature, with: Entities::Feature, current_user: current_user
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index c5df45b7902..d0bd64b2972 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -1,3 +1,5 @@
+require_dependency 'declarative_policy'
+
 module API
   # Projects API
   class Projects < Grape::API
@@ -396,7 +398,7 @@ module API
         use :pagination
       end
       get ':id/users' do
-        users = user_project.team.users
+        users = DeclarativePolicy.subject_scope { user_project.team.users }
         users = users.search(params[:search]) if params[:search].present?
 
         present paginate(users), with: Entities::UserBasic
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 8bc2dd18bda..7a262dd025c 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -216,12 +216,7 @@ module Banzai
         @references_per_project ||= begin
           refs = Hash.new { |hash, key| hash[key] = Set.new }
 
-          regex =
-            if uses_reference_pattern?
-              Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern)
-            else
-              object_class.link_reference_pattern
-            end
+          regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern)
 
           nodes.each do |node|
             node.to_html.scan(regex) do
@@ -323,14 +318,6 @@ module Banzai
           value
         end
       end
-
-      # There might be special cases like filters
-      # that should ignore reference pattern
-      # eg: IssueReferenceFilter when using a external issues tracker
-      # In those cases this method should be overridden on the filter subclass
-      def uses_reference_pattern?
-        true
-      end
     end
   end
 end
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index dce4de3ceaf..53a229256a5 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -3,6 +3,8 @@ module Banzai
     # HTML filter that replaces external issue tracker references with links.
     # References are ignored if the project doesn't use an external issue
     # tracker.
+    #
+    # This filter does not support cross-project references.
     class ExternalIssueReferenceFilter < ReferenceFilter
       self.reference_type = :external_issue
 
@@ -87,7 +89,7 @@ module Banzai
       end
 
       def issue_reference_pattern
-        external_issues_cached(:issue_reference_pattern)
+        external_issues_cached(:external_issue_reference_pattern)
       end
 
       private
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 044d18ff824..ba1a5ac84b3 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -15,10 +15,6 @@ module Banzai
         Issue
       end
 
-      def uses_reference_pattern?
-        context[:project].default_issues_tracker?
-      end
-
       def find_object(project, iid)
         issues_per_project[project][iid]
       end
@@ -38,13 +34,7 @@ module Banzai
 
           projects_per_reference.each do |path, project|
             issue_ids = references_per_project[path]
-
-            issues =
-              if project.default_issues_tracker?
-                project.issues.where(iid: issue_ids.to_a)
-              else
-                issue_ids.map { |id| ExternalIssue.new(id, project) }
-              end
+            issues = project.issues.where(iid: issue_ids.to_a)
 
             issues.each do |issue|
               hash[project][issue.iid.to_i] = issue
@@ -55,26 +45,6 @@ module Banzai
         end
       end
 
-      def object_link_title(object)
-        if object.is_a?(ExternalIssue)
-          "Issue in #{object.project.external_issue_tracker.title}"
-        else
-          super
-        end
-      end
-
-      def data_attributes_for(text, project, object, link: false)
-        if object.is_a?(ExternalIssue)
-          data_attribute(
-            project: project.id,
-            external_issue: object.id,
-            reference_type: ExternalIssueReferenceFilter.reference_type
-          )
-        else
-          super
-        end
-      end
-
       def projects_relation_for_paths(paths)
         super(paths).includes(:gitlab_issue_tracker_service)
       end
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 9fd4bd68d43..a65bbe23958 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -4,9 +4,6 @@ module Banzai
       self.reference_type = :issue
 
       def nodes_visible_to_user(user, nodes)
-        # It is not possible to check access rights for external issue trackers
-        return nodes if project && project.external_issue_tracker
-
         issues = issues_for_nodes(nodes)
 
         readable_issues = Ability
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
new file mode 100644
index 00000000000..d9959bc1aff
--- /dev/null
+++ b/lib/declarative_policy.rb
@@ -0,0 +1,58 @@
+require_dependency 'declarative_policy/cache'
+require_dependency 'declarative_policy/condition'
+require_dependency 'declarative_policy/dsl'
+require_dependency 'declarative_policy/preferred_scope'
+require_dependency 'declarative_policy/rule'
+require_dependency 'declarative_policy/runner'
+require_dependency 'declarative_policy/step'
+
+require_dependency 'declarative_policy/base'
+
+module DeclarativePolicy
+  class << self
+    def policy_for(user, subject, opts = {})
+      cache = opts[:cache] || {}
+      key = Cache.policy_key(user, subject)
+
+      cache[key] ||= class_for(subject).new(user, subject, opts)
+    end
+
+    def class_for(subject)
+      return GlobalPolicy if subject == :global
+      return NilPolicy if subject.nil?
+
+      subject = find_delegate(subject)
+
+      subject.class.ancestors.each do |klass|
+        next unless klass.name
+
+        begin
+          policy_class = "#{klass.name}Policy".constantize
+
+          # NOTE: the < operator here tests whether policy_class
+          # inherits from Base. We can't use #is_a? because that
+          # tests for *instances*, not *subclasses*.
+          return policy_class if policy_class < Base
+        rescue NameError
+          nil
+        end
+      end
+
+      raise "no policy for #{subject.class.name}"
+    end
+
+    private
+
+    def find_delegate(subject)
+      seen = Set.new
+
+      while subject.respond_to?(:declarative_policy_delegate)
+        raise ArgumentError, "circular delegations" if seen.include?(subject.object_id)
+        seen << subject.object_id
+        subject = subject.declarative_policy_delegate
+      end
+
+      subject
+    end
+  end
+end
diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb
new file mode 100644
index 00000000000..df94cafb6a1
--- /dev/null
+++ b/lib/declarative_policy/base.rb
@@ -0,0 +1,329 @@
+module DeclarativePolicy
+  class Base
+    # A map of ability => list of rules together with :enable
+    # or :prevent actions. Used to look up which rules apply to
+    # a given ability. See Base.ability_map
+    class AbilityMap
+      attr_reader :map
+      def initialize(map = {})
+        @map = map
+      end
+
+      # This merge behavior is different than regular hashes - if both
+      # share a key, the values at that key are concatenated, rather than
+      # overridden.
+      def merge(other)
+        conflict_proc = proc { |key, my_val, other_val| my_val + other_val }
+        AbilityMap.new(@map.merge(other.map, &conflict_proc))
+      end
+
+      def actions(key)
+        @map[key] ||= []
+      end
+
+      def enable(key, rule)
+        actions(key) << [:enable, rule]
+      end
+
+      def prevent(key, rule)
+        actions(key) << [:prevent, rule]
+      end
+    end
+
+    class << self
+      # The `own_ability_map` vs `ability_map` distinction is used so that
+      # the data structure is properly inherited - with subclasses recursively
+      # merging their parent class.
+      #
+      # This pattern is also used for conditions, global_actions, and delegations.
+      def ability_map
+        if self == Base
+          own_ability_map
+        else
+          superclass.ability_map.merge(own_ability_map)
+        end
+      end
+
+      def own_ability_map
+        @own_ability_map ||= AbilityMap.new
+      end
+
+      # an inheritable map of conditions, by name
+      def conditions
+        if self == Base
+          own_conditions
+        else
+          superclass.conditions.merge(own_conditions)
+        end
+      end
+
+      def own_conditions
+        @own_conditions ||= {}
+      end
+
+      # a list of global actions, generated by `prevent_all`. these aren't
+      # stored in `ability_map` because they aren't indexed by a particular
+      # ability.
+      def global_actions
+        if self == Base
+          own_global_actions
+        else
+          superclass.global_actions + own_global_actions
+        end
+      end
+
+      def own_global_actions
+        @own_global_actions ||= []
+      end
+
+      # an inheritable map of delegations, indexed by name (which may be
+      # autogenerated)
+      def delegations
+        if self == Base
+          own_delegations
+        else
+          superclass.delegations.merge(own_delegations)
+        end
+      end
+
+      def own_delegations
+        @own_delegations ||= {}
+      end
+
+      # all the [rule, action] pairs that apply to a particular ability.
+      # we combine the specific ones looked up in ability_map with the global
+      # ones.
+      def configuration_for(ability)
+        ability_map.actions(ability) + global_actions
+      end
+
+      ### declaration methods ###
+
+      def delegate(name = nil, &delegation_block)
+        if name.nil?
+          @delegate_name_counter ||= 0
+          @delegate_name_counter += 1
+          name = :"anonymous_#{@delegate_name_counter}"
+        end
+
+        name = name.to_sym
+
+        if delegation_block.nil?
+          delegation_block = proc { @subject.__send__(name) }
+        end
+
+        own_delegations[name] = delegation_block
+      end
+
+      # Declares a rule, constructed using RuleDsl, and returns
+      # a PolicyDsl which is used for registering the rule with
+      # this class. PolicyDsl will call back into Base.enable_when,
+      # Base.prevent_when, and Base.prevent_all_when.
+      def rule(&b)
+        rule = RuleDsl.new(self).instance_eval(&b)
+        PolicyDsl.new(self, rule)
+      end
+
+      # A hash in which to store calls to `desc` and `with_scope`, etc.
+      def last_options
+        @last_options ||= {}.with_indifferent_access
+      end
+
+      # retrieve and zero out the previously set options (used in .condition)
+      def last_options!
+        last_options.tap { @last_options = nil }
+      end
+
+      # Declare a description for the following condition. Currently unused,
+      # but opens the potential for explaining to users why they were or were
+      # not able to do something.
+      def desc(description)
+        last_options[:description] = description
+      end
+
+      def with_options(opts = {})
+        last_options.merge!(opts)
+      end
+
+      def with_scope(scope)
+        with_options scope: scope
+      end
+
+      def with_score(score)
+        with_options score: score
+      end
+
+      # Declares a condition. It gets stored in `own_conditions`, and generates
+      # a query method based on the condition's name.
+      def condition(name, opts = {}, &value)
+        name = name.to_sym
+
+        opts = last_options!.merge(opts)
+        opts[:context_key] ||= self.name
+
+        condition = Condition.new(name, opts, &value)
+
+        self.own_conditions[name] = condition
+
+        define_method(:"#{name}?") { condition(name).pass? }
+      end
+
+      # These next three methods are mainly called from PolicyDsl,
+      # and are responsible for "inverting" the relationship between
+      # an ability and a rule. We store in `ability_map` a map of
+      # abilities to rules that affect them, together with a
+      # symbol indicating :prevent or :enable.
+      def enable_when(abilities, rule)
+        abilities.each { |a| own_ability_map.enable(a, rule) }
+      end
+
+      def prevent_when(abilities, rule)
+        abilities.each { |a| own_ability_map.prevent(a, rule) }
+      end
+
+      # we store global prevents (from `prevent_all`) separately,
+      # so that they can be combined into every decision made.
+      def prevent_all_when(rule)
+        own_global_actions << [:prevent, rule]
+      end
+    end
+
+    # A policy object contains a specific user and subject on which
+    # to compute abilities. For this reason it's sometimes called
+    # "context" within the framework.
+    #
+    # It also stores a reference to the cache, so it can be used
+    # to cache computations by e.g. ManifestCondition.
+    attr_reader :user, :subject, :cache
+    def initialize(user, subject, opts = {})
+      @user = user
+      @subject = subject
+      @cache = opts[:cache] || {}
+    end
+
+    # helper for checking abilities on this and other subjects
+    # for the current user.
+    def can?(ability, new_subject = :_self)
+      return allowed?(ability) if new_subject == :_self
+
+      policy_for(new_subject).allowed?(ability)
+    end
+
+    # This is the main entry point for permission checks. It constructs
+    # or looks up a Runner for the given ability and asks it if it passes.
+    def allowed?(*abilities)
+      abilities.all? { |a| runner(a).pass? }
+    end
+
+    # The inverse of #allowed?, used mainly in specs.
+    def disallowed?(*abilities)
+      abilities.all? { |a| !runner(a).pass? }
+    end
+
+    # computes the given ability and prints a helpful debugging output
+    # showing which 
+    def debug(ability, *a)
+      runner(ability).debug(*a)
+    end
+
+    desc "Unknown user"
+    condition(:anonymous, scope: :user, score: 0) { @user.nil? }
+
+    desc "By default"
+    condition(:default, scope: :global, score: 0) { true }
+
+    def repr
+      subject_repr =
+        if @subject.respond_to?(:id)
+          "#{@subject.class.name}/#{@subject.id}"
+        else
+          @subject.inspect
+        end
+
+      user_repr =
+        if @user
+          @user.to_reference
+        else
+          "<anonymous>"
+        end
+
+      "(#{user_repr} : #{subject_repr})"
+    end
+
+    def inspect
+      "#<#{self.class.name} #{repr}>"
+    end
+
+    # returns a Runner for the given ability, capable of computing whether
+    # the ability is allowed. Runners are cached on the policy (which itself
+    # is cached on @cache), and caches its result. This is how we perform caching
+    # at the ability level.
+    def runner(ability)
+      ability = ability.to_sym
+      @runners ||= {}
+      @runners[ability] ||=
+        begin
+          delegated_runners = delegated_policies.values.compact.map { |p| p.runner(ability) }
+          own_runner = Runner.new(own_steps(ability))
+          delegated_runners.inject(own_runner, &:merge_runner)
+        end
+    end
+
+    # Helpers for caching. Used by ManifestCondition in performing condition
+    # computation.
+    #
+    # NOTE we can't use ||= here because the value might be the
+    # boolean `false`
+    def cache(key, &b)
+      return @cache[key] if cached?(key)
+      @cache[key] = yield
+    end
+
+    def cached?(key)
+      !@cache[key].nil?
+    end
+
+    # returns a ManifestCondition capable of computing itself. The computation
+    # will use our own @cache.
+    def condition(name)
+      name = name.to_sym
+      @_conditions ||= {}
+      @_conditions[name] ||=
+        begin
+          raise "invalid condition #{name}" unless self.class.conditions.key?(name)
+          ManifestCondition.new(self.class.conditions[name], self)
+        end
+    end
+
+    # used in specs - returns true if there is no possible way for any action
+    # to be allowed, determined only by the global :prevent_all rules.
+    def banned?
+      global_steps = self.class.global_actions.map { |(action, rule)| Step.new(self, rule, action) }
+      !Runner.new(global_steps).pass?
+    end
+
+    # A list of other policies that we've delegated to (see `Base.delegate`)
+    def delegated_policies
+      @delegated_policies ||= self.class.delegations.transform_values do |block|
+        new_subject = instance_eval(&block)
+
+        # never delegate to nil, as that would immediately prevent_all
+        next if new_subject.nil?
+
+        policy_for(new_subject)
+      end
+    end
+
+    def policy_for(other_subject)
+      DeclarativePolicy.policy_for(@user, other_subject, cache: @cache)
+    end
+
+    protected
+
+    # constructs steps that come from this policy and not from any delegations
+    def own_steps(ability)
+      rules = self.class.configuration_for(ability)
+      rules.map { |(action, rule)| Step.new(self, rule, action) }
+    end
+  end
+end
diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb
new file mode 100644
index 00000000000..b8cc60074c7
--- /dev/null
+++ b/lib/declarative_policy/cache.rb
@@ -0,0 +1,32 @@
+module DeclarativePolicy
+  module Cache
+    class << self
+      def user_key(user)
+        return '<anonymous>' if user.nil?
+        id_for(user)
+      end
+
+      def policy_key(user, subject)
+        u = user_key(user)
+        s = subject_key(subject)
+        "/dp/policy/#{u}/#{s}"
+      end
+
+      def subject_key(subject)
+        return '<nil>' if subject.nil?
+        return subject.inspect if subject.is_a?(Symbol)
+        "#{subject.class.name}:#{id_for(subject)}"
+      end
+
+      private
+
+      def id_for(obj)
+        if obj.respond_to?(:id) && obj.id
+          obj.id.to_s
+        else
+          "##{obj.object_id}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb
new file mode 100644
index 00000000000..9d7cf6b9726
--- /dev/null
+++ b/lib/declarative_policy/condition.rb
@@ -0,0 +1,102 @@
+module DeclarativePolicy
+  # A Condition is the data structure that is created by the
+  # `condition` declaration on DeclarativePolicy::Base. It is
+  # more or less just a struct of the data passed to that
+  # declaration. It holds on to the block to be instance_eval'd
+  # on a context (instance of Base) later, via #compute.
+  class Condition
+    attr_reader :name, :description, :scope
+    attr_reader :manual_score
+    attr_reader :context_key
+    def initialize(name, opts = {}, &compute)
+      @name = name
+      @compute = compute
+      @scope = opts.fetch(:scope, :normal)
+      @description = opts.delete(:description)
+      @context_key = opts[:context_key]
+      @manual_score = opts.fetch(:score, nil)
+    end
+
+    def compute(context)
+      !!context.instance_eval(&@compute)
+    end
+
+    def key
+      "#{@context_key}/#{@name}"
+    end
+  end
+
+  # In contrast to a Condition, a ManifestCondition contains
+  # a Condition and a context object, and is capable of calculating
+  # a result itself. This is the return value of Base#condition.
+  class ManifestCondition
+    def initialize(condition, context)
+      @condition = condition
+      @context = context
+    end
+
+    # The main entry point - does this condition pass? We reach into
+    # the context's cache here so that we can share in the global
+    # cache (often RequestStore or similar).
+    def pass?
+      @context.cache(cache_key) { @condition.compute(@context) }
+    end
+
+    # Whether we've already computed this condition.
+    def cached?
+      @context.cached?(cache_key)
+    end
+
+    # This is used to score Rule::Condition. See Rule::Condition#score
+    # and Runner#steps_by_score for how scores are used.
+    #
+    # The number here is intended to represent, abstractly, how
+    # expensive it would be to calculate this condition.
+    #
+    # See #cache_key for info about @condition.scope.
+    def score
+      # If we've been cached, no computation is necessary.
+      return 0 if cached?
+
+      # Use the override from condition(score: ...) if present
+      return @condition.manual_score if @condition.manual_score
+
+      # Global scope rules are cheap due to max cache sharing
+      return 2 if  @condition.scope == :global
+
+      # "Normal" rules can't share caches with any other policies
+      return 16 if @condition.scope == :normal
+
+      # otherwise, we're :user or :subject scope, so it's 4 if
+      # the caller has declared a preference
+      return 4 if @condition.scope == DeclarativePolicy.preferred_scope
+
+      # and 8 for all other :user or :subject scope conditions.
+      8
+    end
+
+    private
+
+    # This method controls the caching for the condition. This is where
+    # the condition(scope: ...) option comes into play. Notice that
+    # depending on the scope, we may cache only by the user or only by
+    # the subject, resulting in sharing across different policy objects.
+    def cache_key
+      case @condition.scope
+      when :normal  then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}"
+      when :user    then "/dp/condition/#{@condition.key}/#{user_key}"
+      when :subject then "/dp/condition/#{@condition.key}/#{subject_key}"
+      when :global  then "/dp/condition/#{@condition.key}"
+      else raise 'invalid scope'
+      end
+    end
+
+    def user_key
+      Cache.user_key(@context.user)
+    end
+
+    def subject_key
+      Cache.subject_key(@context.subject)
+    end
+  end
+end
diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb
new file mode 100644
index 00000000000..b26807a7622
--- /dev/null
+++ b/lib/declarative_policy/dsl.rb
@@ -0,0 +1,103 @@
+module DeclarativePolicy
+  # The DSL evaluation context inside rule { ... } blocks.
+  # Responsible for creating and combining Rule objects.
+  #
+  # See Base.rule
+  class RuleDsl
+    def initialize(context_class)
+      @context_class = context_class
+    end
+
+    def can?(ability)
+      Rule::Ability.new(ability)
+    end
+
+    def all?(*rules)
+      Rule::And.make(rules)
+    end
+
+    def any?(*rules)
+      Rule::Or.make(rules)
+    end
+
+    def none?(*rules)
+      ~Rule::Or.new(rules)
+    end
+
+    def cond(condition)
+      Rule::Condition.new(condition)
+    end
+
+    def delegate(delegate_name, condition)
+      Rule::DelegatedCondition.new(delegate_name, condition)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      if @context_class.delegations.key?(m)
+        DelegateDsl.new(self, m)
+      else
+        cond(m.to_sym)
+      end
+    end
+  end
+
+  # Used when the name of a delegate is mentioned in
+  # the rule DSL.
+  class DelegateDsl
+    def initialize(rule_dsl, delegate_name)
+      @rule_dsl = rule_dsl
+      @delegate_name = delegate_name
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      @rule_dsl.delegate(@delegate_name, m)
+    end
+  end
+
+  # The return value of a rule { ... } declaration.
+  # Can call back to register rules with the containing
+  # Policy class (context_class here). See Base.rule
+  #
+  # Note that the #policy method just performs an #instance_eval,
+  # which is useful for multiple #enable or #prevent callse.
+  #
+  # Also provides a #method_missing proxy to the context
+  # class's class methods, so that helper methods can be
+  # defined and used in a #policy { ... } block.
+  class PolicyDsl
+    def initialize(context_class, rule)
+      @context_class = context_class
+      @rule = rule
+    end
+
+    def policy(&b)
+      instance_eval(&b)
+    end
+
+    def enable(*abilities)
+      @context_class.enable_when(abilities, @rule)
+    end
+
+    def prevent(*abilities)
+      @context_class.prevent_when(abilities, @rule)
+    end
+
+    def prevent_all
+      @context_class.prevent_all_when(@rule)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless @context_class.respond_to?(m)
+
+      @context_class.__send__(m, *a, &b)
+    end
+
+    def respond_to_missing?(m)
+      @context_class.respond_to?(m) || super
+    end
+  end
+end
diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb
new file mode 100644
index 00000000000..b0754098149
--- /dev/null
+++ b/lib/declarative_policy/preferred_scope.rb
@@ -0,0 +1,28 @@
+module DeclarativePolicy
+  PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope"
+
+  class << self
+    def with_preferred_scope(scope, &b)
+      Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY]
+      yield
+    ensure
+      Thread.current[PREFERRED_SCOPE_KEY] = old_scope
+    end
+
+    def preferred_scope
+      Thread.current[PREFERRED_SCOPE_KEY]
+    end
+
+    def user_scope(&b)
+      with_preferred_scope(:user, &b)
+    end
+
+    def subject_scope(&b)
+      with_preferred_scope(:subject, &b)
+    end
+
+    def preferred_scope=(scope)
+      Thread.current[PREFERRED_SCOPE_KEY] = scope
+    end
+  end
+end
diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb
new file mode 100644
index 00000000000..bfcec241489
--- /dev/null
+++ b/lib/declarative_policy/rule.rb
@@ -0,0 +1,301 @@
+module DeclarativePolicy
+  module Rule
+    # A Rule is the object that results from the `rule` declaration,
+    # usually built using the DSL in `RuleDsl`. It is a basic logical
+    # combination of building blocks, and is capable of deciding,
+    # given a context (instance of DeclarativePolicy::Base) whether it
+    # passes or not. Note that this decision doesn't by itself know
+    # how that affects the actual ability decision - for that, a
+    # `Step` is used.
+    class Base
+      def self.make(*a)
+        new(*a).simplify
+      end
+
+      # true or false whether this rule passes.
+      # `context` is a policy - an instance of
+      # DeclarativePolicy::Base.
+      def pass?(context)
+        raise 'abstract'
+      end
+
+      # same as #pass? except refuses to do any I/O,
+      # returning nil if the result is not yet cached.
+      # used for accurately scoring And/Or
+      def cached_pass?(context)
+        raise 'abstract'
+      end
+
+      # abstractly, how long would it take to compute
+      # this rule? lower-scored rules are tried first.
+      def score(context)
+        raise 'abstract'
+      end
+
+      # unwrap double negatives and nested and/or
+      def simplify
+        self
+      end
+
+      # convenience combination methods
+      def or(other)
+        Or.make([self, other])
+      end
+
+      def and(other)
+        And.make([self, other])
+      end
+
+      def negate
+        Not.make(self)
+      end
+
+      alias_method :|, :or
+      alias_method :&, :and
+      alias_method :~@, :negate
+
+      def inspect
+        "#<Rule #{repr}>"
+      end
+    end
+
+    # A rule that checks a condition. This is the
+    # type of rule that results from a basic bareword
+    # in the rule dsl (see RuleDsl#method_missing).
+    class Condition < Base
+      def initialize(name)
+        @name = name
+      end
+
+      # we delegate scoring to the condition. See
+      # ManifestCondition#score.
+      def score(context)
+        context.condition(@name).score
+      end
+
+      # Let the ManifestCondition from the context
+      # decide whether we pass.
+      def pass?(context)
+        context.condition(@name).pass?
+      end
+
+      # returns nil unless it's already cached
+      def cached_pass?(context)
+        condition = context.condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      end
+
+      def description(context)
+        context.class.conditions[@name].description
+      end
+
+      def repr
+        @name.to_s
+      end
+    end
+
+    # A rule constructed from DelegateDsl - using a condition from a
+    # delegated policy.
+    class DelegatedCondition < Base
+      # Internal use only - this is rescued each time it's raised.
+      MissingDelegate = Class.new(StandardError)
+
+      def initialize(delegate_name, name)
+        @delegate_name = delegate_name
+        @name = name
+      end
+
+      def delegated_context(context)
+        policy = context.delegated_policies[@delegate_name]
+        raise MissingDelegate if policy.nil?
+        policy
+      end
+
+      def score(context)
+        delegated_context(context).condition(@name).score
+      rescue MissingDelegate
+        0
+      end
+
+      def cached_pass?(context)
+        condition = delegated_context(context).condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def pass?(context)
+        delegated_context(context).condition(@name).pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def repr
+        "#{@delegate_name}.#{@name}"
+      end
+    end
+
+    # A rule constructed from RuleDsl#can?. Computes a different ability
+    # on the same subject.
+    class Ability < Base
+      attr_reader :ability
+      def initialize(ability)
+        @ability = ability
+      end
+
+      # We ask the ability's runner for a score
+      def score(context)
+        context.runner(@ability).score
+      end
+
+      def pass?(context)
+        context.allowed?(@ability)
+      end
+
+      def cached_pass?(context)
+        runner = context.runner(@ability)
+        return nil unless runner.cached?
+        runner.pass?
+      end
+
+      def description(context)
+        "User can #{@ability.inspect}"
+      end
+
+      def repr
+        "can?(#{@ability.inspect})"
+      end
+    end
+
+    # Logical `and`, containing a list of rules. Only passes
+    # if all of them do.
+    class And < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when And then simplified.rules
+          else [simplified]
+          end
+        end
+
+        And.new(simplified_rules)
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+
+        # note that cached rules will have score 0 anyways.
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def pass?(context)
+        # try to find a cached answer before
+        # checking in order
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.all? { |r| r.pass?(context) }
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return false if passes.any? { |p| p == false }
+        return true if passes.all? { |p| p == true }
+
+        nil
+      end
+
+      def repr
+        "all?(#{rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    # Logical `or`. Mirrors And.
+    class Or < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def pass?(context)
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.any? { |r| r.pass?(context) }
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when Or then simplified.rules
+          else [simplified]
+          end
+        end
+
+        Or.new(simplified_rules)
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return true if passes.any? { |p| p == true }
+        return false if passes.all? { |p| p == false }
+
+        nil
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def repr
+        "any?(#{@rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    class Not < Base
+      attr_reader :rule
+      def initialize(rule)
+        @rule = rule
+      end
+
+      def simplify
+        case @rule
+        when And then Or.new(@rule.rules.map(&:negate)).simplify
+        when Or then And.new(@rule.rules.map(&:negate)).simplify
+        when Not then @rule.rule.simplify
+        else Not.new(@rule.simplify)
+        end
+      end
+
+      def pass?(context)
+        !@rule.pass?(context)
+      end
+
+      def cached_pass?(context)
+        case @rule.cached_pass?(context)
+        when nil then nil
+        when true then false
+        when false then true
+        end
+      end
+
+      def score(context)
+        @rule.score(context)
+      end
+
+      def repr
+        "~#{@rule.repr}"
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
new file mode 100644
index 00000000000..b5c615da4e3
--- /dev/null
+++ b/lib/declarative_policy/runner.rb
@@ -0,0 +1,181 @@
+module DeclarativePolicy
+  class Runner
+    class State
+      def initialize
+        @enabled = false
+        @prevented = false
+      end
+
+      def enable!
+        @enabled = true
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      def prevent!
+        @prevented = true
+      end
+
+      def prevented?
+        @prevented
+      end
+
+      def pass?
+        !prevented? && enabled?
+      end
+    end
+
+    # a Runner contains a list of Steps to be run.
+    attr_reader :steps
+    def initialize(steps)
+      @steps = steps
+    end
+
+    # We make sure only to run any given Runner once,
+    # and just continue to use the resulting @state
+    # that's left behind.
+    def cached?
+      !!@state
+    end
+
+    # used by Rule::Ability. See #steps_by_score
+    def score
+      return 0 if cached?
+      steps.map(&:score).inject(0, :+)
+    end
+
+    def merge_runner(other)
+      Runner.new(@steps + other.steps)
+    end
+
+    # The main entry point, called for making an ability decision.
+    # See #run and DeclarativePolicy::Base#can?
+    def pass?
+      run unless cached?
+
+      @state.pass?
+    end
+
+    # see DeclarativePolicy::Base#debug
+    def debug(out = $stderr)
+      run(out)
+    end
+
+    private
+
+    def flatten_steps!
+      @steps = @steps.flat_map { |s| s.flattened(@steps) }
+    end
+
+    # This method implements the semantic of "one enable and no prevents".
+    # It relies on #steps_by_score for the main loop, and updates @state
+    # with the result of the step.
+    def run(debug = nil)
+      @state = State.new
+
+      steps_by_score do |step, score|
+        passed = nil
+        case step.action
+        when :enable then
+          # we only check :enable actions if they have a chance of
+          # changing the outcome - if no other rule has enabled or
+          # prevented.
+          unless @state.enabled? || @state.prevented?
+            passed = step.pass?
+            @state.enable! if passed
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        when :prevent then
+          # we only check :prevent actions if the state hasn't already
+          # been prevented.
+          unless @state.prevented?
+            passed = step.pass?
+            if passed
+              @state.prevent!
+              return unless debug
+            end
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        else raise "invalid action #{step.action.inspect}"
+        end
+      end
+
+      @state
+    end
+
+    # This is the core spot where all those `#score` methods matter.
+    # It is critcal for performance to run steps in the correct order,
+    # so that we don't compute expensive conditions (potentially n times
+    # if we're called on, say, a large list of users).
+    #
+    # In order to determine the cheapest step to run next, we rely on
+    # Step#score, which returns a numerical rating of how expensive
+    # it would be to calculate - the lower the better. It would be
+    # easy enough to statically sort by these scores, but we can do
+    # a little better - the scores are cache-aware (conditions that
+    # are already in the cache have score 0), which means that running
+    # a step can actually change the scores of other steps.
+    #
+    # So! The way we sort here involves re-scoring at every step. This
+    # is by necessity quadratic, but most of the time the number of steps
+    # will be low. But just in case, if the number of steps exceeds 50,
+    # we print a warning and fall back to a static sort.
+    #
+    # For each step, we yield the step object along with the computed score
+    # for debugging purposes.
+    def steps_by_score(&b)
+      flatten_steps!
+
+      if @steps.size > 50
+        warn "DeclarativePolicy: large number of steps (#{steps.size}), falling back to static sort"
+
+        @steps.map { |s| [s.score, s] }.sort_by { |(score, _)| score }.each do |(score, step)|
+          yield step, score
+        end
+
+        return
+      end
+
+      steps = Set.new(@steps)
+
+      loop do
+        return if steps.empty?
+
+        # if the permission hasn't yet been enabled and we only have
+        # prevent steps left, we short-circuit the state here
+        @state.prevent! if !@state.enabled? && steps.all?(&:prevent?)
+
+        lowest_score = Float::INFINITY
+        next_step = nil
+
+        steps.each do |step|
+          score = step.score
+          if score < lowest_score
+            next_step = step
+            lowest_score = score
+          end
+        end
+
+        steps.delete(next_step)
+
+        yield next_step, lowest_score
+      end
+    end
+
+    # Formatter for debugging output.
+    def inspect_step(step, original_score, passed)
+      symbol =
+        case passed
+        when true then '+'
+        when false then '-'
+        when nil then ' '
+        end
+
+      "#{symbol} [#{original_score.to_i}] #{step.repr}\n"
+    end
+  end
+end
diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb
new file mode 100644
index 00000000000..3469fe9f991
--- /dev/null
+++ b/lib/declarative_policy/step.rb
@@ -0,0 +1,86 @@
+module DeclarativePolicy
+  # This object represents one step in the runtime decision of whether
+  # an ability is allowed. It contains a Rule and a context (instance
+  # of DeclarativePolicy::Base), which contains the user, the subject,
+  # and the cache. It also contains an "action", which is the symbol
+  # :prevent or :enable.
+  class Step
+    attr_reader :context, :rule, :action
+    def initialize(context, rule, action)
+      @context = context
+      @rule = rule
+      @action = action
+    end
+
+    # In the flattening process, duplicate steps may be generated in the
+    # same rule. This allows us to eliminate those (see Runner#steps_by_score
+    # and note its use of a Set)
+    def ==(other)
+      @context == other.context && @rule == other.rule && @action == other.action
+    end
+
+    # In the runner, steps are sorted dynamically by score, so that
+    # we are sure to compute them in close to the optimal order.
+    #
+    # See also Rule#score, ManifestCondition#score, and Runner#steps_by_score.
+    def score
+      # we slightly prefer the preventative actions
+      # since they are more likely to short-circuit
+      case @action
+      when :prevent
+        @rule.score(@context) * (7.0 / 8)
+      when :enable
+        @rule.score(@context)
+      end
+    end
+
+    def with_action(action)
+      Step.new(@context, @rule, action)
+    end
+
+    def enable?
+      @action == :enable
+    end
+
+    def prevent?
+      @action == :prevent
+    end
+
+    # This rather complex method allows us to split rules into parts so that
+    # they can be sorted independently for better optimization
+    def flattened(roots)
+      case @rule
+      when Rule::Or
+        # A single `Or` step is the same as each of its elements as separate steps
+        @rule.rules.flat_map { |r| Step.new(@context, r, @action).flattened(roots) }
+      when Rule::Ability
+        # This looks like a weird micro-optimization but it buys us quite a lot
+        # in some cases. If we depend on an Ability (i.e. a `can?(...)` rule),
+        # and that ability *only* has :enable actions (modulo some actions that
+        # we already have taken care of), then its rules can be safely inlined.
+        steps = @context.runner(@rule.ability).steps.reject { |s| roots.include?(s) }
+
+        if steps.all?(&:enable?)
+          # in the case that we are a :prevent step, each inlined step becomes
+          # an independent :prevent, even though it was an :enable in its initial
+          # context.
+          steps.map! { |s| s.with_action(:prevent) } if prevent?
+
+          steps.flat_map { |s| s.flattened(roots) }
+        else
+          [self]
+        end
+      else
+        [self]
+      end
+    end
+
+    def pass?
+      @rule.pass?(@context)
+    end
+
+    def repr
+      "#{@action} when #{@rule.repr} (#{@context.repr})"
+    end
+  end
+end
diff --git a/lib/feature.rb b/lib/feature.rb
index d3d972564af..363f66ba60e 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -12,6 +12,8 @@ class Feature
   end
 
   class << self
+    delegate :group, to: :flipper
+
     def all
       flipper.features.to_a
     end
@@ -27,16 +29,24 @@ class Feature
       all.map(&:name).include?(feature.name)
     end
 
-    def enabled?(key)
-      get(key).enabled?
+    def enabled?(key, thing = nil)
+      get(key).enabled?(thing)
+    end
+
+    def enable(key, thing = true)
+      get(key).enable(thing)
+    end
+
+    def disable(key, thing = false)
+      get(key).disable(thing)
     end
 
-    def enable(key)
-      get(key).enable
+    def enable_group(key, group)
+      get(key).enable_group(group)
     end
 
-    def disable(key)
-      get(key).disable
+    def disable_group(key, group)
+      get(key).disable_group(group)
     end
 
     def flipper
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
index e4f7cad2b79..45c2b01dd8f 100644
--- a/lib/gitlab/allowable.rb
+++ b/lib/gitlab/allowable.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Allowable
-    def can?(user, action, subject = :global)
-      Ability.allowed?(user, action, subject)
+    def can?(*args)
+      Ability.allowed?(*args)
     end
   end
 end
diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb
index 897dcff8012..6555c589173 100644
--- a/lib/gitlab/ci/config/entry/image.rb
+++ b/lib/gitlab/ci/config/entry/image.rb
@@ -15,7 +15,7 @@ module Gitlab
             validates :config, allowed_keys: ALLOWED_KEYS
 
             validates :name, type: String, presence: true
-            validates :entrypoint, type: String, allow_nil: true
+            validates :entrypoint, array_of_strings: true, allow_nil: true
           end
 
           def hash?
diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb
index b52faf48b58..3e2ebcff31a 100644
--- a/lib/gitlab/ci/config/entry/service.rb
+++ b/lib/gitlab/ci/config/entry/service.rb
@@ -15,8 +15,8 @@ module Gitlab
             validates :config, allowed_keys: ALLOWED_KEYS
 
             validates :name, type: String, presence: true
-            validates :entrypoint, type: String, allow_nil: true
-            validates :command, type: String, allow_nil: true
+            validates :entrypoint, array_of_strings: true, allow_nil: true
+            validates :command, array_of_strings: true, allow_nil: true
             validates :alias, type: String, allow_nil: true
           end
 
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
new file mode 100644
index 00000000000..d9400e04b83
--- /dev/null
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Database
+    BINARY_TYPE = if Gitlab::Database.postgresql?
+                    # PostgreSQL defines its own class with slightly different
+                    # behaviour from the default Binary type.
+                    ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+                  else
+                    ActiveRecord::Type::Binary
+                  end
+
+    # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
+    #
+    # Using ShaAttribute allows you to store SHA1 values as binary while still
+    # using them as if they were stored as string values. This gives you the
+    # ease of use of string values, but without the storage overhead.
+    class ShaAttribute < BINARY_TYPE
+      PACK_FORMAT = 'H*'.freeze
+
+      # Casts binary data to a SHA1 in hexadecimal.
+      def type_cast_from_database(value)
+        value = super
+
+        value ? value.unpack(PACK_FORMAT)[0] : nil
+      end
+
+      # Casts a SHA1 in hexadecimal to the proper binary format.
+      def type_cast_for_database(value)
+        arg = value ? [value].pack(PACK_FORMAT) : nil
+
+        super(arg)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index bd90d24a2ec..5042916343b 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -4,9 +4,10 @@ module Gitlab
       GL_PROTOCOL = 'web'.freeze
       attr_reader :name, :repo_path, :path
 
-      def initialize(name, repo_path)
+      def initialize(name, project)
         @name = name
-        @repo_path = repo_path
+        @project = project
+        @repo_path = project.repository.path
         @path = File.join(repo_path.strip, 'hooks', name)
       end
 
@@ -38,7 +39,8 @@ module Gitlab
         vars = {
           'GL_ID' => gl_id,
           'PWD' => repo_path,
-          'GL_PROTOCOL' => GL_PROTOCOL
+          'GL_PROTOCOL' => GL_PROTOCOL,
+          'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false)
         }
 
         options = {
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 319633656ff..2d1ae6a5925 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -2,11 +2,14 @@
 
 module Gitlab
   module GonHelper
+    include WebpackHelper
+
     def add_gon_variables
       gon.api_version            = 'v4'
       gon.default_avatar_url     = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
       gon.max_file_size          = current_application_settings.max_attachment_size
       gon.asset_host             = ActionController::Base.asset_host
+      gon.webpack_public_path    = webpack_public_path
       gon.relative_url_root      = Gitlab.config.gitlab.relative_url_root
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 22554236c38..0baea092e6a 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -2,6 +2,8 @@ require 'securerandom'
 
 module Gitlab
   class Shell
+    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze
+
     Error = Class.new(StandardError)
 
     KeyAdder = Struct.new(:io) do
@@ -67,8 +69,8 @@ module Gitlab
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     def add_repository(storage, name)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'add-project', storage, "#{name}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path,
+                                 'add-project', storage, "#{name}.git"])
     end
 
     # Import repository
@@ -82,10 +84,9 @@ module Gitlab
     def import_repository(storage, name, url)
       # Timeout should be less than 900 ideally, to prevent the memory killer
       # to silently kill the process without knowing we are timing out here.
-      output, status = Popen.popen([gitlab_shell_projects_path, 'import-project',
-                                    storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"])
-      raise Error, output unless status.zero?
-      true
+      cmd = [gitlab_shell_projects_path, 'import-project',
+             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
+      gitlab_shell_fast_execute_raise_error(cmd)
     end
 
     # Fetch remote for repository
@@ -103,9 +104,7 @@ module Gitlab
       args << '--force' if forced
       args << '--no-tags' if no_tags
 
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
-      true
+      gitlab_shell_fast_execute_raise_error(args)
     end
 
     # Move repository
@@ -117,8 +116,8 @@ module Gitlab
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
     def mv_repository(storage, path, new_path)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
-                                   storage, "#{path}.git", "#{new_path}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
+                                 storage, "#{path}.git", "#{new_path}.git"])
     end
 
     # Fork repository to new namespace
@@ -131,9 +130,9 @@ module Gitlab
     #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
     #
     def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
-                                   forked_from_storage, "#{path}.git", forked_to_storage,
-                                   fork_namespace])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
+                                 forked_from_storage, "#{path}.git", forked_to_storage,
+                                 fork_namespace])
     end
 
     # Remove repository from file system
@@ -145,8 +144,8 @@ module Gitlab
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     def remove_repository(storage, name)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'rm-project', storage, "#{name}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path,
+                                 'rm-project', storage, "#{name}.git"])
     end
 
     # Add new key to gitlab-shell
@@ -155,8 +154,8 @@ module Gitlab
     #   add_key("key-42", "sha-rsa ...")
     #
     def add_key(key_id, key_content)
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'add-key', key_id, self.class.strip_key(key_content)])
+      gitlab_shell_fast_execute([gitlab_shell_keys_path,
+                                 'add-key', key_id, self.class.strip_key(key_content)])
     end
 
     # Batch-add keys to authorized_keys
@@ -175,8 +174,10 @@ module Gitlab
     #   remove_key("key-342", "sha-rsa ...")
     #
     def remove_key(key_id, key_content)
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'rm-key', key_id, key_content])
+      args = [gitlab_shell_keys_path, 'rm-key', key_id]
+      args << key_content if key_content
+
+      gitlab_shell_fast_execute(args)
     end
 
     # Remove all ssh keys from gitlab shell
@@ -185,7 +186,7 @@ module Gitlab
     #   remove_all_keys
     #
     def remove_all_keys
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear'])
+      gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
     end
 
     # Add empty directory for storing repositories
@@ -267,5 +268,31 @@ module Gitlab
     def gitlab_shell_keys_path
       File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
     end
+
+    private
+
+    def gitlab_shell_fast_execute(cmd)
+      output, status = gitlab_shell_fast_execute_helper(cmd)
+
+      return true if status.zero?
+
+      Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
+      false
+    end
+
+    def gitlab_shell_fast_execute_raise_error(cmd)
+      output, status = gitlab_shell_fast_execute_helper(cmd)
+
+      raise Error, output unless status.zero?
+      true
+    end
+
+    def gitlab_shell_fast_execute_helper(cmd)
+      vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)
+
+      # Don't pass along the entire parent environment to prevent gitlab-shell
+      # from wasting I/O by searching through GEM_PATH
+      Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
+    end
   end
 end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 38dc82493cf..f19b325a126 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -20,7 +20,8 @@ module Gitlab
           counts: {
             boards: Board.count,
             ci_builds: ::Ci::Build.count,
-            ci_pipelines: ::Ci::Pipeline.count,
+            ci_internal_pipelines: ::Ci::Pipeline.internal.count,
+            ci_external_pipelines: ::Ci::Pipeline.external.count,
             ci_runners: ::Ci::Runner.count,
             ci_triggers: ::Ci::Trigger.count,
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
index dbfe0941e4d..841fb681435 100644
--- a/lib/gitlab/view/presenter/base.rb
+++ b/lib/gitlab/view/presenter/base.rb
@@ -15,6 +15,11 @@ module Gitlab
           super(user, action, overriden_subject || subject)
         end
 
+        # delegate all #can? queries to the subject
+        def declarative_policy_delegate
+          subject
+        end
+
         class_methods do
           def presenter?
             true