Merge branch 'master' into 'trigger-source'

# Conflicts:
#   db/schema.rb

19 files changed, 249 insertions, 64 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index bbdd2039f43..7ae2f3cad40 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -94,6 +94,7 @@ module API
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
+    mount ::API::Features
     mount ::API::Files
     mount ::API::Groups
     mount ::API::Internal
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index e10bd230ae2..fc8183a62c1 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -753,6 +753,28 @@ module API
       expose :impersonation
     end
 
+    class FeatureGate < Grape::Entity
+      expose :key
+      expose :value
+    end
+
+    class Feature < Grape::Entity
+      expose :name
+      expose :state
+      expose :gates, using: FeatureGate do |model|
+        model.gates.map do |gate|
+          value = model.gate_values[gate.key]
+
+          # By default all gate values are populated. Only show relevant ones.
+          if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+            next
+          end
+
+          { key: gate.key, value: value }
+        end.compact
+      end
+    end
+
     module JobRequest
       class JobInfo < Grape::Entity
         expose :name, :stage
diff --git a/lib/api/features.rb b/lib/api/features.rb
new file mode 100644
index 00000000000..cff0ba2ddff
--- /dev/null
+++ b/lib/api/features.rb
@@ -0,0 +1,36 @@
+module API
+  class Features < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :features do
+      desc 'Get a list of all features' do
+        success Entities::Feature
+      end
+      get do
+        features = Feature.all
+
+        present features, with: Entities::Feature, current_user: current_user
+      end
+
+      desc 'Set the gate value for the given feature' do
+        success Entities::Feature
+      end
+      params do
+        requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+      end
+      post ':name' do
+        feature = Feature.get(params[:name])
+
+        if %w(0 false).include?(params[:value])
+          feature.disable
+        elsif params[:value] == 'true'
+          feature.enable
+        else
+          feature.enable_percentage_of_time(params[:value].to_i)
+        end
+
+        present feature, with: Entities::Feature, current_user: current_user
+      end
+    end
+  end
+end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index ee85b777aff..e14a988a153 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -151,8 +151,8 @@ module API
       end
       get ":id/projects" do
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
-        projects = filter_projects(projects)
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
         present paginate(projects), with: entity, current_user: current_user
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 226a7ddd50e..d61450f8258 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -256,31 +256,21 @@ module API
 
     # project helpers
 
-    def filter_projects(projects)
-      if params[:membership]
-        projects = projects.merge(current_user.authorized_projects)
-      end
-
-      if params[:owned]
-        projects = projects.merge(current_user.owned_projects)
-      end
-
-      if params[:starred]
-        projects = projects.merge(current_user.starred_projects)
-      end
-
-      if params[:search].present?
-        projects = projects.search(params[:search])
-      end
-
-      if params[:visibility].present?
-        projects = projects.search_by_visibility(params[:visibility])
-      end
-
-      projects = projects.where(archived: params[:archived])
+    def reorder_projects(projects)
       projects.reorder(params[:order_by] => params[:sort])
     end
 
+    def project_finder_params
+      finder_params = {}
+      finder_params[:owned] = true if params[:owned].present?
+      finder_params[:non_public] = true if params[:membership].present?
+      finder_params[:starred] = true if params[:starred].present?
+      finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
+      finder_params[:archived] = params[:archived]
+      finder_params[:search] = params[:search] if params[:search]
+      finder_params
+    end
+
     # file helpers
 
     def uploaded_file(field, uploads_path)
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index d4fe5c023bf..d00d4fe1737 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -21,6 +21,7 @@ module API
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
         optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
         optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+        optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
       end
 
       params :optional_params do
@@ -67,20 +68,19 @@ module API
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
 
-        def present_projects(projects, options = {})
+        def present_projects(options = {})
+          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+          projects = reorder_projects(projects)
+          projects = projects.with_statistics if params[:statistics]
+          projects = projects.with_issues_enabled if params[:with_issues_enabled]
+          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
           options = options.reverse_merge(
-            with: Entities::Project,
-            current_user: current_user,
-            simple: params[:simple],
-            with_issues_enabled: params[:with_issues_enabled],
-            with_merge_requests_enabled: params[:with_merge_requests_enabled]
+            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+            statistics: params[:statistics],
+            current_user: current_user
           )
-
-          projects = filter_projects(projects)
-          projects = projects.with_statistics if options[:statistics]
-          projects = projects.with_issues_enabled if options[:with_issues_enabled]
-          projects = projects.with_merge_requests_enabled if options[:with_merge_requests_enabled]
-          options[:with] = Entities::BasicProjectDetails if options[:simple]
+          options[:with] = Entities::BasicProjectDetails if params[:simple]
 
           present paginate(projects), options
         end
@@ -94,8 +94,7 @@ module API
         use :statistics_params
       end
       get do
-        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
+        present_projects
       end
 
       desc 'Create new project' do
@@ -231,6 +230,7 @@ module API
             :request_access_enabled,
             :shared_runners_enabled,
             :snippets_enabled,
+            :tag_list,
             :visibility,
             :wiki_enabled
           ]
diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb
index 0f234d4cdad..d9e76560d03 100644
--- a/lib/api/v3/helpers.rb
+++ b/lib/api/v3/helpers.rb
@@ -14,6 +14,33 @@ module API
         authorize! access_level, merge_request
         merge_request
       end
+
+      # project helpers
+
+      def filter_projects(projects)
+        if params[:membership]
+          projects = projects.merge(current_user.authorized_projects)
+        end
+
+        if params[:owned]
+          projects = projects.merge(current_user.owned_projects)
+        end
+
+        if params[:starred]
+          projects = projects.merge(current_user.starred_projects)
+        end
+
+        if params[:search].present?
+          projects = projects.search(params[:search])
+        end
+
+        if params[:visibility].present?
+          projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility]))
+        end
+
+        projects = projects.where(archived: params[:archived])
+        projects.reorder(params[:order_by] => params[:sort])
+      end
     end
   end
 end
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index 164612cb8dd..896c00b88e7 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -147,7 +147,7 @@ module API
         get '/starred' do
           authenticate!
 
-          present_projects current_user.viewable_starred_projects
+          present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute
         end
 
         desc 'Get all projects for admin user' do
diff --git a/lib/feature.rb b/lib/feature.rb
new file mode 100644
index 00000000000..2e2b343f82c
--- /dev/null
+++ b/lib/feature.rb
@@ -0,0 +1,41 @@
+require 'flipper/adapters/active_record'
+
+class Feature
+  # Classes to override flipper table names
+  class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    # Using `self.table_name` won't work. ActiveRecord bug?
+    superclass.table_name = 'features'
+  end
+
+  class FlipperGate < Flipper::Adapters::ActiveRecord::Gate
+    superclass.table_name = 'feature_gates'
+  end
+
+  class << self
+    def all
+      flipper.features.to_a
+    end
+
+    def get(key)
+      flipper.feature(key)
+    end
+
+    def persisted?(feature)
+      # Flipper creates on-memory features when asked for a not-yet-created one.
+      # If we want to check if a feature has been actually set, we look for it
+      # on the persisted features list.
+      all.map(&:name).include?(feature.name)
+    end
+
+    private
+
+    def flipper
+      @flipper ||= begin
+        adapter = Flipper::Adapters::ActiveRecord.new(
+          feature_class: FlipperFeature, gate_class: FlipperGate)
+
+        Flipper.new(adapter)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 57b533bad99..439ef0ce015 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -12,7 +12,7 @@ module Gitlab
           end
 
           def action_path
-            cancel_namespace_project_build_path(subject.project.namespace,
+            cancel_namespace_project_job_path(subject.project.namespace,
                                                 subject.project,
                                                 subject)
           end
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index 3fec2c5d4db..b173c23fba4 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -8,7 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_build_path(subject.project.namespace,
+            namespace_project_job_path(subject.project.namespace,
                                          subject.project,
                                          subject)
           end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index c6139f1b716..e80f3263794 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 505f80848b2..56303e4cb17 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -16,7 +16,7 @@ module Gitlab
           end
 
           def action_path
-            retry_namespace_project_build_path(subject.project.namespace,
+            retry_namespace_project_job_path(subject.project.namespace,
                                                subject.project,
                                                subject)
           end
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index 0b5199e5483..2778d6f3b52 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index cc285162b44..d137cc1bae6 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -11,7 +11,7 @@ module Gitlab
       USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
                           commit pipelines merge_requests new].freeze
       RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
-      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS)
+      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
       ROUTES = [
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z),
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index b9f1ac144b6..9d6adbdb4ac 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1006,31 +1006,39 @@ module Gitlab
       # Parses the contents of a .gitmodules file and returns a hash of
       # submodule information.
       def parse_gitmodules(commit, content)
-        results = {}
+        modules = {}
 
-        current = ""
-        content.split("\n").each do |txt|
-          if txt =~ /^\s*\[/
-            current = txt.match(/(?<=").*(?=")/)[0]
-            results[current] = {}
-          else
-            next unless results[current]
-            match_data = txt.match(/(\w+)\s*=\s*(.*)/)
-            next unless match_data
-            target = match_data[2].chomp
-            results[current][match_data[1]] = target
+        name = nil
+        content.each_line do |line|
+          case line.strip
+          when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header
+            name = $~[:name]
+            modules[name] = {}
+          when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair
+            key = $~[:key]
+            value = $~[:value].chomp
+
+            next unless name && modules[name]
+
+            modules[name][key] = value
 
-            if match_data[1] == "path"
+            if key == 'path'
               begin
-                results[current]["id"] = blob_content(commit, target)
+                modules[name]['id'] = blob_content(commit, value)
               rescue InvalidBlobName
-                results.delete(current)
+                # The current entry is invalid
+                modules.delete(name)
+                name = nil
               end
             end
+          when /\A#/ # Comment
+            next
+          else # Invalid line
+            name = nil
           end
         end
 
-        results
+        modules
       end
 
       # Returns true if +commit+ introduced changes to +path+, using commit
@@ -1086,7 +1094,12 @@ module Gitlab
           elsif tmp_entry.nil?
             return nil
           else
-            tmp_entry = rugged.lookup(tmp_entry[:oid])
+            begin
+              tmp_entry = rugged.lookup(tmp_entry[:oid])
+            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+              return nil
+            end
+
             return nil unless tmp_entry.type == :tree
             tmp_entry = tmp_entry[dir]
           end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 72466700c05..2343446bf22 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -2,6 +2,12 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    module MigrationStatus
+      DISABLED = 1
+      OPT_IN = 2
+      OPT_OUT = 3
+    end
+
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
 
     MUTEX = Mutex.new
@@ -46,8 +52,20 @@ module Gitlab
       Gitlab.config.gitaly.enabled
     end
 
-    def self.feature_enabled?(feature)
-      enabled? && ENV["GITALY_#{feature.upcase}"] == '1'
+    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+      return false if !enabled? || status == MigrationStatus::DISABLED
+
+      feature = Feature.get("gitaly_#{feature}")
+
+      # If the feature hasn't been set, turn it on if it's opt-out
+      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+
+      if feature.percentage_of_time_value > 0
+        # Probabilistically enable this feature
+        return Random.rand() * 100 < feature.percentage_of_time_value
+      end
+
+      feature.enabled?
     end
 
     def self.migrate(feature)
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 1c0abc9f7cf..9ff6829cd49 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -80,6 +80,7 @@ module Gitlab
     # By rejecting `badges` the router can _count_ on the fact that `badges` will
     # be preceded by the `namespace/project`.
     PROJECT_WILDCARD_ROUTES = %w[
+      -
       badges
       blame
       blob
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
new file mode 100644
index 00000000000..36d1a8a6f64
--- /dev/null
+++ b/lib/gitlab/routes/legacy_builds.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Routes
+    class LegacyBuilds
+      def initialize(map)
+        @map = map
+      end
+
+      def draw
+        @map.instance_eval do
+          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
+            collection do
+              resources :artifacts, only: [], controller: 'build_artifacts' do
+                collection do
+                  get :latest_succeeded,
+                    path: '*ref_name_and_path',
+                    format: false
+                end
+              end
+            end
+
+            member do
+              get :raw
+            end
+
+            resource :artifacts, only: [], controller: 'build_artifacts' do
+              get :download
+              get :browse, path: 'browse(/*path)', format: false
+              get :file, path: 'file/*path', format: false
+              get :raw, path: 'raw/*path', format: false
+            end
+          end
+        end
+      end
+    end
+  end
+end