Merge remote-tracking branch 'origin/master' into 21170-cycle-analytics

61 files changed, 1035 insertions, 356 deletions

diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index d02b469dac8..29a97ccbd75 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -20,7 +20,7 @@ module API
 
           access_requesters = paginate(source.requesters.includes(:user))
 
-          present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters
+          present access_requesters.map(&:user), with: Entities::AccessRequester, source: source
         end
 
         # Request access to the group/project
diff --git a/lib/api/api.rb b/lib/api/api.rb
index e14464c1b0d..74ca4728695 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -45,11 +45,13 @@ module API
     mount ::API::Keys
     mount ::API::Labels
     mount ::API::LicenseTemplates
+    mount ::API::Lint
     mount ::API::Members
     mount ::API::MergeRequests
     mount ::API::Milestones
     mount ::API::Namespaces
     mount ::API::Notes
+    mount ::API::NotificationSettings
     mount ::API::Pipelines
     mount ::API::ProjectHooks
     mount ::API::ProjectSnippets
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 5e3c9563703..dfbdd597d29 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -37,7 +37,7 @@ module API
       #   id (required) - The ID of a project
       #   sha (required) - The commit hash
       #   ref (optional) - The ref
-      #   state (required) - The state of the status. Can be: pending, running, success, error or failure
+      #   state (required) - The state of the status. Can be: pending, running, success, failed or canceled
       #   target_url (optional) - The target URL to associate with this status
       #   description (optional) - A short description of the status
       #   name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default"
@@ -46,7 +46,7 @@ module API
       post ':id/statuses/:sha' do
         authorize! :create_commit_status, user_project
         required_attributes! [:state]
-        attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name]
+        attrs = attributes_for_keys [:target_url, :description]
         commit = @project.commit(params[:sha])
         not_found! 'Commit' unless commit
 
@@ -58,36 +58,38 @@ module API
         # the first found branch on that commit
 
         ref = params[:ref]
-        unless ref
-          branches = @project.repository.branch_names_contains(commit.sha)
-          not_found! 'References for commit' if branches.none?
-          ref = branches.first
-        end
+        ref ||= @project.repository.branch_names_contains(commit.sha).first
+        not_found! 'References for commit' unless ref
+
+        name = params[:name] || params[:context] || 'default'
 
         pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
 
-        name = params[:name] || params[:context]
-        status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref])
-        status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user)
-        status.update(attrs)
+        status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
+          project: @project, pipeline: pipeline,
+          user: current_user, name: name, ref: ref)
+        status.attributes = attrs
 
-        case params[:state].to_s
-        when 'running'
-          status.run
-        when 'success'
-          status.success
-        when 'failed'
-          status.drop
-        when 'canceled'
-          status.cancel
-        else
-          status.status = params[:state].to_s
-        end
+        begin
+          case params[:state].to_s
+          when 'pending'
+            status.enqueue!
+          when 'running'
+            status.enqueue
+            status.run!
+          when 'success'
+            status.success!
+          when 'failed'
+            status.drop!
+          when 'canceled'
+            status.cancel!
+          else
+            render_api_error!('invalid state', 400)
+          end
 
-        if status.save
           present status, with: Entities::CommitStatus
-        else
-          render_validation_error!(status)
+        rescue StateMachines::InvalidTransition => e
+          render_api_error!(e.message, 400)
         end
       end
     end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 3faba79415b..92a6f29adb0 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -86,7 +86,8 @@ module API
       expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
 
       expose :created_at, :last_activity_at
-      expose :shared_runners_enabled, :lfs_enabled
+      expose :shared_runners_enabled
+      expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
       expose :namespace
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
@@ -99,30 +100,33 @@ module API
         SharedGroup.represent(project.project_group_links.all, options)
       end
       expose :only_allow_merge_if_build_succeeds
+      expose :request_access_enabled
     end
 
     class Member < UserBasic
       expose :access_level do |user, options|
-        member = options[:member] || options[:members].find { |m| m.user_id == user.id }
+        member = options[:member] || options[:source].members.find_by(user_id: user.id)
         member.access_level
       end
       expose :expires_at do |user, options|
-        member = options[:member] || options[:members].find { |m| m.user_id == user.id }
+        member = options[:member] || options[:source].members.find_by(user_id: user.id)
         member.expires_at
       end
     end
 
     class AccessRequester < UserBasic
       expose :requested_at do |user, options|
-        access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id }
+        access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id)
         access_requester.requested_at
       end
     end
 
     class Group < Grape::Entity
       expose :id, :name, :path, :description, :visibility_level
+      expose :lfs_enabled?, as: :lfs_enabled
       expose :avatar_url
       expose :web_url
+      expose :request_access_enabled
     end
 
     class GroupDetail < Group
@@ -375,7 +379,7 @@ module API
       expose :access_level
       expose :notification_level do |member, options|
         if member.notification_setting
-          NotificationSetting.levels[member.notification_setting.level]
+          ::NotificationSetting.levels[member.notification_setting.level]
         end
       end
     end
@@ -386,6 +390,21 @@ module API
     class GroupAccess < MemberAccess
     end
 
+    class NotificationSetting < Grape::Entity
+      expose :level
+      expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do
+        ::NotificationSetting::EMAIL_EVENTS.each do |event|
+          expose event
+        end
+      end
+    end
+
+    class GlobalNotificationSetting < NotificationSetting
+      expose :notification_email do |notification_setting, options|
+        notification_setting.user.notification_email
+      end
+    end
+
     class ProjectService < Grape::Entity
       expose :id, :title, :created_at, :updated_at, :active
       expose :push_events, :issues_events, :merge_requests_events
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index d2df77238d5..953fa474e88 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -23,17 +23,19 @@ module API
       # Create group. Available only for users who can create groups.
       #
       # Parameters:
-      #   name (required)             - The name of the group
-      #   path (required)             - The path of the group
-      #   description (optional)      - The description of the group
-      #   visibility_level (optional) - The visibility level of the group
+      #   name (required)                   - The name of the group
+      #   path (required)                   - The path of the group
+      #   description (optional)            - The description of the group
+      #   visibility_level (optional)       - The visibility level of the group
+      #   lfs_enabled (optional)            - Enable/disable LFS for the projects in this group
+      #   request_access_enabled (optional) - Allow users to request member access
       # Example Request:
       #   POST /groups
       post do
         authorize! :create_group
         required_attributes! [:name, :path]
 
-        attrs = attributes_for_keys [:name, :path, :description, :visibility_level]
+        attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled]
         @group = Group.new(attrs)
 
         if @group.save
@@ -47,17 +49,19 @@ module API
       # Update group. Available only for users who can administrate groups.
       #
       # Parameters:
-      #   id (required)               - The ID of a group
-      #   path (optional)             - The path of the group
-      #   description (optional)      - The description of the group
-      #   visibility_level (optional) - The visibility level of the group
+      #   id (required)                     - The ID of a group
+      #   path (optional)                   - The path of the group
+      #   description (optional)            - The description of the group
+      #   visibility_level (optional)       - The visibility level of the group
+      #   lfs_enabled (optional)            - Enable/disable LFS for the projects in this group
+      #   request_access_enabled (optional) - Allow users to request member access
       # Example Request:
       #   PUT /groups/:id
       put ':id' do
         group = find_group(params[:id])
         authorize! :admin_group, group
 
-        attrs = attributes_for_keys [:name, :path, :description, :visibility_level]
+        attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled]
 
         if ::Groups::UpdateService.new(group, current_user, attrs).execute
           present group, with: Entities::GroupDetail
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 6a20ba95a79..150875ed4f0 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -269,6 +269,10 @@ module API
       render_api_error!('304 Not Modified', 304)
     end
 
+    def no_content!
+      render_api_error!('204 No Content', 204)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 6e6efece7c4..1114fd21784 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -35,6 +35,14 @@ module API
             Project.find_with_namespace(project_path)
           end
         end
+
+        def ssh_authentication_abilities
+          [
+            :read_project,
+            :download_code,
+            :push_code
+          ]
+        end
       end
 
       post "/allowed" do
@@ -51,9 +59,9 @@ module API
 
         access =
           if wiki?
-            Gitlab::GitAccessWiki.new(actor, project, protocol)
+            Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
           else
-            Gitlab::GitAccess.new(actor, project, protocol)
+            Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
           end
 
         access_status = access.check(params[:action], params[:changes])
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 556684187d8..c9689e6f8ef 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -41,7 +41,8 @@ module API
         issues = current_user.issues.inc_notes_with_associations
         issues = filter_issues_state(issues, params[:state]) unless params[:state].nil?
         issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil?
-        issues.reorder(issuable_order_by => issuable_sort)
+        issues = issues.reorder(issuable_order_by => issuable_sort)
+
         present paginate(issues), with: Entities::Issue, current_user: current_user
       end
     end
@@ -73,7 +74,11 @@ module API
         params[:group_id] = group.id
         params[:milestone_title] = params.delete(:milestone)
         params[:label_name] = params.delete(:labels)
-        params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort]
+
+        if params[:order_by] || params[:sort]
+          # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example)
+          params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}"
+        end
 
         issues = IssuesFinder.new(current_user, params).execute
 
@@ -113,7 +118,8 @@ module API
           issues = filter_issues_milestone(issues, params[:milestone])
         end
 
-        issues.reorder(issuable_order_by => issuable_sort)
+        issues = issues.reorder(issuable_order_by => issuable_sort)
+
         present paginate(issues), with: Entities::Issue, current_user: current_user
       end
 
diff --git a/lib/api/lint.rb b/lib/api/lint.rb
new file mode 100644
index 00000000000..ae43a4a3237
--- /dev/null
+++ b/lib/api/lint.rb
@@ -0,0 +1,21 @@
+module API
+  class Lint < Grape::API
+    namespace :ci do
+      desc 'Validation of .gitlab-ci.yml content'
+      params do
+        requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
+      end
+      post '/lint' do
+        error = Ci::GitlabCiYamlProcessor.validation_message(params[:content])
+
+        status 200
+
+        if error.blank?
+          { status: 'valid', errors: [] }
+        else
+          { status: 'invalid', errors: [error] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 94c16710d9a..37f0a6512f4 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -18,11 +18,11 @@ module API
         get ":id/members" do
           source = find_source(source_type, params[:id])
 
-          members = source.members.includes(:user)
-          members = members.joins(:user).merge(User.search(params[:query])) if params[:query]
-          members = paginate(members)
+          users = source.users
+          users = users.merge(User.search(params[:query])) if params[:query]
+          users = paginate(users)
 
-          present members.map(&:user), with: Entities::Member, members: members
+          present users, with: Entities::Member, source: source
         end
 
         # Get a group/project member
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 8bfa998dc53..c5c214d4d13 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -83,12 +83,12 @@ module API
             opts[:created_at] = params[:created_at]
           end
 
-          @note = ::Notes::CreateService.new(user_project, current_user, opts).execute
+          note = ::Notes::CreateService.new(user_project, current_user, opts).execute
 
-          if @note.valid?
-            present @note, with: Entities::Note
+          if note.valid?
+            present note, with: Entities::const_get(note.class.name)
           else
-            not_found!("Note #{@note.errors.messages}")
+            not_found!("Note #{note.errors.messages}")
           end
         end
 
diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb
new file mode 100644
index 00000000000..a70a7e71073
--- /dev/null
+++ b/lib/api/notification_settings.rb
@@ -0,0 +1,97 @@
+module API
+  # notification_settings API
+  class NotificationSettings < Grape::API
+    before { authenticate! }
+
+    helpers ::API::Helpers::MembersHelpers
+
+    resource :notification_settings do
+      desc 'Get global notification level settings and email, defaults to Participate' do
+        detail 'This feature was introduced in GitLab 8.12'
+        success Entities::GlobalNotificationSetting
+      end
+      get do
+        notification_setting = current_user.global_notification_setting
+
+        present notification_setting, with: Entities::GlobalNotificationSetting
+      end
+
+      desc 'Update global notification level settings and email, defaults to Participate' do
+        detail 'This feature was introduced in GitLab 8.12'
+        success Entities::GlobalNotificationSetting
+      end
+      params do
+        optional :level, type: String, desc: 'The global notification level'
+        optional :notification_email, type: String, desc: 'The email address to send notifications'
+        NotificationSetting::EMAIL_EVENTS.each do |event|
+          optional event, type: Boolean, desc: 'Enable/disable this notification'
+        end
+      end
+      put do
+        notification_setting = current_user.global_notification_setting
+
+        begin
+          notification_setting.transaction do
+            new_notification_email = params.delete(:notification_email)
+            declared_params = declared(params, include_missing: false).to_h
+
+            current_user.update(notification_email: new_notification_email) if new_notification_email
+            notification_setting.update(declared_params)
+          end
+        rescue ArgumentError => e # catch level enum error
+          render_api_error! e.to_s, 400
+        end
+
+        render_validation_error! current_user
+        render_validation_error! notification_setting
+        present notification_setting, with: Entities::GlobalNotificationSetting
+      end
+    end
+
+    %w[group project].each do |source_type|
+      resource source_type.pluralize do
+        desc "Get #{source_type} level notification level settings, defaults to Global" do
+          detail 'This feature was introduced in GitLab 8.12'
+          success Entities::NotificationSetting
+        end
+        params do
+          requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME'
+        end
+        get ":id/notification_settings" do
+          source = find_source(source_type, params[:id])
+
+          notification_setting = current_user.notification_settings_for(source)
+
+          present notification_setting, with: Entities::NotificationSetting
+        end
+
+        desc "Update #{source_type} level notification level settings, defaults to Global" do
+          detail 'This feature was introduced in GitLab 8.12'
+          success Entities::NotificationSetting
+        end
+        params do
+          requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME'
+          optional :level, type: String, desc: "The #{source_type} notification level"
+          NotificationSetting::EMAIL_EVENTS.each do |event|
+            optional event, type: Boolean, desc: 'Enable/disable this notification'
+          end
+        end
+        put ":id/notification_settings" do
+          source = find_source(source_type, params.delete(:id))
+          notification_setting = current_user.notification_settings_for(source)
+
+          begin
+            declared_params = declared(params, include_missing: false).to_h
+
+            notification_setting.update(declared_params)
+          rescue ArgumentError => e # catch level enum error
+            render_api_error! e.to_s, 400
+          end
+
+          render_validation_error! notification_setting
+          present notification_setting, with: Entities::NotificationSetting
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 2aae75c471d..2a0c8e1f2c0 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -13,11 +13,14 @@ module API
       params do
         optional :page,     type: Integer, desc: 'Page number of the current request'
         optional :per_page, type: Integer, desc: 'Number of items per page'
+        optional :scope,    type: String, values: ['running', 'branches', 'tags'],
+                            desc: 'Either running, branches, or tags'
       end
       get ':id/pipelines' do
         authorize! :read_pipeline, user_project
 
-        present paginate(user_project.pipelines), with: Entities::Pipeline
+        pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope])
+        present paginate(pipelines), with: Entities::Pipeline
       end
 
       desc 'Gets a specific pipeline for the project' do
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 4033f597859..5eb83c2c8f8 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -91,8 +91,8 @@ module API
       # Create new project
       #
       # Parameters:
-      #   name (required) - name for new project
-      #   description (optional) - short project description
+      #   name (required)                   - name for new project
+      #   description (optional)            - short project description
       #   issues_enabled (optional)
       #   merge_requests_enabled (optional)
       #   builds_enabled (optional)
@@ -100,33 +100,35 @@ module API
       #   snippets_enabled (optional)
       #   container_registry_enabled (optional)
       #   shared_runners_enabled (optional)
-      #   namespace_id (optional) - defaults to user namespace
-      #   public (optional) - if true same as setting visibility_level = 20
-      #   visibility_level (optional) - 0 by default
+      #   namespace_id (optional)           - defaults to user namespace
+      #   public (optional)                 - if true same as setting visibility_level = 20
+      #   visibility_level (optional)       - 0 by default
       #   import_url (optional)
       #   public_builds (optional)
       #   lfs_enabled (optional)
+      #   request_access_enabled (optional) - Allow users to request member access
       # Example Request
       #   POST /projects
       post do
         required_attributes! [:name]
-        attrs = attributes_for_keys [:name,
-                                     :path,
+        attrs = attributes_for_keys [:builds_enabled,
+                                     :container_registry_enabled,
                                      :description,
+                                     :import_url,
                                      :issues_enabled,
+                                     :lfs_enabled,
                                      :merge_requests_enabled,
-                                     :builds_enabled,
-                                     :wiki_enabled,
-                                     :snippets_enabled,
-                                     :container_registry_enabled,
-                                     :shared_runners_enabled,
+                                     :name,
                                      :namespace_id,
+                                     :only_allow_merge_if_build_succeeds,
+                                     :path,
                                      :public,
-                                     :visibility_level,
-                                     :import_url,
                                      :public_builds,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :lfs_enabled]
+                                     :request_access_enabled,
+                                     :shared_runners_enabled,
+                                     :snippets_enabled,
+                                     :visibility_level,
+                                     :wiki_enabled]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(current_user, attrs).execute
         if @project.saved?
@@ -143,10 +145,10 @@ module API
       # Create new project for a specified user.  Only available to admin users.
       #
       # Parameters:
-      #   user_id (required) - The ID of a user
-      #   name (required) - name for new project
-      #   description (optional) - short project description
-      #   default_branch (optional) - 'master' by default
+      #   user_id (required)                - The ID of a user
+      #   name (required)                   - name for new project
+      #   description (optional)            - short project description
+      #   default_branch (optional)         - 'master' by default
       #   issues_enabled (optional)
       #   merge_requests_enabled (optional)
       #   builds_enabled (optional)
@@ -154,31 +156,33 @@ module API
       #   snippets_enabled (optional)
       #   container_registry_enabled (optional)
       #   shared_runners_enabled (optional)
-      #   public (optional) - if true same as setting visibility_level = 20
+      #   public (optional)                 - if true same as setting visibility_level = 20
       #   visibility_level (optional)
       #   import_url (optional)
       #   public_builds (optional)
       #   lfs_enabled (optional)
+      #   request_access_enabled (optional) - Allow users to request member access
       # Example Request
       #   POST /projects/user/:user_id
       post "user/:user_id" do
         authenticated_as_admin!
         user = User.find(params[:user_id])
-        attrs = attributes_for_keys [:name,
-                                     :description,
+        attrs = attributes_for_keys [:builds_enabled,
                                      :default_branch,
+                                     :description,
+                                     :import_url,
                                      :issues_enabled,
+                                     :lfs_enabled,
                                      :merge_requests_enabled,
-                                     :builds_enabled,
-                                     :wiki_enabled,
-                                     :snippets_enabled,
-                                     :shared_runners_enabled,
+                                     :name,
+                                     :only_allow_merge_if_build_succeeds,
                                      :public,
-                                     :visibility_level,
-                                     :import_url,
                                      :public_builds,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :lfs_enabled]
+                                     :request_access_enabled,
+                                     :shared_runners_enabled,
+                                     :snippets_enabled,
+                                     :visibility_level,
+                                     :wiki_enabled]
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(user, attrs).execute
         if @project.saved?
@@ -242,22 +246,23 @@ module API
       # Example Request
       #   PUT /projects/:id
       put ':id' do
-        attrs = attributes_for_keys [:name,
-                                     :path,
-                                     :description,
+        attrs = attributes_for_keys [:builds_enabled,
+                                     :container_registry_enabled,
                                      :default_branch,
+                                     :description,
                                      :issues_enabled,
+                                     :lfs_enabled,
                                      :merge_requests_enabled,
-                                     :builds_enabled,
-                                     :wiki_enabled,
-                                     :snippets_enabled,
-                                     :container_registry_enabled,
-                                     :shared_runners_enabled,
+                                     :name,
+                                     :only_allow_merge_if_build_succeeds,
+                                     :path,
                                      :public,
-                                     :visibility_level,
                                      :public_builds,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :lfs_enabled]
+                                     :request_access_enabled,
+                                     :shared_runners_enabled,
+                                     :snippets_enabled,
+                                     :visibility_level,
+                                     :wiki_enabled]
         attrs = map_public_to_visibility_level(attrs)
         authorize_admin_project
         authorize! :rename_project, user_project if attrs[:name].present?
@@ -428,18 +433,9 @@ module API
       # Example Request:
       #   GET /projects/search/:query
       get "/search/:query" do
-        ids = current_user.authorized_projects.map(&:id)
-        visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ]
-        projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%")
-        sort = params[:sort] == 'desc' ? 'desc' : 'asc'
-
-        projects = case params["order_by"]
-                   when 'id' then projects.order("id #{sort}")
-                   when 'name' then projects.order("name #{sort}")
-                   when 'created_at' then projects.order("created_at #{sort}")
-                   when 'last_activity_at' then projects.order("last_activity_at #{sort}")
-                   else projects
-                   end
+        search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
+        projects = search_service.objects('projects', params[:page])
+        projects = projects.reorder(project_order_by => project_sort)
 
         present paginate(projects), with: Entities::Project
       end
diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb
index 2e2c8da311e..e7a1ec8457d 100644
--- a/lib/banzai/filter/wiki_link_filter/rewriter.rb
+++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb
@@ -31,6 +31,7 @@ module Banzai
         def apply_relative_link_rules!
           if @uri.relative? && @uri.path.present?
             link = ::File.join(@wiki_base_path, @uri.path)
+            link = "#{link}##{@uri.fragment}" if @uri.fragment
             @uri = Addressable::URI.parse(link)
           end
         end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 9f3b582a263..59f85416ee5 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -12,7 +12,7 @@ module Ci
         #   POST /builds/register
         post "register" do
           authenticate_runner!
-          update_runner_last_contact
+          update_runner_last_contact(save: false)
           update_runner_info
           required_attributes! [:token]
           not_found! unless current_runner.active?
@@ -27,7 +27,7 @@ module Ci
           else
             Gitlab::Metrics.add_event(:build_not_found)
 
-            not_found!
+            build_not_found!
           end
         end
 
@@ -101,6 +101,7 @@ module Ci
         #   POST /builds/:id/artifacts/authorize
         post ":id/artifacts/authorize" do
           require_gitlab_workhorse!
+          Gitlab::Workhorse.verify_api_request!(headers)
           not_allowed! unless Gitlab.config.artifacts.enabled
           build = Ci::Build.find_by_id(params[:id])
           not_found! unless build
@@ -113,7 +114,8 @@ module Ci
           end
 
           status 200
-          { TempPath: ArtifactUploader.artifacts_upload_path }
+          content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+          Gitlab::Workhorse.artifact_upload_ok
         end
 
         # Upload artifacts to build - Runners only
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index 3f5bdaba3f5..66c05773b68 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -15,6 +15,15 @@ module Ci
         expose :filename, :size
       end
 
+      class BuildOptions < Grape::Entity
+        expose :image
+        expose :services
+        expose :artifacts
+        expose :cache
+        expose :dependencies
+        expose :after_script
+      end
+
       class Build < Grape::Entity
         expose :id, :ref, :tag, :sha, :status
         expose :name, :token, :stage
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 199d62d9b8a..23353c62885 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -3,7 +3,7 @@ module Ci
     module Helpers
       BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
       BUILD_TOKEN_PARAM = :token
-      UPDATE_RUNNER_EVERY = 60
+      UPDATE_RUNNER_EVERY = 40 * 60
 
       def authenticate_runners!
         forbidden! unless runner_registration_token_valid?
@@ -14,19 +14,37 @@ module Ci
       end
 
       def authenticate_build_token!(build)
-        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
-        forbidden! unless token && build.valid_token?(token)
+        forbidden! unless build_token_valid?(build)
       end
 
       def runner_registration_token_valid?
-        params[:token] == current_application_settings.runners_registration_token
+        ActiveSupport::SecurityUtils.variable_size_secure_compare(
+          params[:token],
+          current_application_settings.runners_registration_token)
+      end
+
+      def build_token_valid?(build)
+        token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+
+        # We require to also check `runners_token` to maintain compatibility with old version of runners
+        token && (build.valid_token?(token) || build.project.valid_runners_token?(token))
       end
 
-      def update_runner_last_contact
+      def update_runner_last_contact(save: true)
         # Use a random threshold to prevent beating DB updates
+        # it generates a distribution between: [40m, 80m]
         contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
         if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age
-          current_runner.update_attributes(contacted_at: Time.now)
+          current_runner.contacted_at = Time.now
+          current_runner.save if current_runner.changed? && save
+        end
+      end
+
+      def build_not_found!
+        if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /)
+          no_content!
+        else
+          not_found!
         end
       end
 
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 47efd5bd9f2..0369e80312a 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -55,29 +55,36 @@ module Ci
       {
         stage_idx: @stages.index(job[:stage]),
         stage: job[:stage],
-        ##
-        # Refactoring note:
-        #  - before script behaves differently than after script
-        #  - after script returns an array of commands
-        #  - before script should be a concatenated command
-        commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
+        commands: job[:commands],
         tag_list: job[:tags] || [],
         name: job[:name].to_s,
         allow_failure: job[:allow_failure] || false,
         when: job[:when] || 'on_success',
-        environment: job[:environment],
+        environment: job[:environment_name],
         yaml_variables: yaml_variables(name),
         options: {
-          image: job[:image] || @image,
-          services: job[:services] || @services,
+          image: job[:image],
+          services: job[:services],
           artifacts: job[:artifacts],
-          cache: job[:cache] || @cache,
+          cache: job[:cache],
           dependencies: job[:dependencies],
-          after_script: job[:after_script] || @after_script,
+          after_script: job[:after_script],
+          environment: job[:environment],
         }.compact
       }
     end
 
+    def self.validation_message(content)
+      return 'Please provide content of .gitlab-ci.yml' if content.blank?
+
+      begin
+        Ci::GitlabCiYamlProcessor.new(content)
+        nil
+      rescue ValidationError, Psych::SyntaxError => e
+        e.message
+      end
+    end
+
     private
 
     def initial_parsing
diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb
new file mode 100644
index 00000000000..3da04edde70
--- /dev/null
+++ b/lib/ci/mask_secret.rb
@@ -0,0 +1,9 @@
+module Ci::MaskSecret
+  class << self
+    def mask(value, token)
+      return value unless value.present? && token.present?
+
+      value.gsub(token, 'x' * token.length)
+    end
+  end
+end
diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb
new file mode 100644
index 00000000000..7b1533d0d32
--- /dev/null
+++ b/lib/expand_variables.rb
@@ -0,0 +1,17 @@
+module ExpandVariables
+  class << self
+    def expand(value, variables)
+      # Convert hash array to variables
+      if variables.is_a?(Array)
+        variables = variables.reduce({}) do |hash, variable|
+          hash[variable[:key]] = variable[:value]
+          hash
+        end
+      end
+
+      value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do
+        variables[$1 || $2]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 91f0270818a..0a0f1c3b17b 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -1,21 +1,21 @@
 module Gitlab
   module Auth
-    Result = Struct.new(:user, :type)
+    class MissingPersonalTokenError < StandardError; end
 
     class << self
       def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
 
-        result = Result.new
+        result =
+          service_request_check(login, password, project) ||
+          build_access_token_check(login, password) ||
+          user_with_password_for_git(login, password) ||
+          oauth_access_token_check(login, password) ||
+          personal_access_token_check(login, password) ||
+          Gitlab::Auth::Result.new
 
-        if valid_ci_request?(login, password, project)
-          result.type = :ci
-        else
-          result = populate_result(login, password)
-        end
+        rate_limit!(ip, success: result.success?, login: login)
 
-        success = result.user.present? || [:ci, :missing_personal_token].include?(result.type)
-        rate_limit!(ip, success: success, login: login)
         result
       end
 
@@ -57,44 +57,31 @@ module Gitlab
 
       private
 
-      def valid_ci_request?(login, password, project)
+      def service_request_check(login, password, project)
         matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login)
 
-        return false unless project && matched_login.present?
+        return unless project && matched_login.present?
 
         underscored_service = matched_login['service'].underscore
 
-        if underscored_service == 'gitlab_ci'
-          project && project.valid_build_token?(password)
-        elsif Service.available_services_names.include?(underscored_service)
+        if Service.available_services_names.include?(underscored_service)
           # We treat underscored_service as a trusted input because it is included
           # in the Service.available_services_names whitelist.
           service = project.public_send("#{underscored_service}_service")
 
-          service && service.activated? && service.valid_token?(password)
-        end
-      end
-
-      def populate_result(login, password)
-        result =
-          user_with_password_for_git(login, password) ||
-          oauth_access_token_check(login, password) ||
-          personal_access_token_check(login, password)
-
-        if result
-          result.type = nil unless result.user
-
-          if result.user && result.user.two_factor_enabled? && result.type == :gitlab_or_ldap
-            result.type = :missing_personal_token
+          if service && service.activated? && service.valid_token?(password)
+            Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities)
           end
         end
-
-        result || Result.new
       end
 
       def user_with_password_for_git(login, password)
         user = find_with_user_password(login, password)
-        Result.new(user, :gitlab_or_ldap) if user
+        return unless user
+
+        raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled?
+
+        Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities)
       end
 
       def oauth_access_token_check(login, password)
@@ -102,7 +89,7 @@ module Gitlab
           token = Doorkeeper::AccessToken.by_token(password)
           if token && token.accessible?
             user = User.find_by(id: token.resource_owner_id)
-            Result.new(user, :oauth)
+            Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities)
           end
         end
       end
@@ -111,9 +98,52 @@ module Gitlab
         if login && password
           user = User.find_by_personal_access_token(password)
           validation = User.by_login(login)
-          Result.new(user, :personal_token) if user == validation
+          Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation
+        end
+      end
+
+      def build_access_token_check(login, password)
+        return unless login == 'gitlab-ci-token'
+        return unless password
+
+        build = ::Ci::Build.running.find_by_token(password)
+        return unless build
+        return unless build.project.builds_enabled?
+
+        if build.user
+          # If user is assigned to build, use restricted credentials of user
+          Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities)
+        else
+          # Otherwise use generic CI credentials (backward compatibility)
+          Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities)
         end
       end
+
+      public
+
+      def build_authentication_abilities
+        [
+          :read_project,
+          :build_download_code,
+          :build_read_container_image,
+          :build_create_container_image
+        ]
+      end
+
+      def read_authentication_abilities
+        [
+          :read_project,
+          :download_code,
+          :read_container_image
+        ]
+      end
+
+      def full_authentication_abilities
+        read_authentication_abilities + [
+          :push_code,
+          :create_container_image
+        ]
+      end
     end
   end
 end
diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb
new file mode 100644
index 00000000000..bf625649cbf
--- /dev/null
+++ b/lib/gitlab/auth/result.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Auth
+    Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
+      def ci?
+        type == :ci
+      end
+
+      def success?
+        actor.present? || type == :ci
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 839a4fa30d5..79eac66b364 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -6,7 +6,12 @@ module Gitlab
 
     KeyAdder = Struct.new(:io) do
       def add_key(id, key)
-        key.gsub!(/[[:space:]]+/, ' ').strip!
+        key = Gitlab::Shell.strip_key(key)
+        # Newline and tab are part of the 'protocol' used to transmit id+key to the other end
+        if key.include?("\t") || key.include?("\n")
+          raise Error.new("Invalid key: #{key.inspect}")
+        end
+
         io.puts("#{id}\t#{key}")
       end
     end
@@ -16,6 +21,10 @@ module Gitlab
         @version_required ||= File.read(Rails.root.
                                         join('GITLAB_SHELL_VERSION')).strip
       end
+
+      def strip_key(key)
+        key.split(/ /)[0, 2].join(' ')
+      end
     end
 
     # Init new repository
@@ -107,7 +116,7 @@ module Gitlab
     #
     def add_key(key_id, key_content)
       Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'add-key', key_id, key_content])
+                                   'add-key', key_id, self.class.strip_key(key_content)])
     end
 
     # Batch-add keys to authorized_keys
@@ -195,7 +204,7 @@ module Gitlab
     # Create (if necessary) and link the secret token file
     def generate_and_link_secret_token
       secret_file = Gitlab.config.gitlab_shell.secret_file
-      unless File.exist? secret_file
+      unless File.size?(secret_file)
         # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
         token = SecureRandom.hex(16)
         File.write(secret_file, token)
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 7beaecd1cf0..f4b5097adb1 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -21,7 +21,7 @@ module Gitlab
 
       private
 
-      def gl_user_id(project, bitbucket_id)
+      def gitlab_user_id(project, bitbucket_id)
         if bitbucket_id
           user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
           (user && user.id) || project.creator_id
@@ -74,7 +74,7 @@ module Gitlab
             description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gl_user_id(project, reporter)
+            author_id: gitlab_user_id(project, reporter)
           )
         end
       rescue ActiveRecord::RecordInvalid => e
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 4b32eb966aa..cb1065223d4 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -23,6 +23,7 @@ module Gitlab
       protected
 
       def protected_branch_checks
+        return unless @branch_name
         return unless project.protected_branch?(@branch_name)
 
         if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches)
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index ae82c0db3f1..bbfa6cf7d05 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -14,7 +14,7 @@ module Gitlab
         @config = Loader.new(config).load!
 
         @global = Node::Global.new(@config)
-        @global.process!
+        @global.compose!
       end
 
       def valid?
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb
index 2de82d40c9d..6b7ab2fdaf2 100644
--- a/lib/gitlab/ci/config/node/configurable.rb
+++ b/lib/gitlab/ci/config/node/configurable.rb
@@ -23,9 +23,9 @@ module Gitlab
             end
           end
 
-          private
+          def compose!(deps = nil)
+            return unless valid?
 
-          def compose!
             self.class.nodes.each do |key, factory|
               factory
                 .value(@config[key])
@@ -33,6 +33,12 @@ module Gitlab
 
               @entries[key] = factory.create!
             end
+
+            yield if block_given?
+
+            @entries.each_value do |entry|
+              entry.compose!(deps)
+            end
           end
 
           class_methods do
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb
index 0c782c422b5..8717eabf81e 100644
--- a/lib/gitlab/ci/config/node/entry.rb
+++ b/lib/gitlab/ci/config/node/entry.rb
@@ -20,11 +20,14 @@ module Gitlab
             @validator.validate(:new)
           end
 
-          def process!
+          def [](key)
+            @entries[key] || Node::Undefined.new
+          end
+
+          def compose!(deps = nil)
             return unless valid?
 
-            compose!
-            descendants.each(&:process!)
+            yield if block_given?
           end
 
           def leaf?
@@ -73,11 +76,6 @@ module Gitlab
           def self.validator
             Validator
           end
-
-          private
-
-          def compose!
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb
new file mode 100644
index 00000000000..d388ab6b879
--- /dev/null
+++ b/lib/gitlab/ci/config/node/environment.rb
@@ -0,0 +1,68 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # Entry that represents an environment.
+        #
+        class Environment < Entry
+          include Validatable
+
+          ALLOWED_KEYS = %i[name url]
+
+          validations do
+            validate do
+              unless hash? || string?
+                errors.add(:config, 'should be a hash or a string')
+              end
+            end
+
+            validates :name, presence: true
+            validates :name,
+              type: {
+                with: String,
+                message: Gitlab::Regex.environment_name_regex_message }
+
+            validates :name,
+              format: {
+                with: Gitlab::Regex.environment_name_regex,
+                message: Gitlab::Regex.environment_name_regex_message }
+
+            with_options if: :hash? do
+              validates :config, allowed_keys: ALLOWED_KEYS
+
+              validates :url,
+                        length: { maximum: 255 },
+                        addressable_url: true,
+                        allow_nil: true
+            end
+          end
+
+          def hash?
+            @config.is_a?(Hash)
+          end
+
+          def string?
+            @config.is_a?(String)
+          end
+
+          def name
+            value[:name]
+          end
+
+          def url
+            value[:url]
+          end
+
+          def value
+            case @config
+            when String then { name: @config }
+            when Hash then @config
+            else {}
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb
index 707b052e6a8..5387f29ad59 100644
--- a/lib/gitlab/ci/config/node/factory.rb
+++ b/lib/gitlab/ci/config/node/factory.rb
@@ -37,8 +37,8 @@ module Gitlab
             # See issue #18775.
             #
             if @value.nil?
-              Node::Undefined.new(
-                fabricate_undefined
+              Node::Unspecified.new(
+                fabricate_unspecified
               )
             else
               fabricate(@node, @value)
@@ -47,13 +47,13 @@ module Gitlab
 
           private
 
-          def fabricate_undefined
+          def fabricate_unspecified
             ##
             # If node has a default value we fabricate concrete node
             # with default value.
             #
             if @node.default.nil?
-              fabricate(Node::Null)
+              fabricate(Node::Undefined)
             else
               fabricate(@node, @node.default)
             end
diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb
index ccd539fb003..2a2943c9288 100644
--- a/lib/gitlab/ci/config/node/global.rb
+++ b/lib/gitlab/ci/config/node/global.rb
@@ -36,15 +36,15 @@ module Gitlab
           helpers :before_script, :image, :services, :after_script,
                   :variables, :stages, :types, :cache, :jobs
 
-          private
-
-          def compose!
-            super
-
-            compose_jobs!
-            compose_deprecated_entries!
+          def compose!(_deps = nil)
+            super(self) do
+              compose_jobs!
+              compose_deprecated_entries!
+            end
           end
 
+          private
+
           def compose_jobs!
             factory = Node::Factory.new(Node::Jobs)
               .value(@config.except(*self.class.nodes.keys))
diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb
index e84737acbb9..603334d6793 100644
--- a/lib/gitlab/ci/config/node/job.rb
+++ b/lib/gitlab/ci/config/node/job.rb
@@ -13,7 +13,7 @@ module Gitlab
                             type stage when artifacts cache dependencies before_script
                             after_script variables environment]
 
-          attributes :tags, :allow_failure, :when, :environment, :dependencies
+          attributes :tags, :allow_failure, :when, :dependencies
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -29,58 +29,65 @@ module Gitlab
                 inclusion: { in: %w[on_success on_failure always manual],
                              message: 'should be on_success, on_failure, ' \
                                       'always or manual' }
-              validates :environment,
-                type: {
-                  with: String,
-                  message: Gitlab::Regex.environment_name_regex_message }
-              validates :environment,
-                format: {
-                  with: Gitlab::Regex.environment_name_regex,
-                  message: Gitlab::Regex.environment_name_regex_message }
 
               validates :dependencies, array_of_strings: true
             end
           end
 
-          node :before_script, Script,
+          node :before_script, Node::Script,
             description: 'Global before script overridden in this job.'
 
-          node :script, Commands,
+          node :script, Node::Commands,
             description: 'Commands that will be executed in this job.'
 
-          node :stage, Stage,
+          node :stage, Node::Stage,
             description: 'Pipeline stage this job will be executed into.'
 
-          node :type, Stage,
+          node :type, Node::Stage,
             description: 'Deprecated: stage this job will be executed into.'
 
-          node :after_script, Script,
+          node :after_script, Node::Script,
             description: 'Commands that will be executed when finishing job.'
 
-          node :cache, Cache,
+          node :cache, Node::Cache,
             description: 'Cache definition for this job.'
 
-          node :image, Image,
+          node :image, Node::Image,
             description: 'Image that will be used to execute this job.'
 
-          node :services, Services,
+          node :services, Node::Services,
             description: 'Services that will be used to execute this job.'
 
-          node :only, Trigger,
+          node :only, Node::Trigger,
             description: 'Refs policy this job will be executed for.'
 
-          node :except, Trigger,
+          node :except, Node::Trigger,
             description: 'Refs policy this job will be executed for.'
 
-          node :variables, Variables,
+          node :variables, Node::Variables,
             description: 'Environment variables available for this job.'
 
-          node :artifacts, Artifacts,
+          node :artifacts, Node::Artifacts,
             description: 'Artifacts configuration for this job.'
 
+          node :environment, Node::Environment,
+               description: 'Environment configuration for this job.'
+
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts
+                  :artifacts, :commands, :environment
+
+          def compose!(deps = nil)
+            super do
+              if type_defined? && !stage_defined?
+                @entries[:stage] = @entries[:type]
+              end
+
+              @entries.delete(:type)
+            end
+
+            inherit!(deps)
+          end
 
           def name
             @metadata[:name]
@@ -90,12 +97,30 @@ module Gitlab
             @config.merge(to_hash.compact)
           end
 
+          def commands
+            (before_script_value.to_a + script_value.to_a).join("\n")
+          end
+
           private
 
+          def inherit!(deps)
+            return unless deps
+
+            self.class.nodes.each_key do |key|
+              global_entry = deps[key]
+              job_entry = @entries[key]
+
+              if global_entry.specified? && !job_entry.specified?
+                @entries[key] = global_entry
+              end
+            end
+          end
+
           def to_hash
             { name: name,
               before_script: before_script,
               script: script,
+              commands: commands,
               image: image,
               services: services,
               stage: stage,
@@ -103,19 +128,11 @@ module Gitlab
               only: only,
               except: except,
               variables: variables_defined? ? variables : nil,
+              environment: environment_defined? ? environment : nil,
+              environment_name: environment_defined? ? environment[:name] : nil,
               artifacts: artifacts,
               after_script: after_script }
           end
-
-          def compose!
-            super
-
-            if type_defined? && !stage_defined?
-              @entries[:stage] = @entries[:type]
-            end
-
-            @entries.delete(:type)
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb
index a1a26d4fd8f..d10e80d1a7d 100644
--- a/lib/gitlab/ci/config/node/jobs.rb
+++ b/lib/gitlab/ci/config/node/jobs.rb
@@ -26,19 +26,23 @@ module Gitlab
             name.to_s.start_with?('.')
           end
 
-          private
-
-          def compose!
-            @config.each do |name, config|
-              node = hidden?(name) ? Node::Hidden : Node::Job
-
-              factory = Node::Factory.new(node)
-                .value(config || {})
-                .metadata(name: name)
-                .with(key: name, parent: self,
-                      description: "#{name} job definition.")
+          def compose!(deps = nil)
+            super do
+              @config.each do |name, config|
+                node = hidden?(name) ? Node::Hidden : Node::Job
+
+                factory = Node::Factory.new(node)
+                  .value(config || {})
+                  .metadata(name: name)
+                  .with(key: name, parent: self,
+                        description: "#{name} job definition.")
+
+                @entries[name] = factory.create!
+              end
 
-              @entries[name] = factory.create!
+              @entries.each_value do |entry|
+                entry.compose!(deps)
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb
deleted file mode 100644
index 88a5f53f13c..00000000000
--- a/lib/gitlab/ci/config/node/null.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Node
-        ##
-        # This class represents an undefined node.
-        #
-        # Implements the Null Object pattern.
-        #
-        class Null < Entry
-          def value
-            nil
-          end
-
-          def valid?
-            true
-          end
-
-          def errors
-            []
-          end
-
-          def specified?
-            false
-          end
-
-          def relevant?
-            false
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb
index 45fef8c3ae5..33e78023539 100644
--- a/lib/gitlab/ci/config/node/undefined.rb
+++ b/lib/gitlab/ci/config/node/undefined.rb
@@ -3,15 +3,34 @@ module Gitlab
     class Config
       module Node
         ##
-        # This class represents an unspecified entry node.
+        # This class represents an undefined node.
         #
-        # It decorates original entry adding method that indicates it is
-        # unspecified.
+        # Implements the Null Object pattern.
         #
-        class Undefined < SimpleDelegator
+        class Undefined < Entry
+          def initialize(*)
+            super(nil)
+          end
+
+          def value
+            nil
+          end
+
+          def valid?
+            true
+          end
+
+          def errors
+            []
+          end
+
           def specified?
             false
           end
+
+          def relevant?
+            false
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/node/unspecified.rb
new file mode 100644
index 00000000000..a7d1f6131b8
--- /dev/null
+++ b/lib/gitlab/ci/config/node/unspecified.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        ##
+        # This class represents an unspecified entry node.
+        #
+        # It decorates original entry adding method that indicates it is
+        # unspecified.
+        #
+        class Unspecified < SimpleDelegator
+          def specified?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb
new file mode 100644
index 00000000000..a210e76acaa
--- /dev/null
+++ b/lib/gitlab/ci/pipeline_duration.rb
@@ -0,0 +1,141 @@
+module Gitlab
+  module Ci
+    # # Introduction - total running time
+    #
+    # The problem this module is trying to solve is finding the total running
+    # time amongst all the jobs, excluding retries and pending (queue) time.
+    # We could reduce this problem down to finding the union of periods.
+    #
+    # So each job would be represented as a `Period`, which consists of
+    # `Period#first` as when the job started and `Period#last` as when the
+    # job was finished. A simple example here would be:
+    #
+    # * A (1, 3)
+    # * B (2, 4)
+    # * C (6, 7)
+    #
+    # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4.
+    # C begins from 6, and ends to 7. Visually it could be viewed as:
+    #
+    #     0  1  2  3  4  5  6  7
+    #        AAAAAAA
+    #           BBBBBBB
+    #                       CCCC
+    #
+    # The union of A, B, and C would be (1, 4) and (6, 7), therefore the
+    # total running time should be:
+    #
+    #     (4 - 1) + (7 - 6) => 4
+    #
+    # # The Algorithm
+    #
+    # The algorithm used here for union would be described as follow.
+    # First we make sure that all periods are sorted by `Period#first`.
+    # Then we try to merge periods by iterating through the first period
+    # to the last period. The goal would be merging all overlapped periods
+    # so that in the end all the periods are discrete. When all periods
+    # are discrete, we're free to just sum all the periods to get real
+    # running time.
+    #
+    # Here we begin from A, and compare it to B. We could find that
+    # before A ends, B already started. That is `B.first <= A.last`
+    # that is `2 <= 3` which means A and B are overlapping!
+    #
+    # When we found that two periods are overlapping, we would need to merge
+    # them into a new period and disregard the old periods. To make a new
+    # period, we take `A.first` as the new first because remember? we sorted
+    # them, so `A.first` must be smaller or equal to `B.first`. And we take
+    # `[A.last, B.last].max` as the new last because we want whoever ended
+    # later. This could be broken into two cases:
+    #
+    #     0  1  2  3  4
+    #        AAAAAAA
+    #           BBBBBBB
+    #
+    # Or:
+    #
+    #     0  1  2  3  4
+    #        AAAAAAAAAA
+    #           BBBB
+    #
+    # So that we need to take whoever ends later. Back to our example,
+    # after merging and discard A and B it could be visually viewed as:
+    #
+    #     0  1  2  3  4  5  6  7
+    #        DDDDDDDDDD
+    #                       CCCC
+    #
+    # Now we could go on and compare the newly created D and the old C.
+    # We could figure out that D and C are not overlapping by checking
+    # `C.first <= D.last` is `false`. Therefore we need to keep both C
+    # and D. The example would end here because there are no more jobs.
+    #
+    # After having the union of all periods, we just need to sum the length
+    # of all periods to get total time.
+    #
+    #     (4 - 1) + (7 - 6) => 4
+    #
+    # That is 4 is the answer in the example.
+    module PipelineDuration
+      extend self
+
+      Period = Struct.new(:first, :last) do
+        def duration
+          last - first
+        end
+      end
+
+      def from_pipeline(pipeline)
+        status = %w[success failed running canceled]
+        builds = pipeline.builds.latest.
+          where(status: status).where.not(started_at: nil).order(:started_at)
+
+        from_builds(builds)
+      end
+
+      def from_builds(builds)
+        now = Time.now
+
+        periods = builds.map do |b|
+          Period.new(b.started_at, b.finished_at || now)
+        end
+
+        from_periods(periods)
+      end
+
+      # periods should be sorted by `first`
+      def from_periods(periods)
+        process_duration(process_periods(periods))
+      end
+
+      private
+
+      def process_periods(periods)
+        return periods if periods.empty?
+
+        periods.drop(1).inject([periods.first]) do |result, current|
+          previous = result.last
+
+          if overlap?(previous, current)
+            result[-1] = merge(previous, current)
+            result
+          else
+            result << current
+          end
+        end
+      end
+
+      def overlap?(previous, current)
+        current.first <= previous.last
+      end
+
+      def merge(previous, current)
+        Period.new(previous.first, [previous.last, current.last].max)
+      end
+
+      def process_duration(periods)
+        periods.sum(&:duration)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb
index 2d4d55daeeb..98e842cded3 100644
--- a/lib/gitlab/conflict/parser.rb
+++ b/lib/gitlab/conflict/parser.rb
@@ -18,7 +18,7 @@ module Gitlab
 
       def parse(text, our_path:, their_path:, parent_file: nil)
         raise UnmergeableFile if text.blank? # Typically a binary file
-        raise UnmergeableFile if text.length > 102400
+        raise UnmergeableFile if text.length > 200.kilobytes
 
         begin
           text.to_json
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index bd681f03173..b164f5a2eea 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -1,16 +1,16 @@
 module Gitlab
   class ContributionsCalendar
-    attr_reader :timestamps, :projects, :user
+    attr_reader :activity_dates, :projects, :user
 
     def initialize(projects, user)
       @projects = projects
       @user = user
     end
 
-    def timestamps
-      return @timestamps if @timestamps.present?
+    def activity_dates
+      return @activity_dates if @activity_dates.present?
 
-      @timestamps = {}
+      @activity_dates = {}
       date_from = 1.year.ago
 
       events = Event.reorder(nil).contributions.where(author_id: user.id).
@@ -19,18 +19,17 @@ module Gitlab
         select('date(created_at) as date, count(id) as total_amount').
         map(&:attributes)
 
-      dates = (1.year.ago.to_date..Date.today).to_a
+      activity_dates = (1.year.ago.to_date..Date.today).to_a
 
-      dates.each do |date|
-        date_id = date.to_time.to_i.to_s
+      activity_dates.each do |date|
         day_events = events.find { |day_events| day_events["date"] == date }
 
         if day_events
-          @timestamps[date_id] = day_events["total_amount"]
+          @activity_dates[date] = day_events["total_amount"]
         end
       end
 
-      @timestamps
+      @activity_dates
     end
 
     def events_by_date(date)
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 927f9dad20b..0bd6e148ba8 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -129,12 +129,14 @@ module Gitlab
       # column - The name of the column to add.
       # type - The column type (e.g. `:integer`).
       # default - The default value for the column.
+      # limit - Sets a column limit. For example, for :integer, the default is
+      #         4-bytes. Set `limit: 8` to allow 8-byte integers.
       # allow_null - When set to `true` the column will allow NULL values, the
       #              default is to not allow NULL values.
       #
       # This method can also take a block which is passed directly to the
       # `update_column_in_batches` method.
-      def add_column_with_default(table, column, type, default:, allow_null: false, &block)
+      def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block)
         if transaction_open?
           raise 'add_column_with_default can not be run inside a transaction, ' \
             'you can disable transactions by calling disable_ddl_transaction! ' \
@@ -144,7 +146,11 @@ module Gitlab
         disable_statement_timeout
 
         transaction do
-          add_column(table, column, type, default: nil)
+          if limit
+            add_column(table, column, type, default: nil, limit: limit)
+          else
+            add_column(table, column, type, default: nil)
+          end
 
           # Changing the default before the update ensures any newly inserted
           # rows already use the proper default value.
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index 9b681e636c7..bd90d24a2ec 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -17,11 +17,13 @@ module Gitlab
       def trigger(gl_id, oldrev, newrev, ref)
         return [true, nil] unless exists?
 
-        case name
-        when "pre-receive", "post-receive"
-          call_receive_hook(gl_id, oldrev, newrev, ref)
-        when "update"
-          call_update_hook(gl_id, oldrev, newrev, ref)
+        Bundler.with_clean_env do
+          case name
+          when "pre-receive", "post-receive"
+            call_receive_hook(gl_id, oldrev, newrev, ref)
+          when "update"
+            call_update_hook(gl_id, oldrev, newrev, ref)
+          end
         end
       end
 
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 1882eb8d050..799794c0171 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -5,12 +5,13 @@ module Gitlab
     DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }
     PUSH_COMMANDS = %w{ git-receive-pack }
 
-    attr_reader :actor, :project, :protocol, :user_access
+    attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities
 
-    def initialize(actor, project, protocol)
+    def initialize(actor, project, protocol, authentication_abilities:)
       @actor    = actor
       @project  = project
       @protocol = protocol
+      @authentication_abilities = authentication_abilities
       @user_access = UserAccess.new(user, project: project)
     end
 
@@ -60,14 +61,26 @@ module Gitlab
     end
 
     def user_download_access_check
-      unless user_access.can_do_action?(:download_code)
+      unless user_can_download_code? || build_can_download_code?
         return build_status_object(false, "You are not allowed to download code from this project.")
       end
 
       build_status_object(true)
     end
 
+    def user_can_download_code?
+      authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code)
+    end
+
+    def build_can_download_code?
+      authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code)
+    end
+
     def user_push_access_check(changes)
+      unless authentication_abilities.include?(:push_code)
+        return build_status_object(false, "You are not allowed to upload code for this project.")
+      end
+
       if changes.blank?
         return build_status_object(true)
       end
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 72992baffd4..8cacf4f4925 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -15,11 +15,16 @@ module Gitlab
 
       private
 
-      def gl_user_id(github_id)
+      def gitlab_user_id(github_id)
         User.joins(:identities).
           find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s).
           try(:id)
       end
+
+      def gitlab_author_id
+        return @gitlab_author_id if defined?(@gitlab_author_id)
+        @gitlab_author_id = gitlab_user_id(raw_data.user.id)
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb
index 2c1b94ef2cd..2bddcde2b7c 100644
--- a/lib/gitlab/github_import/comment_formatter.rb
+++ b/lib/gitlab/github_import/comment_formatter.rb
@@ -21,7 +21,7 @@ module Gitlab
       end
 
       def author_id
-        gl_user_id(raw_data.user.id) || project.creator_id
+        gitlab_author_id || project.creator_id
       end
 
       def body
@@ -52,7 +52,11 @@ module Gitlab
       end
 
       def note
-        formatter.author_line(author) + body
+        if gitlab_author_id
+          body
+        else
+          formatter.author_line(author) + body
+        end
       end
 
       def type
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 4fdc2f46be0..d35ee2a1c65 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -24,6 +24,7 @@ module Gitlab
         import_issues
         import_pull_requests
         import_wiki
+        import_releases
         handle_errors
 
         true
@@ -133,8 +134,7 @@ module Gitlab
 
         if issue.labels.count > 0
           label_ids = issue.labels
-            .map { |raw| LabelFormatter.new(project, raw).attributes }
-            .map { |attrs| Label.find_by(attrs).try(:id) }
+            .map { |attrs| project.labels.find_by(title: attrs.name).try(:id) }
             .compact
 
           issuable.update_attribute(:label_ids, label_ids)
@@ -178,6 +178,18 @@ module Gitlab
           errors << { type: :wiki, errors: e.message }
         end
       end
+
+      def import_releases
+        releases = client.releases(repo, per_page: 100)
+        releases.each do |raw|
+          begin
+            gh_release = ReleaseFormatter.new(project, raw)
+            gh_release.create! if gh_release.valid?
+          rescue => e
+            errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index 07edbe37a13..77621de9f4c 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -40,7 +40,7 @@ module Gitlab
 
       def assignee_id
         if assigned?
-          gl_user_id(raw_data.assignee.id)
+          gitlab_user_id(raw_data.assignee.id)
         end
       end
 
@@ -49,7 +49,7 @@ module Gitlab
       end
 
       def author_id
-        gl_user_id(raw_data.user.id) || project.creator_id
+        gitlab_author_id || project.creator_id
       end
 
       def body
@@ -57,7 +57,11 @@ module Gitlab
       end
 
       def description
-        @formatter.author_line(author) + body
+        if gitlab_author_id
+          body
+        else
+          formatter.author_line(author) + body
+        end
       end
 
       def milestone
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
index 9f18244e7d7..2cad7fca88e 100644
--- a/lib/gitlab/github_import/label_formatter.rb
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -13,6 +13,12 @@ module Gitlab
         Label
       end
 
+      def create!
+        project.labels.find_or_create_by!(title: title) do |label|
+          label.color = color
+        end
+      end
+
       private
 
       def color
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index d9d436d7490..1408683100f 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -68,7 +68,7 @@ module Gitlab
 
       def assignee_id
         if assigned?
-          gl_user_id(raw_data.assignee.id)
+          gitlab_user_id(raw_data.assignee.id)
         end
       end
 
@@ -77,7 +77,7 @@ module Gitlab
       end
 
       def author_id
-        gl_user_id(raw_data.user.id) || project.creator_id
+        gitlab_author_id || project.creator_id
       end
 
       def body
@@ -85,7 +85,11 @@ module Gitlab
       end
 
       def description
-        formatter.author_line(author) + body
+        if gitlab_author_id
+          body
+        else
+          formatter.author_line(author) + body
+        end
       end
 
       def milestone
diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb
new file mode 100644
index 00000000000..73d643b00ad
--- /dev/null
+++ b/lib/gitlab/github_import/release_formatter.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module GithubImport
+    class ReleaseFormatter < BaseFormatter
+      def attributes
+        {
+          project: project,
+          tag: raw_data.tag_name,
+          description: raw_data.body,
+          created_at: raw_data.created_at,
+          updated_at: raw_data.created_at
+        }
+      end
+
+      def klass
+        Release
+      end
+
+      def valid?
+        !raw_data.draft
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index 46d40f75be6..e44d7934fda 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -41,7 +41,8 @@ module Gitlab
               title: issue["title"],
               state: issue["state"],
               updated_at: issue["updated_at"],
-              author_id: gl_user_id(project, issue["author"]["id"])
+              author_id: gitlab_user_id(project, issue["author"]["id"]),
+              confidential: issue["confidential"]
             )
           end
         end
@@ -51,7 +52,7 @@ module Gitlab
 
       private
 
-      def gl_user_id(project, gitlab_id)
+      def gitlab_user_id(project, gitlab_id)
         user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s)
         (user && user.id) || project.creator_id
       end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index bb562bdcd2c..181e288a014 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -2,7 +2,8 @@ module Gitlab
   module ImportExport
     extend self
 
-    VERSION = '0.1.3'
+    # For every version update, the version history in import_export.md has to be kept up to date.
+    VERSION = '0.1.4'
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index c2e8a1ca5dd..925a952156f 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -35,7 +35,9 @@ project_tree:
   - :deploy_keys
   - :services
   - :hooks
-  - :protected_branches
+  - protected_branches:
+    - :merge_access_levels
+    - :push_access_levels
   - :labels
   - milestones:
     - :events
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index b0726268ca6..354ccd64696 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -7,7 +7,9 @@ module Gitlab
                     variables: 'Ci::Variable',
                     triggers: 'Ci::Trigger',
                     builds: 'Ci::Build',
-                    hooks: 'ProjectHook' }.freeze
+                    hooks: 'ProjectHook',
+                    merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
+                    push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
 
@@ -17,6 +19,8 @@ module Gitlab
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze
 
+      FINDER_ATTRIBUTES = %w[title project_id].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -149,7 +153,7 @@ module Gitlab
       end
 
       def parsed_relation_hash
-        @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+        @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
       end
 
       def set_st_diffs
@@ -161,14 +165,30 @@ module Gitlab
         # Otherwise always create the record, skipping the extra SELECT clause.
         @existing_or_new_object ||= begin
           if EXISTING_OBJECT_CHECK.include?(@relation_name)
-            existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id'))
-            existing_object.assign_attributes(parsed_relation_hash)
+            events = parsed_relation_hash.delete('events')
+
+            unless events.blank?
+              existing_object.assign_attributes(events: events)
+            end
+
             existing_object
           else
             relation_class.new(parsed_relation_hash)
           end
         end
       end
+
+      def existing_object
+        @existing_object ||=
+          begin
+            finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES)
+            existing_object = relation_class.find_or_create_by(finder_hash)
+            # Done in two steps, as MySQL behaves differently than PostgreSQL using
+            # the +find_or_create_by+ method and does not return the ID the second time.
+            existing_object.update(parsed_relation_hash)
+            existing_object
+          end
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 6d9379acf25..d1e33ea8678 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -22,10 +22,6 @@ module Gitlab
 
       private
 
-      def repos_path
-        Gitlab.config.gitlab_shell.repos_path
-      end
-
       def path_to_repo
         @project.repository.path_to_repo
       end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index de3fe6d822e..fc08082fc86 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -24,8 +24,8 @@ module Gitlab
       end
 
       def verify_version!(version)
-        if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
-          raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+        if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
+          raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}")
         else
           true
         end
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 9a5bcfb5c9b..82cb8cef754 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -23,31 +23,7 @@ module Gitlab
       end
 
       def users(field, value, limit = nil)
-        if field.to_sym == :dn
-          options = {
-            base: value,
-            scope: Net::LDAP::SearchScope_BaseObject
-          }
-        else
-          options = {
-            base: config.base,
-            filter: Net::LDAP::Filter.eq(field, value)
-          }
-        end
-
-        if config.user_filter.present?
-          user_filter = Net::LDAP::Filter.construct(config.user_filter)
-
-          options[:filter] = if options[:filter]
-                               Net::LDAP::Filter.join(options[:filter], user_filter)
-                             else
-                               user_filter
-                             end
-        end
-
-        if limit.present?
-          options.merge!(size: limit)
-        end
+        options = user_options(field, value, limit)
 
         entries = ldap_search(options).select do |entry|
           entry.respond_to? config.uid
@@ -90,6 +66,42 @@ module Gitlab
         Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds")
         []
       end
+
+      private
+
+      def user_options(field, value, limit)
+        options = { attributes: user_attributes }
+        options[:size] = limit if limit
+
+        if field.to_sym == :dn
+          options[:base] = value
+          options[:scope] = Net::LDAP::SearchScope_BaseObject
+          options[:filter] = user_filter
+        else
+          options[:base] = config.base
+          options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value))
+        end
+
+        options
+      end
+
+      def user_filter(filter = nil)
+        if config.user_filter.present?
+          user_filter = Net::LDAP::Filter.construct(config.user_filter)
+        end
+
+        if user_filter && filter
+          Net::LDAP::Filter.join(filter, user_filter)
+        elsif user_filter
+          user_filter
+        else
+          filter
+        end
+      end
+
+      def user_attributes
+        %W(#{config.uid} cn mail dn)
+      end
     end
   end
 end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index a0fd41161a5..cc74bb29087 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -18,18 +18,18 @@ module Gitlab
         FileUtils.mkdir_p(path)
       end
 
-      @cmd_output = ""
-      @cmd_status = 0
+      cmd_output = ""
+      cmd_status = 0
       Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
         yield(stdin) if block_given?
         stdin.close
 
-        @cmd_output << stdout.read
-        @cmd_output << stderr.read
-        @cmd_status = wait_thr.value.exitstatus
+        cmd_output << stdout.read
+        cmd_output << stderr.read
+        cmd_status = wait_thr.value.exitstatus
       end
 
-      [@cmd_output, @cmd_status]
+      [cmd_output, cmd_status]
     end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index ffad5e17c78..776bbcbb5d0 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -44,7 +44,7 @@ module Gitlab
     end
 
     def file_name_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'. "
+      "can contain only letters, digits, '_', '-', '@' and '.'."
     end
 
     def file_path_regex
@@ -52,7 +52,7 @@ module Gitlab
     end
 
     def file_path_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. "
+      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'."
     end
 
     def directory_traversal_regex
@@ -60,7 +60,7 @@ module Gitlab
     end
 
     def directory_traversal_regex_message
-      "cannot include directory traversal. "
+      "cannot include directory traversal."
     end
 
     def archive_formats_regex
@@ -96,11 +96,11 @@ module Gitlab
     end
 
     def environment_name_regex
-      @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze
+      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
     end
 
     def environment_name_regex_message
-      "can contain only letters, digits, '-' and '_'."
+      "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces"
     end
   end
 end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index c6826a09bd2..60aae541d46 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -1,19 +1,38 @@
 require 'base64'
 require 'json'
+require 'securerandom'
 
 module Gitlab
   class Workhorse
     SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
     VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
+    INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'
+    INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'
+
+    # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32
+    # bytes https://tools.ietf.org/html/rfc4868#section-2.6
+    SECRET_LENGTH = 32
 
     class << self
       def git_http_ok(repository, user)
         {
-          'GL_ID' => Gitlab::GlId.gl_id(user),
-          'RepoPath' => repository.path_to_repo,
+          GL_ID: Gitlab::GlId.gl_id(user),
+          RepoPath: repository.path_to_repo,
         }
       end
 
+      def lfs_upload_ok(oid, size)
+        {
+          StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+          LfsOid: oid,
+          LfsSize: size,
+        }
+      end
+
+      def artifact_upload_ok
+        { TempPath: ArtifactUploader.artifacts_upload_path }
+      end
+
       def send_git_blob(repository, blob)
         params = {
           'RepoPath' => repository.path_to_repo,
@@ -81,6 +100,35 @@ module Gitlab
         path.readable? ? path.read.chomp : 'unknown'
       end
 
+      def secret
+        @secret ||= begin
+          bytes = Base64.strict_decode64(File.read(secret_path).chomp)
+          raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH
+          bytes
+        end
+      end
+      
+      def write_secret
+        bytes = SecureRandom.random_bytes(SECRET_LENGTH)
+        File.open(secret_path, 'w:BINARY', 0600) do |f| 
+          f.chmod(0600)
+          f.write(Base64.strict_encode64(bytes))
+        end
+      end
+      
+      def verify_api_request!(request_headers)
+        JWT.decode(
+          request_headers[INTERNAL_API_REQUEST_HEADER],
+          secret,
+          true,
+          { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' },
+        )
+      end
+
+      def secret_path
+        Rails.root.join('.gitlab_workhorse_secret')
+      end
+      
       protected
 
       def encode(hash)
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
new file mode 100644
index 00000000000..609dfaa48e3
--- /dev/null
+++ b/lib/tasks/haml-lint.rake
@@ -0,0 +1,5 @@
+unless Rails.env.production?
+  require 'haml_lint/rake_task'
+
+  HamlLint::RakeTask.new
+end