Add GitLab Pages

- The pages are created when build artifacts for `pages` job are uploaded
- Pages serve the content under: http://group.pages.domain.com/project
- Pages can be used to serve the group page, special project named as host: group.pages.domain.com
- User can provide own 403 and 404 error pages by creating 403.html and 404.html in group page project
- Pages can be explicitly removed from the project by clicking Remove Pages in Project Settings
- The size of pages is limited by Application Setting: max pages size, which limits the maximum size of unpacked archive (default: 100MB)
- The public/ is extracted from artifacts and content is served as static pages
- Pages asynchronous worker use `dd` to limit the unpacked tar size
- Pages needs to be explicitly enabled and domain needs to be specified in gitlab.yml
- Pages are part of backups
- Pages notify the deployment status using Commit Status API
- Pages use a new sidekiq queue: pages
- Pages use a separate nginx config which needs to be explicitly added

9 files changed, 220 insertions, 1 deletions

diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb
index 543d5eac504..df8682e246e 100644
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :plantuml_url,
       :max_artifacts_size,
       :max_attachment_size,
+      :max_pages_size,
       :metrics_enabled,
       :metrics_host,
       :metrics_method_call_threshold,
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index 444ff837bb3..123dc179e73 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -151,6 +151,16 @@ class ProjectsController < Projects::ApplicationController
     end
   end
 
+  def remove_pages
+    return access_denied! unless can?(current_user, :remove_pages, @project)
+
+    @project.remove_pages
+
+    respond_to do |format|
+      format.html { redirect_to project_path(@project) }
+    end
+  end
+
   def housekeeping
     ::Projects::HousekeepingService.new(@project).execute
 
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 5fe8ddf69d7..095a346f337 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -256,7 +256,7 @@ module Ci
     end
 
     def project_id
-      pipeline.project_id
+      gl_project_id
     end
 
     def project_name
@@ -457,6 +457,7 @@ module Ci
       build_data = Gitlab::DataBuilder::Build.build(self)
       project.execute_hooks(build_data.dup, :build_hooks)
       project.execute_services(build_data.dup, :build_hooks)
+      UpdatePagesService.new(build_data).execute
       project.running_or_pending_build_count(force: true)
     end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index 37f4705adbd..48ff5ec7fc7 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -53,6 +53,8 @@ class Project < ActiveRecord::Base
     update_column(:last_activity_at, self.created_at)
   end
 
+  after_destroy :remove_pages
+
   # update visibility_level of forks
   after_update :update_forks_visibility_level
   def update_forks_visibility_level
@@ -1160,6 +1162,29 @@ class Project < ActiveRecord::Base
     ensure_runners_token!
   end
 
+  def pages_url
+    if Dir.exist?(public_pages_path)
+      host = "#{namespace.path}.#{Settings.pages.domain}"
+
+      # If the project path is the same as host, leave the short version
+      return "http://#{host}" if host == path
+
+      "http://#{host}/#{path}"
+    end
+  end
+
+  def pages_path
+    File.join(Settings.pages.path, path_with_namespace)
+  end
+
+  def public_pages_path
+    File.join(pages_path, 'public')
+  end
+
+  def remove_pages
+    FileUtils.rm_r(pages_path, force: true)
+  end
+
   def wiki
     @wiki ||= ProjectWiki.new(self, self.owner)
   end
diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb
index 71ef8901932..63bc639688d 100644
--- a/app/policies/project_policy.rb
+++ b/app/policies/project_policy.rb
@@ -136,6 +136,7 @@ class ProjectPolicy < BasePolicy
     can! :remove_fork_project
     can! :destroy_merge_request
     can! :destroy_issue
+    can! :remove_pages
   end
 
   def team_member_owner_access!
diff --git a/app/services/update_pages_service.rb b/app/services/update_pages_service.rb
new file mode 100644
index 00000000000..818bb94a293
--- /dev/null
+++ b/app/services/update_pages_service.rb
@@ -0,0 +1,15 @@
+class UpdatePagesService
+  attr_reader :data
+
+  def initialize(data)
+    @data = data
+  end
+
+  def execute
+    return unless Settings.pages.enabled
+    return unless data[:build_name] == 'pages'
+    return unless data[:build_status] == 'success'
+
+    PagesWorker.perform_async(data[:build_id])
+  end
+end
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 558bbe07b16..125a805a897 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -187,6 +187,14 @@
         .help-block Markdown enabled
 
   %fieldset
+    %legend Pages
+    .form-group
+      = f.label :max_pages_size, 'Maximum size of pages (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_pages_size, class: 'form-control'
+        .help-block Zero for unlimited
+
+  %fieldset
     %legend Continuous Integration
     .form-group
       .col-sm-offset-2.col-sm-10
diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml
index ec944d4ffb7..89e2d4046b8 100644
--- a/app/views/projects/edit.html.haml
+++ b/app/views/projects/edit.html.haml
@@ -133,6 +133,41 @@
               %hr
               = link_to 'Remove avatar', namespace_project_avatar_path(@project.namespace, @project), data: { confirm: "Project avatar will be removed. Are you sure?"}, method: :delete, class: "btn btn-remove btn-sm remove-avatar"
         = f.submit 'Save changes', class: "btn btn-save"
+
+    - if Settings.pages.enabled
+      .pages-settings
+        .panel.panel-default
+          .panel-heading Pages
+          .errors-holder
+          .panel-body
+            - if @project.pages_url
+              %strong
+                Congratulations. Your pages are served at:
+                %p= link_to @project.pages_url, @project.pages_url
+            - else
+              %p
+                To publish pages create .gitlab-ci.yml with
+                %strong pages job
+                and send public/ folder to GitLab.
+              %p
+                Use existing tools:
+                %ul
+                  %li
+                    %pre
+                      :plain
+                        pages:
+                          image: jekyll
+                          script: jekyll build
+                          artifacts:
+                            paths:
+                            - public
+
+            - if @project.pages_url && can?(current_user, :remove_pages, @project)
+              .form-actions
+                = link_to 'Remove pages', remove_pages_namespace_project_path(@project.namespace, @project),
+                    data: { confirm: "Are you sure that you want to remove pages for this project?" },
+                    method: :post, class: "btn btn-warning"
+
   .row.prepend-top-default
   %hr
   .row.prepend-top-default
diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb
new file mode 100644
index 00000000000..9aa3030264b
--- /dev/null
+++ b/app/workers/pages_worker.rb
@@ -0,0 +1,123 @@
+class PagesWorker
+  include Sidekiq::Worker
+  include Gitlab::CurrentSettings
+
+  BLOCK_SIZE = 32.kilobytes
+  MAX_SIZE = 1.terabyte
+
+  sidekiq_options queue: :pages
+
+  def perform(build_id)
+    @build_id = build_id
+    return unless valid?
+
+    # Create status notifying the deployment of pages
+    @status = GenericCommitStatus.new(
+      project: project,
+      commit: build.commit,
+      user: build.user,
+      ref: build.ref,
+      stage: 'deploy',
+      name: 'pages:deploy'
+    )
+    @status.run!
+
+    FileUtils.mkdir_p(tmp_path)
+
+    # Calculate dd parameters: we limit the size of pages
+    max_size = current_application_settings.max_pages_size.megabytes
+    max_size ||= MAX_SIZE
+    blocks = 1 + max_size / BLOCK_SIZE
+
+    # Create temporary directory in which we will extract the artifacts
+    Dir.mktmpdir(nil, tmp_path) do |temp_path|
+      # We manually extract the archive and limit the archive size with dd
+      results = Open3.pipeline(%W(gunzip -c #{artifacts}),
+                               %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
+                               %W(tar -x -C #{temp_path} public/))
+      return unless results.compact.all?(&:success?)
+
+      # Check if we did extract public directory
+      temp_public_path = File.join(temp_path, 'public')
+      return unless Dir.exists?(temp_public_path)
+
+      FileUtils.mkdir_p(pages_path)
+
+      # Lock file for time of deployment to prevent the two processes from doing the concurrent deployment
+      File.open(lock_path, File::RDWR|File::CREAT, 0644) do |f|
+        f.flock(File::LOCK_EX)
+        return unless valid?
+
+        # Do atomic move of pages
+        # Move and removal may not be atomic, but they are significantly faster then extracting and removal
+        # 1. We move deployed public to previous public path (file removal is slow)
+        # 2. We move temporary public to be deployed public
+        # 3. We remove previous public path
+        if File.exists?(public_path)
+          FileUtils.move(public_path, previous_public_path)
+        end
+        FileUtils.move(temp_public_path, public_path)
+      end
+
+      if File.exists?(previous_public_path)
+        FileUtils.rm_r(previous_public_path, force: true)
+      end
+
+      @status.success
+    end
+  ensure
+    @status.drop if @status && @status.active?
+  end
+
+  private
+
+  def valid?
+    # check if sha for the ref is still the most recent one
+    # this helps in case when multiple deployments happens
+    build && build.artifacts_file? && sha == latest_sha
+  end
+
+  def build
+    @build ||= Ci::Build.find_by(id: @build_id)
+  end
+
+  def project
+    @project ||= build.project
+  end
+
+  def tmp_path
+    @tmp_path ||= File.join(Settings.pages.path, 'tmp')
+  end
+
+  def pages_path
+    @pages_path ||= project.pages_path
+  end
+
+  def public_path
+    @public_path ||= File.join(pages_path, 'public')
+  end
+
+  def previous_public_path
+    @previous_public_path ||= File.join(pages_path, "public.#{SecureRandom.hex}")
+  end
+
+  def lock_path
+    @lock_path ||= File.join(pages_path, 'deploy.lock')
+  end
+
+  def ref
+    build.ref
+  end
+
+  def artifacts
+    build.artifacts_file.path
+  end
+
+  def latest_sha
+    project.commit(build.ref).try(:sha).to_s
+  end
+
+  def sha
+    build.sha
+  end
+end