Add GitLab Pages

- The pages are created when build artifacts for `pages` job are uploaded
- Pages serve the content under: http://group.pages.domain.com/project
- Pages can be used to serve the group page, special project named as host: group.pages.domain.com
- User can provide own 403 and 404 error pages by creating 403.html and 404.html in group page project
- Pages can be explicitly removed from the project by clicking Remove Pages in Project Settings
- The size of pages is limited by Application Setting: max pages size, which limits the maximum size of unpacked archive (default: 100MB)
- The public/ is extracted from artifacts and content is served as static pages
- Pages asynchronous worker use `dd` to limit the unpacked tar size
- Pages needs to be explicitly enabled and domain needs to be specified in gitlab.yml
- Pages are part of backups
- Pages notify the deployment status using Commit Status API
- Pages use a new sidekiq queue: pages
- Pages use a separate nginx config which needs to be explicitly added

1 files changed, 123 insertions, 0 deletions

diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb
new file mode 100644
index 00000000000..9aa3030264b
--- /dev/null
+++ b/app/workers/pages_worker.rb
@@ -0,0 +1,123 @@
+class PagesWorker
+  include Sidekiq::Worker
+  include Gitlab::CurrentSettings
+
+  BLOCK_SIZE = 32.kilobytes
+  MAX_SIZE = 1.terabyte
+
+  sidekiq_options queue: :pages
+
+  def perform(build_id)
+    @build_id = build_id
+    return unless valid?
+
+    # Create status notifying the deployment of pages
+    @status = GenericCommitStatus.new(
+      project: project,
+      commit: build.commit,
+      user: build.user,
+      ref: build.ref,
+      stage: 'deploy',
+      name: 'pages:deploy'
+    )
+    @status.run!
+
+    FileUtils.mkdir_p(tmp_path)
+
+    # Calculate dd parameters: we limit the size of pages
+    max_size = current_application_settings.max_pages_size.megabytes
+    max_size ||= MAX_SIZE
+    blocks = 1 + max_size / BLOCK_SIZE
+
+    # Create temporary directory in which we will extract the artifacts
+    Dir.mktmpdir(nil, tmp_path) do |temp_path|
+      # We manually extract the archive and limit the archive size with dd
+      results = Open3.pipeline(%W(gunzip -c #{artifacts}),
+                               %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
+                               %W(tar -x -C #{temp_path} public/))
+      return unless results.compact.all?(&:success?)
+
+      # Check if we did extract public directory
+      temp_public_path = File.join(temp_path, 'public')
+      return unless Dir.exists?(temp_public_path)
+
+      FileUtils.mkdir_p(pages_path)
+
+      # Lock file for time of deployment to prevent the two processes from doing the concurrent deployment
+      File.open(lock_path, File::RDWR|File::CREAT, 0644) do |f|
+        f.flock(File::LOCK_EX)
+        return unless valid?
+
+        # Do atomic move of pages
+        # Move and removal may not be atomic, but they are significantly faster then extracting and removal
+        # 1. We move deployed public to previous public path (file removal is slow)
+        # 2. We move temporary public to be deployed public
+        # 3. We remove previous public path
+        if File.exists?(public_path)
+          FileUtils.move(public_path, previous_public_path)
+        end
+        FileUtils.move(temp_public_path, public_path)
+      end
+
+      if File.exists?(previous_public_path)
+        FileUtils.rm_r(previous_public_path, force: true)
+      end
+
+      @status.success
+    end
+  ensure
+    @status.drop if @status && @status.active?
+  end
+
+  private
+
+  def valid?
+    # check if sha for the ref is still the most recent one
+    # this helps in case when multiple deployments happens
+    build && build.artifacts_file? && sha == latest_sha
+  end
+
+  def build
+    @build ||= Ci::Build.find_by(id: @build_id)
+  end
+
+  def project
+    @project ||= build.project
+  end
+
+  def tmp_path
+    @tmp_path ||= File.join(Settings.pages.path, 'tmp')
+  end
+
+  def pages_path
+    @pages_path ||= project.pages_path
+  end
+
+  def public_path
+    @public_path ||= File.join(pages_path, 'public')
+  end
+
+  def previous_public_path
+    @previous_public_path ||= File.join(pages_path, "public.#{SecureRandom.hex}")
+  end
+
+  def lock_path
+    @lock_path ||= File.join(pages_path, 'deploy.lock')
+  end
+
+  def ref
+    build.ref
+  end
+
+  def artifacts
+    build.artifacts_file.path
+  end
+
+  def latest_sha
+    project.commit(build.ref).try(:sha).to_s
+  end
+
+  def sha
+    build.sha
+  end
+end