diff options
35 files changed, 796 insertions, 87 deletions
diff --git a/.gitignore b/.gitignore index 3ffe4263c4f..b44cf74bc06 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ eslint-report.html /builds* /shared/* /.gitlab_workhorse_secret +/.gitlab_pages_shared_secret /webpack-report/ /knapsack/ /rspec_flaky/ diff --git a/app/models/generic_commit_status.rb b/app/models/generic_commit_status.rb index 8a768b3a2c0..fb339377571 100644 --- a/app/models/generic_commit_status.rb +++ b/app/models/generic_commit_status.rb @@ -3,6 +3,8 @@ class GenericCommitStatus < CommitStatus before_validation :set_default_values + scope :successful_pages_deploy, -> { success.where(stage: 'deploy', name: 'pages:deploy') } + validates :target_url, addressable_url: true, length: { maximum: 255 }, allow_nil: true diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9f9c4288667..2b2194aaaa1 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -120,6 +120,13 @@ class Namespace < ApplicationRecord uniquify = Uniquify.new uniquify.string(path) { |s| Namespace.find_by_path_or_name(s) } end + + def find_by_pages_host(host) + gitlab_host = "." + Settings.pages.host.downcase + name = host.downcase.delete_suffix(gitlab_host) + + Namespace.find_by_full_path(name) + end end def visibility_level_field @@ -305,8 +312,17 @@ class Namespace < ApplicationRecord aggregation_schedule.present? end + def pages_virtual_domain + Pages::VirtualDomain.new(all_projects_with_pages, trim_prefix: full_path) + end + private + def all_projects_with_pages + all_projects.migrate_project_pages_metadata + all_projects.with_pages_deployed + end + def parent_changed? parent_id_changed? end diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb new file mode 100644 index 00000000000..51c496c77d3 --- /dev/null +++ b/app/models/pages/lookup_path.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Pages + class LookupPath + def initialize(project, trim_prefix: nil, domain: nil) + @project = project + @domain = domain + @trim_prefix = trim_prefix || project.full_path + end + + def project_id + project.id + end + + def access_control + project.private_pages? + end + + def https_only + domain_https = domain ? domain.https? : true + project.pages_https_only? && domain_https + end + + def source + { + type: 'file', + path: File.join(project.full_path, 'public/') + } + end + + def prefix + if project.pages_group_root? + '/' + else + project.full_path.delete_prefix(trim_prefix) + '/' + end + end + + private + + attr_reader :project, :trim_prefix, :domain + end +end diff --git a/app/models/pages/virtual_domain.rb b/app/models/pages/virtual_domain.rb new file mode 100644 index 00000000000..7e42b8e6ae2 --- /dev/null +++ b/app/models/pages/virtual_domain.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Pages + class VirtualDomain + def initialize(projects, trim_prefix: nil, domain: nil) + @projects = projects + @trim_prefix = trim_prefix + @domain = domain + end + + def certificate + domain&.certificate + end + + def key + domain&.key + end + + def lookup_paths + projects.map do |project| + project.pages_lookup_path(trim_prefix: trim_prefix, domain: domain) + end.sort_by(&:prefix).reverse + end + + private + + attr_reader :projects, :trim_prefix, :domain + end +end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 12ce717efd7..142967f886d 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -185,6 +185,10 @@ class PagesDomain < ApplicationRecord self.certificate_source = 'gitlab_provided' if key_changed? end + def pages_virtual_domain + Pages::VirtualDomain.new([project], domain: self) + end + private def set_verification_code diff --git a/app/models/project.rb b/app/models/project.rb index 17b52d0578e..a1a5e36e5f8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -61,11 +61,11 @@ class Project < ApplicationRecord cache_markdown_field :description, pipeline: :description - delegate :feature_available?, :builds_enabled?, :wiki_enabled?, - :merge_requests_enabled?, :issues_enabled?, :pages_enabled?, :public_pages?, - :merge_requests_access_level, :issues_access_level, :wiki_access_level, - :snippets_access_level, :builds_access_level, :repository_access_level, - to: :project_feature, allow_nil: true + delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, + :issues_enabled?, :pages_enabled?, :public_pages?, :private_pages?, + :merge_requests_access_level, :issues_access_level, :wiki_access_level, + :snippets_access_level, :builds_access_level, :repository_access_level, + to: :project_feature, allow_nil: true delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage @@ -291,6 +291,8 @@ class Project < ApplicationRecord has_many :remote_mirrors, inverse_of: :project has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage' + has_one :project_pages_metadatum, inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data @@ -421,6 +423,14 @@ class Project < ApplicationRecord .where(project_ci_cd_settings: { group_runners_enabled: true }) end + scope :project_pages_metadata_not_migrated, -> do + where('NOT EXISTS (SELECT 1 FROM project_pages_metadata WHERE projects.id=project_pages_metadata.project_id)') + end + + scope :with_pages_deployed, -> do + where('EXISTS (SELECT 1 FROM project_pages_metadata WHERE projects.id=project_pages_metadata.project_id AND deployed = TRUE)') + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -593,6 +603,25 @@ class Project < ApplicationRecord from_union([with_issues_enabled, with_merge_requests_enabled]).select(:id) end + + def migrate_project_pages_metadata + successful_pages_deploy = GenericCommitStatus + .successful_pages_deploy + .select('TRUE') + .where("ci_builds.project_id = projects.id").limit(1) + .to_sql + + select_from = project_pages_metadata_not_migrated + .select("projects.id, COALESCE((#{successful_pages_deploy}), FALSE), NOW(), NOW()") + .to_sql + + connection_pool.with_connection do |connection| + connection.execute <<~INSERT_SQL + INSERT INTO project_pages_metadata (project_id, deployed, created_at, updated_at) + #{select_from} + INSERT_SQL + end + end end def initialize(attributes = nil) @@ -1630,6 +1659,10 @@ class Project < ApplicationRecord "#{url}/#{url_path}" end + def pages_group_root? + pages_group_url == pages_url + end + def pages_subdomain full_path.partition('/').first end @@ -1668,6 +1701,7 @@ class Project < ApplicationRecord # Projects with a missing namespace cannot have their pages removed return unless namespace + mark_pages_as_not_deployed unless destroyed? ::Projects::UpdatePagesConfigurationService.new(self).execute # 1. We rename pages to temporary directory @@ -1681,6 +1715,14 @@ class Project < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass + def mark_pages_as_deployed + update_pages_deployed(true) + end + + def mark_pages_as_not_deployed + update_pages_deployed(false) + end + # rubocop:disable Gitlab/RailsLogger def write_repository_config(gl_full_path: full_path) # We'd need to keep track of project full path otherwise directory tree @@ -2199,6 +2241,10 @@ class Project < ApplicationRecord members.maintainers.order_recent_sign_in.limit(ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end + def pages_lookup_path(trim_prefix: nil, domain: nil) + Pages::LookupPath.new(self, trim_prefix: trim_prefix, domain: domain) + end + private def merge_requests_allowing_collaboration(source_branch = nil) @@ -2324,4 +2370,19 @@ class Project < ApplicationRecord def services_templates @services_templates ||= Service.where(template: true) end + + def update_pages_deployed(flag) + flag = flag ? 'TRUE' : 'FALSE' + + upsert = <<~SQL + INSERT INTO project_pages_metadata (project_id, deployed, created_at, updated_at) + VALUES (#{id}, #{flag}, NOW(), NOW()) + ON CONFLICT (project_id) DO UPDATE + SET deployed = EXCLUDED.deployed, updated_at = NOW() + SQL + + self.class.connection_pool.with_connection do |connection| + connection.execute(upsert) + end + end end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 78e82955342..efa3fbcf015 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -129,6 +129,10 @@ class ProjectFeature < ApplicationRecord pages_access_level == PUBLIC || pages_access_level == ENABLED && project.public? end + def private_pages? + !public_pages? + end + private # Validates builds and merge requests access level diff --git a/app/models/project_pages_metadatum.rb b/app/models/project_pages_metadatum.rb new file mode 100644 index 00000000000..85471a32f2c --- /dev/null +++ b/app/models/project_pages_metadatum.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ProjectPagesMetadatum < ApplicationRecord + belongs_to :project, inverse_of: :project_pages_metadatum +end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 5caeb4cfa5f..6fc676599e2 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -53,6 +53,7 @@ module Projects def success @status.success + @project.mark_pages_as_deployed super end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 20b1020e025..1f0b3065adf 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -320,6 +320,9 @@ production: &base # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages admin: address: unix:/home/git/gitlab/tmp/sockets/private/pages-admin.socket # TCP connections are supported too (e.g. tcp://host:port) + # File that contains the shared secret key for verifying access for gitlab-pages. + # Default is '.gitlab_pages_shared_secret' relative to Rails.root (i.e. root of the GitLab app). + # secret_file: /home/git/gitlab/.gitlab_pages_shared_secret ## Mattermost ## For enabling Add to Mattermost button diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 4160f488a7a..dbbb7ba1b60 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -292,6 +292,7 @@ Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pa Settings.pages['admin'] ||= Settingslogic.new({}) Settings.pages.admin['certificate'] ||= '' +Settings.pages['secret_file'] ||= Rails.root.join('.gitlab_pages_shared_secret') # # Geo diff --git a/db/migrate/20190716121002_add_successfull_pages_deploy_partial_index_on_ci_builds.rb b/db/migrate/20190716121002_add_successfull_pages_deploy_partial_index_on_ci_builds.rb new file mode 100644 index 00000000000..a0fd70ec4f0 --- /dev/null +++ b/db/migrate/20190716121002_add_successfull_pages_deploy_partial_index_on_ci_builds.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddSuccessfullPagesDeployPartialIndexOnCiBuilds < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index( + :ci_builds, :project_id, + name: 'index_ci_builds_on_project_id_for_successfull_pages_deploy', + where: "type='GenericCommitStatus' AND stage='deploy' AND name='pages:deploy' AND status = 'success'" + ) + end + + def down + remove_concurrent_index_by_name :ci_builds, 'index_ci_builds_on_project_id_for_successfull_pages_deploy' + end +end diff --git a/db/migrate/20190806112508_create_project_pages_metadata.rb b/db/migrate/20190806112508_create_project_pages_metadata.rb new file mode 100644 index 00000000000..849b942d55c --- /dev/null +++ b/db/migrate/20190806112508_create_project_pages_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class CreateProjectPagesMetadata < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :project_pages_metadata do |t| + t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }, type: :integer + t.boolean :deployed, null: false, default: false, index: true + + t.timestamps_with_timezone null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 5b89cdf0b98..b61be7da7dc 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -599,6 +599,7 @@ ActiveRecord::Schema.define(version: 2019_09_02_131045) do t.index ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))" t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id" t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))" + t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))" t.index ["protected"], name: "index_ci_builds_on_protected" t.index ["queued_at"], name: "index_ci_builds_on_queued_at" t.index ["runner_id"], name: "index_ci_builds_on_runner_id" @@ -2686,6 +2687,15 @@ ActiveRecord::Schema.define(version: 2019_09_02_131045) do t.index ["status"], name: "index_project_mirror_data_on_status" end + create_table "project_pages_metadata", force: :cascade do |t| + t.integer "project_id", null: false + t.boolean "deployed", default: false, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.index ["deployed"], name: "index_project_pages_metadata_on_deployed" + t.index ["project_id"], name: "index_project_pages_metadata_on_project_id", unique: true + end + create_table "project_repositories", force: :cascade do |t| t.integer "shard_id", null: false t.string "disk_path", null: false @@ -3976,6 +3986,7 @@ ActiveRecord::Schema.define(version: 2019_09_02_131045) do add_foreign_key "project_incident_management_settings", "projects", on_delete: :cascade add_foreign_key "project_metrics_settings", "projects", on_delete: :cascade add_foreign_key "project_mirror_data", "projects", on_delete: :cascade + add_foreign_key "project_pages_metadata", "projects", on_delete: :cascade add_foreign_key "project_repositories", "projects", on_delete: :cascade add_foreign_key "project_repositories", "shards", on_delete: :restrict add_foreign_key "project_repository_states", "projects", on_delete: :cascade diff --git a/lib/api/api.rb b/lib/api/api.rb index aa6a67d817a..de6e528ed09 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -119,6 +119,7 @@ module API mount ::API::GroupVariables mount ::API::ImportGithub mount ::API::Internal::Base + mount ::API::Internal::Pages mount ::API::Issues mount ::API::JobArtifacts mount ::API::Jobs diff --git a/lib/api/entities/internal.rb b/lib/api/entities/internal.rb new file mode 100644 index 00000000000..8f79bd14833 --- /dev/null +++ b/lib/api/entities/internal.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Entities + module Internal + module Pages + class LookupPath < Grape::Entity + expose :project_id, :access_control, + :source, :https_only, :prefix + end + + class VirtualDomain < Grape::Entity + expose :certificate, :key + expose :lookup_paths, using: LookupPath + end + end + end + end +end diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb new file mode 100644 index 00000000000..6b6ccce0275 --- /dev/null +++ b/lib/api/internal/pages.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module API + # Internal access API + module Internal + class Pages < Grape::API + before do + not_found! unless Feature.enabled?(:pages_internal_api) + authenticate_gitlab_pages_request! + end + + helpers do + def authenticate_gitlab_pages_request! + unauthorized! unless Gitlab::Pages.verify_api_request(headers) + end + end + + namespace 'internal' do + namespace 'pages' do + desc 'Get GitLab Pages domain configuration by hostname' do + detail 'This feature was introduced in GitLab 12.2.' + end + params do + requires :host, type: String, desc: 'The host to query for' + end + get "/" do + host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host]) + not_found! unless host + + present host.pages_virtual_domain, with: Entities::Internal::Pages::VirtualDomain + end + end + end + end + end +end diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb new file mode 100644 index 00000000000..1270a148e8d --- /dev/null +++ b/lib/gitlab/jwt_authenticatable.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module JwtAuthenticatable + # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 + # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + SECRET_LENGTH = 32 + + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + include Gitlab::Utils::StrongMemoize + + def decode_jwt_for_issuer(issuer, encoded_message) + JWT.decode( + encoded_message, + secret, + true, + { iss: issuer, verify_iss: true, algorithm: 'HS256' } + ) + end + + def secret + strong_memoize(:secret) do + Base64.strict_decode64(File.read(secret_path).chomp).tap do |bytes| + raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + end + end + end + + def write_secret + bytes = SecureRandom.random_bytes(SECRET_LENGTH) + File.open(secret_path, 'w:BINARY', 0600) do |f| + f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. + f.write(Base64.strict_encode64(bytes)) + end + end + end + end +end diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 16df0700b08..4899b1d3234 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -1,7 +1,22 @@ # frozen_string_literal: true module Gitlab - module Pages + class Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze + + include JwtAuthenticatable + + class << self + def verify_api_request(request_headers) + decode_jwt_for_issuer('gitlab-pages', request_headers[INTERNAL_API_REQUEST_HEADER]) + rescue JWT::DecodeError + false + end + + def secret_path + Gitlab.config.pages.secret_file + end + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 29087d26007..139ec6e384a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -15,9 +15,7 @@ module Gitlab ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type'.freeze - # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 - # bytes https://tools.ietf.org/html/rfc4868#section-2.6 - SECRET_LENGTH = 32 + include JwtAuthenticatable class << self def git_http_ok(repository, repo_type, user, action, show_all_refs: false) @@ -187,34 +185,12 @@ module Gitlab path.readable? ? path.read.chomp : 'unknown' end - def secret - @secret ||= begin - bytes = Base64.strict_decode64(File.read(secret_path).chomp) - raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH - - bytes - end - end - - def write_secret - bytes = SecureRandom.random_bytes(SECRET_LENGTH) - File.open(secret_path, 'w:BINARY', 0600) do |f| - f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. - f.write(Base64.strict_encode64(bytes)) - end - end - def verify_api_request!(request_headers) decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) end def decode_jwt(encoded_message) - JWT.decode( - encoded_message, - secret, - true, - { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' } - ) + decode_jwt_for_issuer('gitlab-workhorse', encoded_message) end def secret_path diff --git a/spec/factories/project_pages_metadata.rb b/spec/factories/project_pages_metadata.rb new file mode 100644 index 00000000000..30efebb5f85 --- /dev/null +++ b/spec/factories/project_pages_metadata.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :project_pages_metadatum, class: ProjectPagesMetadatum do + project + deployed false + end +end diff --git a/spec/features/projects/pages_spec.rb b/spec/features/projects/pages_spec.rb index d55e9d12801..10a0d68bf26 100644 --- a/spec/features/projects/pages_spec.rb +++ b/spec/features/projects/pages_spec.rb @@ -311,6 +311,7 @@ shared_examples 'pages settings editing' do result = Projects::UpdatePagesService.new(project, ci_build).execute expect(result[:status]).to eq(:success) expect(project).to be_pages_deployed + expect(project.project_pages_metadatum.deployed).to eq(true) end it 'removes the pages' do @@ -321,6 +322,7 @@ shared_examples 'pages settings editing' do click_link 'Remove pages' expect(project.pages_deployed?).to be_falsey + expect(project.project_pages_metadatum.reload.deployed).to eq(false) end end end diff --git a/spec/fixtures/api/schemas/internal/namespace_domain.json b/spec/fixtures/api/schemas/internal/namespace_domain.json new file mode 100644 index 00000000000..4370c90ac4c --- /dev/null +++ b/spec/fixtures/api/schemas/internal/namespace_domain.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "required": [ + "lookup_paths" + ], + "properties": { + "lookup_paths": { "type": "array", "items": { "$ref": "pages/lookup_path.json" } } + }, + "additionalProperties": false +} diff --git a/spec/fixtures/api/schemas/internal/pages/lookup_path.json b/spec/fixtures/api/schemas/internal/pages/lookup_path.json new file mode 100644 index 00000000000..37385ba986a --- /dev/null +++ b/spec/fixtures/api/schemas/internal/pages/lookup_path.json @@ -0,0 +1,26 @@ +{ + "type": "object", + "required": [ + "project_id", + "https_only", + "access_control", + "source", + "prefix" + ], + "properties": { + "project_id": { "type": "integer" }, + "https_only": { "type": "boolean" }, + "access_control": { "type": "boolean" }, + "source": { "type": "object", + "required": ["type", "path"], + "properties" : { + "type": { "type": "string", "enum": ["file"] }, + "path": { "type": "string" } + }, + "additionalProperties": false + }, + "prefix": { "type": "string" } + }, + "additionalProperties": false +} + diff --git a/spec/fixtures/api/schemas/internal/pages/virtual_domain.json b/spec/fixtures/api/schemas/internal/pages/virtual_domain.json new file mode 100644 index 00000000000..02df69026b0 --- /dev/null +++ b/spec/fixtures/api/schemas/internal/pages/virtual_domain.json @@ -0,0 +1,16 @@ +{ + "type": "object", + "required": [ + "lookup_paths" + ], + "optional": [ + "certificate", + "key" + ], + "properties": { + "certificate": { "type": ["string", "null"] }, + "key": { "type": ["string", "null"] }, + "lookup_paths": { "type": "array", "items": { "$ref": "lookup_path.json" } } + }, + "additionalProperties": false +} diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index ec4a6ef05b9..037ee5ee2bb 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -397,6 +397,7 @@ project: - merge_trains - designs - project_aliases +- project_pages_metadatum award_emoji: - awardable - user diff --git a/spec/lib/gitlab/jwt_authenticatable_spec.rb b/spec/lib/gitlab/jwt_authenticatable_spec.rb new file mode 100644 index 00000000000..0c1c491b308 --- /dev/null +++ b/spec/lib/gitlab/jwt_authenticatable_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::JwtAuthenticatable do + let(:test_class) do + Class.new do + include Gitlab::JwtAuthenticatable + + def self.secret_path + Rails.root.join('tmp', 'tests', '.jwt_shared_secret') + end + end + end + + before do + begin + File.delete(test_class.secret_path) + rescue Errno::ENOENT + end + + test_class.write_secret + end + + describe '.secret' do + subject(:secret) { test_class.secret } + + it 'returns 32 bytes' do + expect(secret).to be_a(String) + expect(secret.length).to eq(32) + expect(secret.encoding).to eq(Encoding::ASCII_8BIT) + end + + it 'accepts a trailing newline' do + File.open(test_class.secret_path, 'a') { |f| f.write "\n" } + + expect(secret.length).to eq(32) + end + + it 'raises an exception if the secret file cannot be read' do + File.delete(test_class.secret_path) + + expect { secret }.to raise_exception(Errno::ENOENT) + end + + it 'raises an exception if the secret file contains the wrong number of bytes' do + File.truncate(test_class.secret_path, 0) + + expect { secret }.to raise_exception(RuntimeError) + end + end + + describe '.write_secret' do + it 'uses mode 0600' do + expect(File.stat(test_class.secret_path).mode & 0777).to eq(0600) + end + + it 'writes base64 data' do + bytes = Base64.strict_decode64(File.read(test_class.secret_path)) + + expect(bytes).not_to be_empty + end + end + + describe '.decode_jwt_for_issuer' do + let(:payload) { { 'iss' => 'test_issuer' } } + + it 'accepts a correct header' do + encoded_message = JWT.encode(payload, test_class.secret, 'HS256') + + expect { test_class.decode_jwt_for_issuer('test_issuer', encoded_message) }.not_to raise_error + end + + it 'raises an error when the JWT is not signed' do + encoded_message = JWT.encode(payload, nil, 'none') + + expect { test_class.decode_jwt_for_issuer('test_issuer', encoded_message) }.to raise_error(JWT::DecodeError) + end + + it 'raises an error when the header is signed with the wrong secret' do + encoded_message = JWT.encode(payload, 'wrongsecret', 'HS256') + + expect { test_class.decode_jwt_for_issuer('test_issuer', encoded_message) }.to raise_error(JWT::DecodeError) + end + + it 'raises an error when the issuer is incorrect' do + payload['iss'] = 'somebody else' + encoded_message = JWT.encode(payload, test_class.secret, 'HS256') + + expect { test_class.decode_jwt_for_issuer('test_issuer', encoded_message) }.to raise_error(JWT::DecodeError) + end + end +end diff --git a/spec/lib/gitlab/pages_spec.rb b/spec/lib/gitlab/pages_spec.rb new file mode 100644 index 00000000000..affa2ebab2a --- /dev/null +++ b/spec/lib/gitlab/pages_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Pages do + let(:pages_shared_secret) { SecureRandom.random_bytes(Gitlab::Pages::SECRET_LENGTH) } + + before do + allow(described_class).to receive(:secret).and_return(pages_shared_secret) + end + + describe '.verify_api_request' do + let(:payload) { { 'iss' => 'gitlab-pages' } } + + it 'returns false if fails to validate the JWT' do + encoded_token = JWT.encode(payload, 'wrongsecret', 'HS256') + headers = { described_class::INTERNAL_API_REQUEST_HEADER => encoded_token } + + expect(described_class.verify_api_request(headers)).to eq(false) + end + + it 'returns the decoded JWT' do + encoded_token = JWT.encode(payload, described_class.secret, 'HS256') + headers = { described_class::INTERNAL_API_REQUEST_HEADER => encoded_token } + + expect(described_class.verify_api_request(headers)).to eq([{ "iss" => "gitlab-pages" }, { "alg" => "HS256" }]) + end + end +end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 98421cd12d3..88bc5034da5 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -200,57 +200,6 @@ describe Gitlab::Workhorse do end end - describe ".secret" do - subject { described_class.secret } - - before do - described_class.instance_variable_set(:@secret, nil) - described_class.write_secret - end - - it 'returns 32 bytes' do - expect(subject).to be_a(String) - expect(subject.length).to eq(32) - expect(subject.encoding).to eq(Encoding::ASCII_8BIT) - end - - it 'accepts a trailing newline' do - File.open(described_class.secret_path, 'a') { |f| f.write "\n" } - expect(subject.length).to eq(32) - end - - it 'raises an exception if the secret file cannot be read' do - File.delete(described_class.secret_path) - expect { subject }.to raise_exception(Errno::ENOENT) - end - - it 'raises an exception if the secret file contains the wrong number of bytes' do - File.truncate(described_class.secret_path, 0) - expect { subject }.to raise_exception(RuntimeError) - end - end - - describe ".write_secret" do - let(:secret_path) { described_class.secret_path } - before do - begin - File.delete(secret_path) - rescue Errno::ENOENT - end - - described_class.write_secret - end - - it 'uses mode 0600' do - expect(File.stat(secret_path).mode & 0777).to eq(0600) - end - - it 'writes base64 data' do - bytes = Base64.strict_decode64(File.read(secret_path)) - expect(bytes).not_to be_empty - end - end - describe '#verify_api_request!' do let(:header_key) { described_class::INTERNAL_API_REQUEST_HEADER } let(:payload) { { 'iss' => 'gitlab-workhorse' } } diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 972f26ac745..6c555d73020 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -191,6 +191,16 @@ describe Namespace do end end + describe '.find_by_pages_host' do + it 'finds namespace by GitLab Pages host and is case-insensitive' do + namespace = create(:namespace, name: 'topnamespace') + create(:namespace, name: 'annother_namespace') + host = "TopNamespace.#{Settings.pages.host.upcase}" + + expect(described_class.find_by_pages_host(host)).to eq(namespace) + end + end + describe '#ancestors_upto' do let(:parent) { create(:group) } let(:child) { create(:group, parent: parent) } diff --git a/spec/models/pages/lookup_path_spec.rb b/spec/models/pages/lookup_path_spec.rb new file mode 100644 index 00000000000..6dd8dd91d65 --- /dev/null +++ b/spec/models/pages/lookup_path_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_relative '../../../app/models/pages/lookup_path' + +describe Pages::LookupPath do + let(:project) do + double( + id: 12345, + private_pages?: true, + pages_https_only?: true, + https?: false, + full_path: 'the/full/path' + ) + end + + subject(:lookup_path) { described_class.new(project) } + + describe '#project_id' do + it 'delegates to Project#id' do + expect(lookup_path.project_id).to eq(12345) + end + end + + describe '#access_control' do + it 'delegates to Project#private_pages?' do + expect(lookup_path.access_control).to eq(true) + end + end + + describe '#https_only' do + subject(:lookup_path) { described_class.new(project, domain: domain) } + + context 'when no domain provided' do + let(:domain) { nil } + + it 'delegates to Project#pages_https_only?' do + expect(lookup_path.https_only).to eq(true) + end + end + + context 'when there is domain provided' do + let(:domain) { double(https?: false) } + + it 'takes into account the https setting of the domain' do + expect(lookup_path.https_only).to eq(false) + end + end + end + + describe '#source' do + it 'sets the source type to "file"' do + expect(lookup_path.source[:type]).to eq('file') + end + + it 'sets the source path to the project full path suffixed with "public/' do + expect(lookup_path.source[:path]).to eq('the/full/path/public/') + end + end + + describe '#prefix' do + it 'returns "/" for pages group root projects' do + project = double(pages_group_root?: true) + lookup_path = described_class.new(project, trim_prefix: 'mygroup') + + expect(lookup_path.prefix).to eq('/') + end + + it 'returns the project full path wth the provided prefix removed' do + project = double(pages_group_root?: false, full_path: 'mygroup/myproject') + lookup_path = described_class.new(project, trim_prefix: 'mygroup') + + expect(lookup_path.prefix).to eq('/myproject/') + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index bfbcac60fea..0b25d9baec6 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3489,6 +3489,7 @@ describe Project do describe '#remove_pages' do let(:project) { create(:project) } + let(:project_pages_metadatum) { create(:project_pages_metadatum, project: project, deployed: true) } let(:namespace) { project.namespace } let(:pages_path) { project.pages_path } @@ -3501,12 +3502,12 @@ describe Project do end end - it 'removes the pages directory' do + it 'removes the pages directory and marks the project as not having pages deployed' do expect_any_instance_of(Projects::UpdatePagesConfigurationService).to receive(:execute) expect_any_instance_of(Gitlab::PagesTransfer).to receive(:rename_project).and_return(true) expect(PagesWorker).to receive(:perform_in).with(5.minutes, :remove, namespace.full_path, anything) - project.remove_pages + expect { project.remove_pages }.to change { project_pages_metadatum.reload.deployed }.from(true).to(false) end it 'is a no-op when there is no namespace' do @@ -3516,13 +3517,13 @@ describe Project do expect_any_instance_of(Projects::UpdatePagesConfigurationService).not_to receive(:execute) expect_any_instance_of(Gitlab::PagesTransfer).not_to receive(:rename_project) - project.remove_pages + expect { project.remove_pages }.not_to change { project_pages_metadatum.reload.deployed } end it 'is run when the project is destroyed' do expect(project).to receive(:remove_pages).and_call_original - project.destroy + expect { project.destroy }.not_to raise_error end end @@ -4976,6 +4977,50 @@ describe Project do end end + describe '.migrate_project_pages_metadata' do + it 'marks projects with successful pages deployment' do + not_migrated_with_pages = create(:generic_commit_status, :success, stage: 'deploy', name: 'pages:deploy').project + not_migrated_no_pages = create(:project) + migrated = create(:project_pages_metadatum, deployed: true).project + other_project = create(:project) + + projects = described_class.where(id: [not_migrated_with_pages, not_migrated_no_pages, migrated]) + + expect { projects.migrate_project_pages_metadata }.not_to raise_error + + expect(not_migrated_with_pages.project_pages_metadatum.deployed).to eq(true) + expect(not_migrated_no_pages.project_pages_metadatum.deployed).to eq(false) + expect(migrated.project_pages_metadatum.deployed).to eq(true) + expect(other_project.project_pages_metadatum).to be_nil + end + end + + context 'pages deployed' do + { + 'mark_pages_as_deployed' => true, + 'mark_pages_as_not_deployed' => false + }.each do |method, flag| + describe "##{method}" do + it "creates new record and sets deployed to #{flag} if none exists yet" do + project = create(:project) + + project.send(method) + + expect(project.project_pages_metadatum.deployed).to eq(flag) + end + + it "updates the existing record and sets deployed to #{flag}" do + project_pages_metadatum = create(:project_pages_metadatum, deployed: !flag) + project = project_pages_metadatum.project + + expect { project.send(method) }.to change { + project_pages_metadatum.reload.deployed + }.from(!flag).to(flag) + end + end + end + end + describe '#has_pool_repsitory?' do it 'returns false when it does not have a pool repository' do subject = create(:project, :repository) diff --git a/spec/requests/api/internal/pages_spec.rb b/spec/requests/api/internal/pages_spec.rb new file mode 100644 index 00000000000..4efa4276be7 --- /dev/null +++ b/spec/requests/api/internal/pages_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::Internal::Pages do + describe "GET /internal/pages" do + let(:pages_shared_secret) { SecureRandom.random_bytes(Gitlab::Pages::SECRET_LENGTH) } + + before do + allow(Gitlab::Pages).to receive(:secret).and_return(pages_shared_secret) + end + + def query_host(host, headers = {}) + get api("/internal/pages"), headers: headers, params: { host: host } + end + + context 'feature flag disabled' do + before do + stub_feature_flags(pages_internal_api: false) + end + + it 'responds with 404 Not Found' do + query_host('pages.gitlab.io') + + expect(response).to have_gitlab_http_status(404) + end + end + + context 'feature flag enabled' do + context 'not authenticated' do + it 'responds with 401 Unauthorized' do + query_host('pages.gitlab.io') + + expect(response).to have_gitlab_http_status(401) + end + end + + context 'authenticated' do + def query_host(host) + jwt_token = JWT.encode({ 'iss' => 'gitlab-pages' }, Gitlab::Pages.secret, 'HS256') + headers = { Gitlab::Pages::INTERNAL_API_REQUEST_HEADER => jwt_token } + + super(host, headers) + end + + context 'not existing host' do + it 'responds with 404 Not Found' do + query_host('pages.gitlab.io') + + expect(response).to have_gitlab_http_status(404) + end + end + + context 'custom domain' do + it 'responds with the correct domain configuration' do + namespace = create(:namespace, name: 'gitlab-org') + project = create(:project, namespace: namespace, name: 'gitlab-ce') + pages_domain = create(:pages_domain, domain: 'pages.gitlab.io', project: project) + + query_host('pages.gitlab.io') + + expect(response).to have_gitlab_http_status(200) + expect(response).to match_response_schema('internal/pages/virtual_domain') + + expect(json_response['certificate']).to eq(pages_domain.certificate) + expect(json_response['key']).to eq(pages_domain.key) + + lookup_path = json_response['lookup_paths'][0] + expect(lookup_path['prefix']).to eq('/') + expect(lookup_path['source']['path']).to eq('gitlab-org/gitlab-ce/public/') + end + end + + context 'namespaced domain' do + let(:group) { create(:group, name: 'mygroup') } + + def deploy_pages(project) + generic_commit_status = create(:generic_commit_status, :success, stage: 'deploy', name: 'pages:deploy') + generic_commit_status.update!(project: project) + end + + before do + allow(Settings.pages).to receive(:host).and_return('gitlab-pages.io') + allow(Gitlab.config.pages).to receive(:url).and_return("http://gitlab-pages.io") + end + + context 'regular project' do + it 'responds with the correct domain configuration' do + project = create(:project, group: group, name: 'myproject') + deploy_pages(project) + + query_host('mygroup.gitlab-pages.io') + + expect(response).to have_gitlab_http_status(200) + expect(response).to match_response_schema('internal/pages/virtual_domain') + + lookup_path = json_response['lookup_paths'][0] + expect(lookup_path['prefix']).to eq('/myproject/') + expect(lookup_path['source']['path']).to eq('mygroup/myproject/public/') + end + end + + context 'group root project' do + it 'responds with the correct domain configuration' do + project = create(:project, group: group, name: 'mygroup.gitlab-pages.io') + deploy_pages(project) + + query_host('mygroup.gitlab-pages.io') + + expect(response).to have_gitlab_http_status(200) + expect(response).to match_response_schema('internal/pages/virtual_domain') + + lookup_path = json_response['lookup_paths'][0] + expect(lookup_path['prefix']).to eq('/') + expect(lookup_path['source']['path']).to eq('mygroup/mygroup.gitlab-pages.io/public/') + end + end + end + end + end + end +end diff --git a/spec/services/projects/update_pages_service_spec.rb b/spec/services/projects/update_pages_service_spec.rb index b597717c347..25e77f232a4 100644 --- a/spec/services/projects/update_pages_service_spec.rb +++ b/spec/services/projects/update_pages_service_spec.rb @@ -40,6 +40,7 @@ describe Projects::UpdatePagesService do it "doesn't delete artifacts after deploying" do expect(execute).to eq(:success) + expect(project.project_pages_metadatum.deployed).to eq(true) expect(build.artifacts?).to eq(true) end end @@ -47,6 +48,7 @@ describe Projects::UpdatePagesService do it 'succeeds' do expect(project.pages_deployed?).to be_falsey expect(execute).to eq(:success) + expect(project.project_pages_metadatum.deployed).to eq(true) expect(project.pages_deployed?).to be_truthy # Check that all expected files are extracted @@ -63,16 +65,23 @@ describe Projects::UpdatePagesService do it 'removes pages after destroy' do expect(PagesWorker).to receive(:perform_in) expect(project.pages_deployed?).to be_falsey + expect(execute).to eq(:success) + + expect(project.project_pages_metadatum.deployed).to eq(true) expect(project.pages_deployed?).to be_truthy + project.destroy + expect(project.pages_deployed?).to be_falsey + expect(ProjectPagesMetadatum.find_by_project_id(project)).to be_nil end it 'fails if sha on branch is not latest' do build.update(ref: 'feature') expect(execute).not_to eq(:success) + expect(project.project_pages_metadatum.deployed).to eq(false) end context 'when using empty file' do @@ -94,6 +103,7 @@ describe Projects::UpdatePagesService do it 'succeeds to extract' do expect(execute).to eq(:success) + expect(project.project_pages_metadatum.deployed).to eq(true) end end end @@ -109,6 +119,7 @@ describe Projects::UpdatePagesService do build.reload expect(deploy_status).to be_failed + expect(project.project_pages_metadatum.deployed).to eq(false) end end @@ -125,6 +136,7 @@ describe Projects::UpdatePagesService do build.reload expect(deploy_status).to be_failed + expect(project.project_pages_metadatum.deployed).to eq(false) end end @@ -138,6 +150,7 @@ describe Projects::UpdatePagesService do build.reload expect(deploy_status).to be_failed + expect(project.project_pages_metadatum.deployed).to eq(false) end end end @@ -179,6 +192,7 @@ describe Projects::UpdatePagesService do expect(deploy_status.description) .to match(/artifacts for pages are too large/) expect(deploy_status).to be_script_failure + expect(project.project_pages_metadatum.deployed).to eq(false) end end @@ -196,6 +210,7 @@ describe Projects::UpdatePagesService do subject.execute expect(deploy_status.description).not_to be_present + expect(project.project_pages_metadatum.deployed).to eq(true) end end |