diff options
56 files changed, 1046 insertions, 82 deletions
diff --git a/app/assets/javascripts/lib/utils/bootstrap_linked_tabs.js b/app/assets/javascripts/lib/utils/bootstrap_linked_tabs.js index a24c71aeab1..28a7ebfdc69 100644 --- a/app/assets/javascripts/lib/utils/bootstrap_linked_tabs.js +++ b/app/assets/javascripts/lib/utils/bootstrap_linked_tabs.js @@ -51,6 +51,7 @@ export default class LinkedTabs { this.defaultAction = this.options.defaultAction; this.action = this.options.action || this.defaultAction; + this.hashedTabs = this.options.hashedTabs || false; if (this.action === 'show') { this.action = this.defaultAction; @@ -58,6 +59,10 @@ export default class LinkedTabs { this.currentLocation = window.location; + if (this.hashedTabs) { + this.action = this.currentLocation.hash || this.action; + } + const tabSelector = `${this.options.parentEl} a[data-toggle="tab"]`; // since this is a custom event we need jQuery :( @@ -91,7 +96,9 @@ export default class LinkedTabs { copySource.replace(/\/+$/, ''); - const newState = `${copySource}${this.currentLocation.search}${this.currentLocation.hash}`; + const newState = this.hashedTabs + ? copySource + : `${copySource}${this.currentLocation.search}${this.currentLocation.hash}`; window.history.replaceState( { diff --git a/app/assets/javascripts/lib/utils/number_utils.js b/app/assets/javascripts/lib/utils/number_utils.js index 9ddfb4bca11..61c8b8803d7 100644 --- a/app/assets/javascripts/lib/utils/number_utils.js +++ b/app/assets/javascripts/lib/utils/number_utils.js @@ -100,3 +100,9 @@ export function numberToHumanSize(size) { * @returns {Float} The summed value */ export const sum = (a = 0, b = 0) => a + b; + +/** + * Checks if the provided number is odd + * @param {Int} number + */ +export const isOdd = (number = 0) => number % 2; diff --git a/app/controllers/acme_challenges_controller.rb b/app/controllers/acme_challenges_controller.rb new file mode 100644 index 00000000000..67a39d8870b --- /dev/null +++ b/app/controllers/acme_challenges_controller.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AcmeChallengesController < ActionController::Base + def show + if acme_order + render plain: acme_order.challenge_file_content, content_type: 'text/plain' + else + head :not_found + end + end + + private + + def acme_order + @acme_order ||= PagesDomainAcmeOrder.find_by_domain_and_token(params[:domain], params[:token]) + end +end diff --git a/app/controllers/projects/clusters_controller.rb b/app/controllers/projects/clusters_controller.rb index cb02581da37..98cd66cf6f9 100644 --- a/app/controllers/projects/clusters_controller.rb +++ b/app/controllers/projects/clusters_controller.rb @@ -4,6 +4,10 @@ class Projects::ClustersController < Clusters::ClustersController prepend_before_action :project before_action :repository + before_action do + push_frontend_feature_flag(:prometheus_computed_alerts) + end + layout 'project' private diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index c342e1c80b0..681b49e0bd7 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -15,6 +15,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint) push_frontend_feature_flag(:environment_metrics_show_multiple_dashboards) push_frontend_feature_flag(:grafana_dashboard_link) + push_frontend_feature_flag(:prometheus_computed_alerts) end def index diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 407d85b1520..524df30289e 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -5,6 +5,7 @@ class PagesDomain < ApplicationRecord VERIFICATION_THRESHOLD = 3.days.freeze belongs_to :project + has_many :acme_orders, class_name: "PagesDomainAcmeOrder" validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } @@ -134,6 +135,14 @@ class PagesDomain < ApplicationRecord "#{VERIFICATION_KEY}=#{verification_code}" end + def certificate=(certificate) + super(certificate) + + # set nil, if certificate is nil + self.certificate_valid_not_before = x509&.not_before + self.certificate_valid_not_after = x509&.not_after + end + private def set_verification_code @@ -186,7 +195,7 @@ class PagesDomain < ApplicationRecord end def x509 - return unless certificate + return unless certificate.present? @x509 ||= OpenSSL::X509::Certificate.new(certificate) rescue OpenSSL::X509::CertificateError diff --git a/app/models/pages_domain_acme_order.rb b/app/models/pages_domain_acme_order.rb new file mode 100644 index 00000000000..63d7fbc8206 --- /dev/null +++ b/app/models/pages_domain_acme_order.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class PagesDomainAcmeOrder < ApplicationRecord + belongs_to :pages_domain + + scope :expired, -> { where("expires_at < ?", Time.now) } + + validates :pages_domain, presence: true + validates :expires_at, presence: true + validates :url, presence: true + validates :challenge_token, presence: true + validates :challenge_file_content, presence: true + validates :private_key, presence: true + + attr_encrypted :private_key, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_truncated, + algorithm: 'aes-256-gcm', + encode: true + + def self.find_by_domain_and_token(domain_name, challenge_token) + joins(:pages_domain).find_by(pages_domains: { domain: domain_name }, challenge_token: challenge_token) + end +end diff --git a/app/services/pages_domains/create_acme_order_service.rb b/app/services/pages_domains/create_acme_order_service.rb new file mode 100644 index 00000000000..c600f497fa5 --- /dev/null +++ b/app/services/pages_domains/create_acme_order_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module PagesDomains + class CreateAcmeOrderService + attr_reader :pages_domain + + def initialize(pages_domain) + @pages_domain = pages_domain + end + + def execute + lets_encrypt_client = Gitlab::LetsEncrypt::Client.new + order = lets_encrypt_client.new_order(pages_domain.domain) + + challenge = order.new_challenge + + private_key = OpenSSL::PKey::RSA.new(4096) + saved_order = pages_domain.acme_orders.create!( + url: order.url, + expires_at: order.expires, + private_key: private_key.to_pem, + + challenge_token: challenge.token, + challenge_file_content: challenge.file_content + ) + + challenge.request_validation + saved_order + end + end +end diff --git a/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb b/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb new file mode 100644 index 00000000000..2dfe1a3d8ca --- /dev/null +++ b/app/services/pages_domains/obtain_lets_encrypt_certificate_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module PagesDomains + class ObtainLetsEncryptCertificateService + attr_reader :pages_domain + + def initialize(pages_domain) + @pages_domain = pages_domain + end + + def execute + pages_domain.acme_orders.expired.delete_all + acme_order = pages_domain.acme_orders.first + + unless acme_order + ::PagesDomains::CreateAcmeOrderService.new(pages_domain).execute + return + end + + api_order = ::Gitlab::LetsEncrypt::Client.new.load_order(acme_order.url) + + # https://tools.ietf.org/html/rfc8555#section-7.1.6 - statuses diagram + case api_order.status + when 'ready' + api_order.request_certificate(private_key: acme_order.private_key, domain: pages_domain.domain) + when 'valid' + save_certificate(acme_order.private_key, api_order) + acme_order.destroy! + # when 'invalid' + # TODO: implement error handling + end + end + + private + + def save_certificate(private_key, api_order) + certificate = api_order.certificate + pages_domain.update!(key: private_key, certificate: certificate) + end + end +end diff --git a/changelogs/unreleased/60303-replace-sidekiq-mtail-metrics.yml b/changelogs/unreleased/60303-replace-sidekiq-mtail-metrics.yml new file mode 100644 index 00000000000..90b72ec05c7 --- /dev/null +++ b/changelogs/unreleased/60303-replace-sidekiq-mtail-metrics.yml @@ -0,0 +1,5 @@ +--- +title: Replaces sidekiq mtail metrics with ruby instrumentation metrics +merge_request: 29215 +author: +type: changed diff --git a/changelogs/unreleased/docs-add-chatops-request-doc.yml b/changelogs/unreleased/docs-add-chatops-request-doc.yml new file mode 100644 index 00000000000..85ba86a73af --- /dev/null +++ b/changelogs/unreleased/docs-add-chatops-request-doc.yml @@ -0,0 +1,5 @@ +--- +title: Add section to dev docs on accessing chatops +merge_request: 28623 +author: +type: other diff --git a/changelogs/unreleased/sh-fix-openid-connect-defaults.yml b/changelogs/unreleased/sh-fix-openid-connect-defaults.yml new file mode 100644 index 00000000000..1ed977c9be6 --- /dev/null +++ b/changelogs/unreleased/sh-fix-openid-connect-defaults.yml @@ -0,0 +1,5 @@ +--- +title: Make OpenID Connect work without requiring a name +merge_request: 29312 +author: +type: fixed diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 68f8487d377..4da683014d4 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -19,12 +19,6 @@ Gitlab::Application.configure do |config| config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware) end -Sidekiq.configure_server do |config| - config.on(:startup) do - Gitlab::Metrics::SidekiqMetricsExporter.instance.start - end -end - if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? Gitlab::Cluster::LifecycleEvents.on_worker_start do defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change diff --git a/config/routes.rb b/config/routes.rb index f5957f43655..cb90a0134c4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -75,6 +75,8 @@ Rails.application.routes.draw do resources :issues, module: :boards, only: [:index, :update] end + get 'acme-challenge/' => 'acme_challenges#show' + # UserCallouts resources :user_callouts, only: [:create] diff --git a/danger/commit_messages/Dangerfile b/danger/commit_messages/Dangerfile index 048c539bcf9..bdb4343b1d6 100644 --- a/danger/commit_messages/Dangerfile +++ b/danger/commit_messages/Dangerfile @@ -163,10 +163,10 @@ def lint_commit(commit) end if emoji_checker.includes_emoji?(commit.message) - fail_commit( + warn_commit( commit, 'Avoid the use of Markdown Emoji such as `:+1:`. ' \ - 'These add no value to the commit message, ' \ + 'These add limited value to the commit message, ' \ 'and are displayed as plain text outside of GitLab' ) diff --git a/db/migrate/20190429082448_create_pages_domain_acme_orders.rb b/db/migrate/20190429082448_create_pages_domain_acme_orders.rb new file mode 100644 index 00000000000..af811e83518 --- /dev/null +++ b/db/migrate/20190429082448_create_pages_domain_acme_orders.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreatePagesDomainAcmeOrders < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + create_table :pages_domain_acme_orders do |t| + t.references :pages_domain, null: false, index: true, foreign_key: { on_delete: :cascade }, type: :integer + + t.datetime_with_timezone :expires_at, null: false + t.timestamps_with_timezone null: false + + t.string :url, null: false + + t.string :challenge_token, null: false, index: true + t.text :challenge_file_content, null: false + + t.text :encrypted_private_key, null: false + t.text :encrypted_private_key_iv, null: false + end + end +end diff --git a/db/migrate/20190524071727_add_ssl_valid_period_to_pages_domain.rb b/db/migrate/20190524071727_add_ssl_valid_period_to_pages_domain.rb new file mode 100644 index 00000000000..18544dcb6d3 --- /dev/null +++ b/db/migrate/20190524071727_add_ssl_valid_period_to_pages_domain.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddSslValidPeriodToPagesDomain < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column :pages_domains, :certificate_valid_not_before, :datetime_with_timezone + add_column :pages_domains, :certificate_valid_not_after, :datetime_with_timezone + end +end diff --git a/db/post_migrate/20190524073827_schedule_fill_valid_time_for_pages_domain_certificates.rb b/db/post_migrate/20190524073827_schedule_fill_valid_time_for_pages_domain_certificates.rb new file mode 100644 index 00000000000..1d8510e4514 --- /dev/null +++ b/db/post_migrate/20190524073827_schedule_fill_valid_time_for_pages_domain_certificates.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ScheduleFillValidTimeForPagesDomainCertificates < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + MIGRATION = 'FillValidTimeForPagesDomainCertificate' + BATCH_SIZE = 500 + BATCH_TIME = 5.minutes + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + class PagesDomain < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'pages_domains' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + PagesDomain.where.not(certificate: [nil, '']), + MIGRATION, + BATCH_TIME, + batch_size: BATCH_SIZE) + end + + def down + end +end diff --git a/db/schema.rb b/db/schema.rb index fcf9e397ac1..7de5b0352f0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1571,6 +1571,20 @@ ActiveRecord::Schema.define(version: 20190530154715) do t.index ["access_grant_id"], name: "index_oauth_openid_requests_on_access_grant_id", using: :btree end + create_table "pages_domain_acme_orders", force: :cascade do |t| + t.integer "pages_domain_id", null: false + t.datetime_with_timezone "expires_at", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.string "url", null: false + t.string "challenge_token", null: false + t.text "challenge_file_content", null: false + t.text "encrypted_private_key", null: false + t.text "encrypted_private_key_iv", null: false + t.index ["challenge_token"], name: "index_pages_domain_acme_orders_on_challenge_token", using: :btree + t.index ["pages_domain_id"], name: "index_pages_domain_acme_orders_on_pages_domain_id", using: :btree + end + create_table "pages_domains", id: :serial, force: :cascade do |t| t.integer "project_id" t.text "certificate" @@ -1583,6 +1597,8 @@ ActiveRecord::Schema.define(version: 20190530154715) do t.datetime_with_timezone "enabled_until" t.datetime_with_timezone "remove_at" t.boolean "auto_ssl_enabled", default: false, null: false + t.datetime_with_timezone "certificate_valid_not_before" + t.datetime_with_timezone "certificate_valid_not_after" t.index ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree t.index ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree t.index ["project_id"], name: "index_pages_domains_on_project_id", using: :btree @@ -2560,6 +2576,7 @@ ActiveRecord::Schema.define(version: 20190530154715) do add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade add_foreign_key "notification_settings", "users", name: "fk_0c95e91db7", on_delete: :cascade add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id" + add_foreign_key "pages_domain_acme_orders", "pages_domains", on_delete: :cascade add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade add_foreign_key "personal_access_tokens", "users" add_foreign_key "pool_repositories", "projects", column: "source_project_id", on_delete: :nullify diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index e55f7dbb4df..df4f22aa3e7 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -31,6 +31,7 @@ The OpenID Connect will provide you with a client details and secret for you to { 'name' => 'openid_connect', 'label' => '<your_oidc_label>', 'args' => { + "name' => 'openid_connect', 'scope' => ['openid','profile'], 'response_type' => 'code', 'issuer' => '<your_oidc_url>', @@ -53,6 +54,7 @@ The OpenID Connect will provide you with a client details and secret for you to - { name: 'openid_connect', label: '<your_oidc_label>', args: { + name: 'openid_connect', scope: ['openid','profile'], response_type: 'code', issuer: '<your_oidc_url>', @@ -103,3 +105,36 @@ On the sign in page, there should now be an OpenID Connect icon below the regula Click the icon to begin the authentication process. The OpenID Connect provider will ask the user to sign in and authorize the GitLab application (if confirmation required by the client). If everything goes well, the user will be redirected to GitLab and will be signed in. + +## Example configurations + +The following configurations illustrate how to set up OpenID with +different providers with Omnibus GitLab. + +### Google + +See the [Google +documentation](https://developers.google.com/identity/protocols/OpenIDConnect) +for more details: + +```ruby + gitlab_rails['omniauth_providers'] = [ + { + 'name' => 'openid_connect', + 'label' => 'Google OpenID', + 'args' => { + 'name' => 'openid_connect', + 'scope' => ['openid', 'profile', 'email'], + 'response_type' => 'code', + 'issuer' => 'https://accounts.google.com', + 'client_auth_method' => 'query', + 'discovery' => true, + 'uid_field' => 'preferred_username', + 'client_options' => { + 'identifier' => '<YOUR PROJECT CLIENT ID>', + 'secret' => '<YOUR PROJECT CLIENT SECRET>', + 'redirect_uri' => 'https://example.com/users/auth/openid_connect/callback', + } + } + } +``` diff --git a/doc/ci/README.md b/doc/ci/README.md index 93b82a065b3..635cce13b4e 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -161,6 +161,33 @@ See also the [Why CI/CD?](https://docs.google.com/presentation/d/1OGgk2Tcxbpl7DJ As GitLab CI/CD has evolved, certain breaking changes have been necessary. These are: +#### 12.0 + +- [Use refspec to clone/fetch git + repository](https://gitlab.com/gitlab-org/gitlab-runner/issues/4069). +- [Old cache + configuration](https://gitlab.com/gitlab-org/gitlab-runner/issues/4070). +- [Old metrics server + configuration](https://gitlab.com/gitlab-org/gitlab-runner/issues/4072). +- [Remove + `FF_K8S_USE_ENTRYPOINT_OVER_COMMAND`](https://gitlab.com/gitlab-org/gitlab-runner/issues/4073). +- [Remove Linux distributions that reach + EOL](https://gitlab.com/gitlab-org/gitlab-runner/merge_requests/1130). +- [Update command line API for helper + images](https://gitlab.com/gitlab-org/gitlab-runner/issues/4013). +- [Remove old `git clean` + flow](https://gitlab.com/gitlab-org/gitlab-runner/issues/4175). + +#### 11.0 + +- No breaking changes. + +#### 10.0 + +- No breaking changes. + +#### 9.0 + - [CI variables renaming for GitLab 9.0](variables/deprecated_variables.md#gitlab-90-renamed-variables). Read about the deprecated CI variables and what you should use for GitLab 9.0+. - [New CI job permissions model](../user/project/new_ci_build_permissions_model.md). diff --git a/doc/development/README.md b/doc/development/README.md index 624665a42d1..d2f09fc01de 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -20,6 +20,7 @@ description: 'Learn how to contribute to GitLab.' - [Automatic CE->EE merge](automatic_ce_ee_merge.md) - [Guidelines for implementing Enterprise Edition features](ee_features.md) - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) +- [Requesting access to Chatops on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLabbers) ## UX and frontend guides diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md new file mode 100644 index 00000000000..c63ec53414c --- /dev/null +++ b/doc/development/chatops_on_gitlabcom.md @@ -0,0 +1,21 @@ +# Chatops on GitLab.com + +Chatops on GitLab.com allows GitLabbers to run various automation tasks on GitLab.com using Slack. + +## Requesting access + +GitLabbers may need access to Chatops on GitLab.com for administration tasks such as: + +- Configuring feature flags on staging. +- Running `EXPLAIN` queries against the GitLab.com production replica. + +To request access to Chatops on GitLab.com: + +1. Log into <https://ops.gitlab.net/users/sign_in> using the same username as for GitLab.com. +1. Ask [anyone in the `chatops` project](https://gitlab.com/gitlab-com/chatops/project_members) to add you by running `/chatops run member add <username> gitlab-com/chatops --ops`. + +## See also + + - [Chatops Usage](https://docs.gitlab.com/ee/ci/chatops/README.html) + - [Understanding EXPLAIN plans](understanding_explain_plans.md) + - [Feature Groups](feature_flags.md#feature-groups) diff --git a/doc/development/feature_flags.md b/doc/development/feature_flags.md index c871015aaf6..13f0c5cc33e 100644 --- a/doc/development/feature_flags.md +++ b/doc/development/feature_flags.md @@ -20,7 +20,7 @@ dynamic (querying the DB etc.). Once defined in `lib/feature.rb`, you will be able to activate a feature for a given feature group via the [`feature_group` param of the features API](../api/features.md#set-or-create-a-feature) -For GitLab.com, team members have access to feature flags through chatops. Only +For GitLab.com, [team members have access to feature flags through Chatops](chatops_on_gitlabcom.md). Only percentage gates are supported at this time. Setting a feature to be used 50% of the time, you should execute `/chatops run feature set my_feature_flag 50`. diff --git a/doc/development/testing_guide/end_to_end/dynamic_element_validation.md b/doc/development/testing_guide/end_to_end/dynamic_element_validation.md new file mode 100644 index 00000000000..f7b3ca8bc89 --- /dev/null +++ b/doc/development/testing_guide/end_to_end/dynamic_element_validation.md @@ -0,0 +1,113 @@ +# Dynamic Element Validation + +We devised a solution to solve common test automation problems such as the dreaded `NoSuchElementException`. + +Other problems that dynamic element validations solve are... + +- When we perform an action with the mouse, we expect something to occur. +- When our test is navigating to (or from) a page, we ensure that we are on the page we expect before +test continuation. + +## How it works + +We interpret user actions on the page to have some sort of effect. These actions are + +- [Navigation](#navigation) +- [Clicks](#clicks) + +### Navigation + +When a page is navigated to, there are elements that will always appear on the page unconditionally. + +Dynamic element validation is instituted when using + +```ruby +Runtime::Browser.visit(:gitlab, Some::Page) +``` + +### Clicks + +When we perform a click within our tests, we expect something to occur. That something could be a component to now +appear on the webpage, or the test to navigate away from the page entirely. + +Dynamic element validation is instituted when using + +```ruby +click_element :my_element, Some::Page +``` + +### Required Elements + +#### Definition + +First it is important to define what a "required element" is. + +Simply put, a required element is a visible HTML element that appears on a UI component without any user input. + +"Visible" can be defined as + +- Not having any CSS preventing its display. E.g.: `display: none` or `width: 0px; height: 0px;` +- Being able to be interacted with by the user + +"UI component" can be defined as + +- Anything the user sees +- A button, a text field +- A layer that sits atop the page + +#### Application + +Requiring elements is very easy. By adding `required: true` as a parameter to an `element`, you've now made it +a requirement that the element appear on the page upon navigation. + +## Examples + +Given ... + +```ruby +class MyPage < Page::Base + view 'app/views/view.html.haml' do + element :my_element, required: true + element :another_element, required: true + element :conditional_element + end + + def open_layer + click_element :my_element, Layer::MyLayer + end +end + +class Layer < Page::Component + view 'app/views/mylayer/layer.html.haml' do + element :message_content, required: true + end +end +``` + +### Navigating + +Given the [source](#examples) ... + +```ruby +Runtime::Browser.visit(:gitlab, Page::MyPage) + +execute_stuff +``` + +will invoke GitLab QA to scan `MyPage` for `my_element` and `another_element` to be on the page before continuing to +`execute_stuff` + +### Clicking + +Given the [source](#examples) ... + +```ruby +def open_layer + click_element :my_element, Layer::MyLayer +end +``` + +will invoke GitLab QA to ensure that `message_content` appears on +the Layer upon clicking `my_element`. + +This will imply that the Layer is indeed rendered before we continue our test. diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index d0de33892c4..73e1fd862c1 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -82,15 +82,17 @@ module Page end # ... + end end end ``` -The `view` DSL method declares the filename of the view where an -`element` is implemented. +### Defining Elements + +The `view` DSL method will correspond to the rails View, partial, or vue component that renders the elements. The `element` DSL method in turn declares an element for which a corresponding -`qa-element-name-dasherized` CSS class need to be added to the view file. +`qa-element-name-dasherized` CSS class will need to be added to the view file. You can also define a value (String or Regexp) to match to the actual view code but **this is deprecated** in favor of the above method for two reasons: @@ -115,6 +117,37 @@ view 'app/views/my/view.html.haml' do end ``` +### Adding Elements to a View + +Given the following elements... + +```ruby +view 'app/views/my/view.html.haml' do + element :login_field + element :password_field + element :sign_in_button +end +``` + +To add these elements to the view, you must change the rails View, partial, or vue component by adding a `qa-element-descriptor` class +for each element defined. + +In our case, `qa-login-field`, `qa-password-field` and `qa-sign-in-button` + +**app/views/my/view.html.haml** + +```haml += f.text_field :login, class: "form-control top qa-login-field", autofocus: "autofocus", autocapitalize: "off", autocorrect: "off", required: true, title: "This field is required." += f.password_field :password, class: "form-control bottom qa-password-field", required: true, title: "This field is required." += f.submit "Sign in", class: "btn btn-success qa-sign-in-button" +``` + +Things to note: + +- The CSS class must be `kebab-cased` (separated with hyphens "`-`") +- If the element appears on the page unconditionally, add `required: true` to the element. See +[Dynamic element validation](dynamic_element_validation.md) + ## Running the test locally During development, you can run the `qa:selectors` test by running diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index 2ef8b3148e4..bfbb7be70e3 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -654,6 +654,7 @@ and related tools such as: - <https://explain.depesz.com/> - <http://tatiyants.com/postgres-query-plan-visualization/> + ## Producing query plans There are a few ways to get the output of a query plan. Of course you @@ -683,9 +684,9 @@ Execution time: 0.113 ms ### Chatops -GitLab employees can also use our chatops solution, available in Slack using the -`/chatops` slash command. You can use chatops to get a query plan by running the -following: +[GitLab employees can also use our chatops solution, available in Slack using the +`/chatops` slash command](chatops_on_gitlabcom.md). +You can use chatops to get a query plan by running the following: ``` /chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 1d355824760..001e4b6bf48 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -40,11 +40,9 @@ These settings can be found within: - **Admin Area > Settings > General**. - The path `/admin/application_settings`. -The very first push of a new project cannot be checked for size as of now, so -the first push will allow you to upload more than the limit dictates, but every -subsequent push will be denied. LFS objects, however, can be checked on first -push and **will** be rejected if the sum of their sizes exceeds the maximum -allowed repository size. +The first push of a new project, including LFS objects, will be checked for size +and **will** be rejected if the sum of their sizes exceeds the maximum allowed +repository size. For details on manually purging files, see [reducing the repository size using Git](../../project/repository/reducing_the_repo_size_using_git.md). diff --git a/doc/user/project/integrations/img/jira_add_user_to_group.png b/doc/user/project/integrations/img/jira_add_user_to_group.png Binary files differindex 27dac49260c..d8cf541a81e 100644 --- a/doc/user/project/integrations/img/jira_add_user_to_group.png +++ b/doc/user/project/integrations/img/jira_add_user_to_group.png diff --git a/doc/user/project/integrations/img/jira_added_user_to_group.png b/doc/user/project/integrations/img/jira_added_user_to_group.png Binary files differnew file mode 100644 index 00000000000..b3e29a65d6e --- /dev/null +++ b/doc/user/project/integrations/img/jira_added_user_to_group.png diff --git a/doc/user/project/integrations/img/jira_create_new_group.png b/doc/user/project/integrations/img/jira_create_new_group.png Binary files differindex 06c4e84fc61..84be3a94a45 100644 --- a/doc/user/project/integrations/img/jira_create_new_group.png +++ b/doc/user/project/integrations/img/jira_create_new_group.png diff --git a/doc/user/project/integrations/img/jira_create_new_user.png b/doc/user/project/integrations/img/jira_create_new_user.png Binary files differindex e9c03ed770d..8460dc98ef9 100644 --- a/doc/user/project/integrations/img/jira_create_new_user.png +++ b/doc/user/project/integrations/img/jira_create_new_user.png diff --git a/doc/user/project/integrations/img/jira_group_access.png b/doc/user/project/integrations/img/jira_group_access.png Binary files differindex 448cc55504d..58cf114bd55 100644 --- a/doc/user/project/integrations/img/jira_group_access.png +++ b/doc/user/project/integrations/img/jira_group_access.png diff --git a/doc/user/project/integrations/img/jira_user_management_link.png b/doc/user/project/integrations/img/jira_user_management_link.png Binary files differindex 5eb9d031c3e..43ef18da6c8 100644 --- a/doc/user/project/integrations/img/jira_user_management_link.png +++ b/doc/user/project/integrations/img/jira_user_management_link.png diff --git a/doc/user/project/integrations/jira.md b/doc/user/project/integrations/jira.md index a90167b9767..c652149052e 100644 --- a/doc/user/project/integrations/jira.md +++ b/doc/user/project/integrations/jira.md @@ -47,11 +47,11 @@ project in Jira and then enter the correct values in GitLab. When connecting to **JIRA Server**, which supports basic authentication, a **username and password** are required. Check the link below and proceed to the next step: -- [Setting up an user in JIRA server](jira_server_configuration.md) +- [Setting up a user in JIRA server](jira_server_configuration.md) When connecting to **JIRA Cloud**, which supports authentication via API token, an **email and API token**, are required. Check the link below and proceed to the next step: -- [Setting up an user in JIRA cloud](jira_cloud_configuration.md) +- [Setting up a user in JIRA cloud](jira_cloud_configuration.md) ### Configuring GitLab diff --git a/doc/user/project/integrations/jira_server_configuration.md b/doc/user/project/integrations/jira_server_configuration.md index 20036183187..13d65c4d8e4 100644 --- a/doc/user/project/integrations/jira_server_configuration.md +++ b/doc/user/project/integrations/jira_server_configuration.md @@ -1,18 +1,16 @@ # Creating a username and password for JIRA server We need to create a user in Jira which will have access to all projects that -need to integrate with GitLab. Login to your Jira instance as admin and under -*Administration*, go to *User Management* and create a new user. +need to integrate with GitLab. As an example, we'll create a user named `gitlab` and add it to the `Jira-developers` group. NOTE: **Note** -It is important that the user `gitlab` has 'write' access to projects in Jira. +It is important that the Jira user created for the integration is given 'write' +access to your Jira projects. This is covered in the process below. -We have split this stage in steps so it is easier to follow. - -1. Log in to your Jira instance as an administrator and under **Administration** +1. Log in to your Jira instance as an administrator and under **Jira Administration** go to **User Management** to create a new user.  @@ -27,27 +25,34 @@ We have split this stage in steps so it is easier to follow.  -1. Create a `gitlab-developers` group which will have write access - to projects in Jira. Go to the **Groups** tab and select **Create group**. +1. Create a `gitlab-developers` group. (We will give this group write access to Jira + projects in a later step). Go to the **Groups** tab on the left, and select **Add group**.  - Give it an optional description and click **Create group**. + Give it a name and click **Add group**. -  +1. Add the `gitlab` user to the `gitlab-developers` group by clicking **Edit members**. + The `gitlab-developers` group should be listed in the leftmost box as a selected group. + Under **Add members to selected group(s)**, enter `gitlab`. -1. To give the newly-created group 'write' access, go to - **Application access > View configuration** and add the `gitlab-developers` - group to Jira Core. +  + + Click **Add selected users** and `gitlab` should appear in the **Group member(s)** box. + This membership is saved automatically. + +  + +1. To give the newly-created group 'write' access, you need to create a **Permission Scheme**. + To do this, click the gear icon and select **Issues**. Then click **Permission Schemes**. + Click **Add Permission Scheme** and enter a **Name** and, optionally, a **Description**. + +1. Once your permission scheme is created, you'll be taken back to the permissions scheme list. + Locate your new permissions scheme and click **Permissions**. Next to **Administer Projects**, + click **Edit**. In the resulting dialog box, select **Group** and select `gitlab-developers` + from the dropdown.  -1. Add the `gitlab` user to the `gitlab-developers` group by going to - **Users > GitLab user > Add group** and selecting the `gitlab-developers` - group from the dropdown menu. Notice that the group says _Access_, which is - intended as part of this process. - -  - The Jira configuration is complete. Write down the new Jira username and its password as they will be needed when [configuring GitLab in the next section](jira.md#configuring-gitlab). diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb new file mode 100644 index 00000000000..0e93b2cb2fa --- /dev/null +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # save validity time pages domain + class FillValidTimeForPagesDomainCertificate + # define PagesDomain with only needed code + class PagesDomain < ActiveRecord::Base + self.table_name = 'pages_domains' + + def x509 + return unless certificate.present? + + @x509 ||= OpenSSL::X509::Certificate.new(certificate) + rescue OpenSSL::X509::CertificateError + nil + end + end + + def perform(start_id, stop_id) + PagesDomain.where(id: start_id..stop_id).find_each do |domain| + if Gitlab::Database.mysql? + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before, + certificate_valid_not_after: domain.x509&.not_after + ) + else + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) + end + rescue => e + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + end + end + end + end +end diff --git a/lib/gitlab/lets_encrypt/challenge.rb b/lib/gitlab/lets_encrypt/challenge.rb index 6a7f5e965c5..136268c974b 100644 --- a/lib/gitlab/lets_encrypt/challenge.rb +++ b/lib/gitlab/lets_encrypt/challenge.rb @@ -7,7 +7,7 @@ module Gitlab @acme_challenge = acme_challenge end - delegate :url, :token, :file_content, :status, :request_validation, to: :acme_challenge + delegate :token, :file_content, :status, :request_validation, to: :acme_challenge private diff --git a/lib/gitlab/lets_encrypt/order.rb b/lib/gitlab/lets_encrypt/order.rb index 5109b5e9843..9c2365f98a8 100644 --- a/lib/gitlab/lets_encrypt/order.rb +++ b/lib/gitlab/lets_encrypt/order.rb @@ -13,7 +13,16 @@ module Gitlab ::Gitlab::LetsEncrypt::Challenge.new(challenge) end - delegate :url, :status, to: :acme_order + def request_certificate(domain:, private_key:) + csr = ::Acme::Client::CertificateRequest.new( + private_key: OpenSSL::PKey.read(private_key), + subject: { common_name: domain } + ) + + acme_order.finalize(csr: csr) + end + + delegate :url, :status, :expires, :certificate, to: :acme_order private diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4d9c43f37e7..17eacbd21d8 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -77,10 +77,10 @@ module Gitlab end def worker_label + return { worker: 'sidekiq' } if Sidekiq.server? return {} unless defined?(Unicorn::Worker) worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no { worker: worker_no } else diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 2a2083ebae0..83204fa5d18 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -63,6 +63,12 @@ module Gitlab { remote_sign_out_handler: authentiq_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } + when 'openid_connect' + # If a name argument is omitted, OmniAuth will expect that the + # matching route is /auth/users/openidconnect instead of + # /auth/users/openid_connect because of + # https://gitlab.com/gitlab-org/gitlab-ce/issues/62208#note_178780341. + { name: 'openid_connect' } else {} end diff --git a/qa/qa/resource/repository/project_push.rb b/qa/qa/resource/repository/project_push.rb index cad89ebb0bb..e98880ce195 100644 --- a/qa/qa/resource/repository/project_push.rb +++ b/qa/qa/resource/repository/project_push.rb @@ -32,8 +32,8 @@ module QA def fabricate! super - project.visit! project.wait_for_push @commit_message if @wait_for_push + project.visit! end end end diff --git a/spec/controllers/acme_challenges_controller_spec.rb b/spec/controllers/acme_challenges_controller_spec.rb new file mode 100644 index 00000000000..cee06bed27b --- /dev/null +++ b/spec/controllers/acme_challenges_controller_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe AcmeChallengesController do + describe '#show' do + let!(:acme_order) { create(:pages_domain_acme_order) } + + def make_request(domain, token) + get(:show, params: { domain: domain, token: token }) + end + + before do + make_request(domain, token) + end + + context 'with right domain and token' do + let(:domain) { acme_order.pages_domain.domain } + let(:token) { acme_order.challenge_token } + + it 'renders acme challenge file content' do + expect(response.body).to eq(acme_order.challenge_file_content) + end + end + + context 'when domain is invalid' do + let(:domain) { 'somewrongdomain.com' } + let(:token) { acme_order.challenge_token } + + it 'renders not found' do + expect(response).to have_gitlab_http_status(404) + end + end + + context 'when token is invalid' do + let(:domain) { acme_order.pages_domain.domain } + let(:token) { 'wrongtoken' } + + it 'renders not found' do + expect(response).to have_gitlab_http_status(404) + end + end + end +end diff --git a/spec/factories/pages_domain_acme_orders.rb b/spec/factories/pages_domain_acme_orders.rb new file mode 100644 index 00000000000..7f9ee1c8f9c --- /dev/null +++ b/spec/factories/pages_domain_acme_orders.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :pages_domain_acme_order do + pages_domain + url { 'https://example.com/' } + expires_at { 1.day.from_now } + challenge_token { 'challenge_token' } + challenge_file_content { 'filecontent' } + + private_key { OpenSSL::PKey::RSA.new(4096).to_pem } + + trait :expired do + expires_at { 1.day.ago } + end + end +end diff --git a/spec/frontend/lib/utils/number_utility_spec.js b/spec/frontend/lib/utils/number_utility_spec.js index 818404bad81..77d7478d317 100644 --- a/spec/frontend/lib/utils/number_utility_spec.js +++ b/spec/frontend/lib/utils/number_utility_spec.js @@ -5,6 +5,7 @@ import { bytesToGiB, numberToHumanSize, sum, + isOdd, } from '~/lib/utils/number_utils'; describe('Number Utils', () => { @@ -98,4 +99,14 @@ describe('Number Utils', () => { expect([1, 2, 3, 4, 5].reduce(sum)).toEqual(15); }); }); + + describe('isOdd', () => { + it('should return 0 with a even number', () => { + expect(isOdd(2)).toEqual(0); + }); + + it('should return 1 with a odd number', () => { + expect(isOdd(1)).toEqual(1); + }); + }); }); diff --git a/spec/lib/gitlab/lets_encrypt/challenge_spec.rb b/spec/lib/gitlab/lets_encrypt/challenge_spec.rb index 74622f356de..fcd92586362 100644 --- a/spec/lib/gitlab/lets_encrypt/challenge_spec.rb +++ b/spec/lib/gitlab/lets_encrypt/challenge_spec.rb @@ -3,23 +3,11 @@ require 'spec_helper' describe ::Gitlab::LetsEncrypt::Challenge do - delegated_methods = { - url: 'https://example.com/', - status: 'pending', - token: 'tokenvalue', - file_content: 'hereisfilecontent', - request_validation: true - } + include LetsEncryptHelpers - let(:acme_challenge) do - acme_challenge = instance_double('Acme::Client::Resources::Challenge') - allow(acme_challenge).to receive_messages(delegated_methods) - acme_challenge - end - - let(:challenge) { described_class.new(acme_challenge) } + let(:challenge) { described_class.new(acme_challenge_double) } - delegated_methods.each do |method, value| + LetsEncryptHelpers::ACME_CHALLENGE_METHODS.each do |method, value| describe "##{method}" do it 'delegates to Acme::Client::Resources::Challenge' do expect(challenge.public_send(method)).to eq(value) diff --git a/spec/lib/gitlab/lets_encrypt/order_spec.rb b/spec/lib/gitlab/lets_encrypt/order_spec.rb index ee7058baf8d..1a759103c44 100644 --- a/spec/lib/gitlab/lets_encrypt/order_spec.rb +++ b/spec/lib/gitlab/lets_encrypt/order_spec.rb @@ -3,20 +3,13 @@ require 'spec_helper' describe ::Gitlab::LetsEncrypt::Order do - delegated_methods = { - url: 'https://example.com/', - status: 'valid' - } - - let(:acme_order) do - acme_order = instance_double('Acme::Client::Resources::Order') - allow(acme_order).to receive_messages(delegated_methods) - acme_order - end + include LetsEncryptHelpers + + let(:acme_order) { acme_order_double } let(:order) { described_class.new(acme_order) } - delegated_methods.each do |method, value| + LetsEncryptHelpers::ACME_ORDER_METHODS.each do |method, value| describe "##{method}" do it 'delegates to Acme::Client::Resources::Order' do expect(order.public_send(method)).to eq(value) @@ -25,15 +18,24 @@ describe ::Gitlab::LetsEncrypt::Order do end describe '#new_challenge' do - before do - challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01') - authorization = instance_double('Acme::Client::Resources::Authorization') - allow(authorization).to receive(:http).and_return(challenge) - allow(acme_order).to receive(:authorizations).and_return([authorization]) - end - it 'returns challenge' do expect(order.new_challenge).to be_a(::Gitlab::LetsEncrypt::Challenge) end end + + describe '#request_certificate' do + let(:private_key) do + OpenSSL::PKey::RSA.new(4096).to_pem + end + + it 'generates csr and finalizes order' do + expect(acme_order).to receive(:finalize) do |csr:| + expect do + csr.csr # it's being evaluated lazily + end.not_to raise_error + end + + order.request_certificate(domain: 'example.com', private_key: private_key) + end + end end diff --git a/spec/lib/gitlab/omniauth_initializer_spec.rb b/spec/lib/gitlab/omniauth_initializer_spec.rb index f9c0daf1ef1..32296caf819 100644 --- a/spec/lib/gitlab/omniauth_initializer_spec.rb +++ b/spec/lib/gitlab/omniauth_initializer_spec.rb @@ -83,5 +83,13 @@ describe Gitlab::OmniauthInitializer do subject.execute([cas3_config]) end + + it 'configures name for openid_connect' do + openid_connect_config = { 'name' => 'openid_connect', 'args' => {} } + + expect(devise_config).to receive(:omniauth).with(:openid_connect, name: 'openid_connect') + + subject.execute([openid_connect_config]) + end end end diff --git a/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb index afcaefa0591..abf39317188 100644 --- a/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb +++ b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb @@ -8,9 +8,13 @@ describe EnqueueVerifyPagesDomainWorkers, :sidekiq, :migration do end end + let(:domains_table) { table(:pages_domains) } + describe '#up' do it 'enqueues a verification worker for every domain' do - domains = 1.upto(3).map { |i| PagesDomain.create!(domain: "my#{i}.domain.com") } + domains = Array.new(3) do |i| + domains_table.create!(domain: "my#{i}.domain.com", verification_code: "123#{i}") + end expect { migrate! }.to change(PagesDomainVerificationWorker.jobs, :size).by(3) diff --git a/spec/migrations/schedule_fill_valid_time_for_pages_domain_certificates_spec.rb b/spec/migrations/schedule_fill_valid_time_for_pages_domain_certificates_spec.rb new file mode 100644 index 00000000000..54f3e264df0 --- /dev/null +++ b/spec/migrations/schedule_fill_valid_time_for_pages_domain_certificates_spec.rb @@ -0,0 +1,46 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20190524073827_schedule_fill_valid_time_for_pages_domain_certificates.rb') + +describe ScheduleFillValidTimeForPagesDomainCertificates, :migration, :sidekiq do + let(:migration_class) { described_class::MIGRATION } + let(:migration_name) { migration_class.to_s.demodulize } + + let(:domains_table) { table(:pages_domains) } + + let(:certificate) do + File.read('spec/fixtures/passphrase_x509_certificate.crt') + end + + before do + domains_table.create!(domain: "domain1.example.com", verification_code: "123") + domains_table.create!(domain: "domain2.example.com", verification_code: "123", certificate: '') + domains_table.create!(domain: "domain3.example.com", verification_code: "123", certificate: certificate) + domains_table.create!(domain: "domain4.example.com", verification_code: "123", certificate: certificate) + end + + it 'correctly schedules background migrations' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + first_id = domains_table.find_by_domain("domain3.example.com").id + last_id = domains_table.find_by_domain("domain4.example.com").id + + expect(migration_name).to be_scheduled_delayed_migration(5.minutes, first_id, last_id) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + end + end + + it 'sets certificate valid_not_before/not_after' do + perform_enqueued_jobs do + migrate! + + domain = domains_table.find_by_domain("domain3.example.com") + expect(domain.certificate_valid_not_before) + .to eq(Time.parse("2018-03-23 14:02:08 UTC")) + expect(domain.certificate_valid_not_after) + .to eq(Time.parse("2019-03-23 14:02:08 UTC")) + end + end +end diff --git a/spec/models/pages_domain_acme_order_spec.rb b/spec/models/pages_domain_acme_order_spec.rb new file mode 100644 index 00000000000..4ffb4fc7389 --- /dev/null +++ b/spec/models/pages_domain_acme_order_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe PagesDomainAcmeOrder do + using RSpec::Parameterized::TableSyntax + + describe '.expired' do + let!(:not_expired_order) { create(:pages_domain_acme_order) } + let!(:expired_order) { create(:pages_domain_acme_order, :expired) } + + it 'returns only expired orders' do + expect(described_class.count).to eq(2) + expect(described_class.expired).to eq([expired_order]) + end + end + + describe '.find_by_domain_and_token' do + let!(:domain) { create(:pages_domain, domain: 'test.com') } + let!(:acme_order) { create(:pages_domain_acme_order, challenge_token: 'righttoken', pages_domain: domain) } + + where(:domain_name, :challenge_token, :present) do + 'test.com' | 'righttoken' | true + 'test.com' | 'wrongtoken' | false + 'test.org' | 'righttoken' | false + end + + with_them do + subject { described_class.find_by_domain_and_token(domain_name, challenge_token).present? } + + it { is_expected.to eq(present) } + end + end + + subject { create(:pages_domain_acme_order) } + + describe 'associations' do + it { is_expected.to belong_to(:pages_domain) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:pages_domain) } + it { is_expected.to validate_presence_of(:expires_at) } + it { is_expected.to validate_presence_of(:url) } + it { is_expected.to validate_presence_of(:challenge_token) } + it { is_expected.to validate_presence_of(:challenge_file_content) } + it { is_expected.to validate_presence_of(:private_key) } + end +end diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index ec4d4517f82..fdc81359d34 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -81,6 +81,17 @@ describe PagesDomain do end end + describe 'when certificate is specified' do + let(:domain) { build(:pages_domain) } + + it 'saves validity time' do + domain.save + + expect(domain.certificate_valid_not_before).to be_like_time(Time.parse("2016-02-12 14:32:00 UTC")) + expect(domain.certificate_valid_not_after).to be_like_time(Time.parse("2020-04-12 14:32:00 UTC")) + end + end + describe 'validate certificate' do subject { domain } diff --git a/spec/services/pages_domains/create_acme_order_service_spec.rb b/spec/services/pages_domains/create_acme_order_service_spec.rb new file mode 100644 index 00000000000..d59aa9b979e --- /dev/null +++ b/spec/services/pages_domains/create_acme_order_service_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe PagesDomains::CreateAcmeOrderService do + include LetsEncryptHelpers + + let(:pages_domain) { create(:pages_domain) } + + let(:challenge) { ::Gitlab::LetsEncrypt::Challenge.new(acme_challenge_double) } + + let(:order_double) do + Gitlab::LetsEncrypt::Order.new(acme_order_double).tap do |order| + allow(order).to receive(:new_challenge).and_return(challenge) + end + end + + let(:lets_encrypt_client) do + instance_double('Gitlab::LetsEncrypt::Client').tap do |client| + allow(client).to receive(:new_order).with(pages_domain.domain) + .and_return(order_double) + end + end + + let(:service) { described_class.new(pages_domain) } + + before do + allow(::Gitlab::LetsEncrypt::Client).to receive(:new).and_return(lets_encrypt_client) + end + + it 'saves order to database before requesting validation' do + allow(pages_domain.acme_orders).to receive(:create!).and_call_original + allow(challenge).to receive(:request_validation).and_call_original + + service.execute + + expect(pages_domain.acme_orders).to have_received(:create!).ordered + expect(challenge).to have_received(:request_validation).ordered + end + + it 'generates and saves private key' do + service.execute + + saved_order = PagesDomainAcmeOrder.last + expect { OpenSSL::PKey::RSA.new(saved_order.private_key) }.not_to raise_error + end + + it 'properly saves order attributes' do + service.execute + + saved_order = PagesDomainAcmeOrder.last + expect(saved_order.url).to eq(order_double.url) + expect(saved_order.expires_at).to be_like_time(order_double.expires) + end + + it 'properly saves challenge attributes' do + service.execute + + saved_order = PagesDomainAcmeOrder.last + expect(saved_order.challenge_token).to eq(challenge.token) + expect(saved_order.challenge_file_content).to eq(challenge.file_content) + end +end diff --git a/spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb b/spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb new file mode 100644 index 00000000000..6d7be27939c --- /dev/null +++ b/spec/services/pages_domains/obtain_lets_encrypt_certificate_service_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe PagesDomains::ObtainLetsEncryptCertificateService do + include LetsEncryptHelpers + + let(:pages_domain) { create(:pages_domain, :without_certificate, :without_key) } + let(:service) { described_class.new(pages_domain) } + + before do + stub_lets_encrypt_settings + end + + def expect_to_create_acme_challenge + expect(::PagesDomains::CreateAcmeOrderService).to receive(:new).with(pages_domain) + .and_wrap_original do |m, *args| + create_service = m.call(*args) + + expect(create_service).to receive(:execute) + + create_service + end + end + + def stub_lets_encrypt_order(url, status) + order = ::Gitlab::LetsEncrypt::Order.new(acme_order_double(status: status)) + + allow_any_instance_of(::Gitlab::LetsEncrypt::Client).to( + receive(:load_order).with(url).and_return(order) + ) + + order + end + + context 'when there is no acme order' do + it 'creates acme order' do + expect_to_create_acme_challenge + + service.execute + end + end + + context 'when there is expired acme order' do + let!(:existing_order) do + create(:pages_domain_acme_order, :expired, pages_domain: pages_domain) + end + + it 'removes acme order and creates new one' do + expect_to_create_acme_challenge + + service.execute + + expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil + end + end + + %w(pending processing).each do |status| + context "there is an order in '#{status}' status" do + let(:existing_order) do + create(:pages_domain_acme_order, pages_domain: pages_domain) + end + + before do + stub_lets_encrypt_order(existing_order.url, status) + end + + it 'does not raise errors' do + expect do + service.execute + end.not_to raise_error + end + end + end + + context 'when order is ready' do + let(:existing_order) do + create(:pages_domain_acme_order, pages_domain: pages_domain) + end + + let!(:api_order) do + stub_lets_encrypt_order(existing_order.url, 'ready') + end + + it 'request certificate' do + expect(api_order).to receive(:request_certificate).and_call_original + + service.execute + end + end + + context 'when order is valid' do + let(:existing_order) do + create(:pages_domain_acme_order, pages_domain: pages_domain) + end + + let!(:api_order) do + stub_lets_encrypt_order(existing_order.url, 'valid') + end + + let(:certificate) do + key = OpenSSL::PKey.read(existing_order.private_key) + + subject = "/C=BE/O=Test/OU=Test/CN=#{pages_domain.domain}" + + cert = OpenSSL::X509::Certificate.new + cert.subject = cert.issuer = OpenSSL::X509::Name.parse(subject) + cert.not_before = Time.now + cert.not_after = 1.year.from_now + cert.public_key = key.public_key + cert.serial = 0x0 + cert.version = 2 + + ef = OpenSSL::X509::ExtensionFactory.new + ef.subject_certificate = cert + ef.issuer_certificate = cert + cert.extensions = [ + ef.create_extension("basicConstraints", "CA:TRUE", true), + ef.create_extension("subjectKeyIdentifier", "hash") + ] + cert.add_extension ef.create_extension("authorityKeyIdentifier", + "keyid:always,issuer:always") + + cert.sign key, OpenSSL::Digest::SHA1.new + + cert.to_pem + end + + before do + expect(api_order).to receive(:certificate) { certificate } + end + + it 'saves private_key and certificate for domain' do + service.execute + + expect(pages_domain.key).to be_present + expect(pages_domain.certificate).to eq(certificate) + end + + it 'removes order from database' do + service.execute + + expect(PagesDomainAcmeOrder.find_by_id(existing_order.id)).to be_nil + end + end +end diff --git a/spec/support/helpers/lets_encrypt_helpers.rb b/spec/support/helpers/lets_encrypt_helpers.rb index 7f0886b451c..2857416ad95 100644 --- a/spec/support/helpers/lets_encrypt_helpers.rb +++ b/spec/support/helpers/lets_encrypt_helpers.rb @@ -1,6 +1,26 @@ # frozen_string_literal: true module LetsEncryptHelpers + ACME_ORDER_METHODS = { + url: 'https://example.com/', + status: 'valid', + expires: 2.days.from_now + }.freeze + + ACME_CHALLENGE_METHODS = { + status: 'pending', + token: 'tokenvalue', + file_content: 'hereisfilecontent', + request_validation: true + }.freeze + + def stub_lets_encrypt_settings + stub_application_setting( + lets_encrypt_notification_email: 'myemail@test.example.com', + lets_encrypt_terms_of_service_accepted: true + ) + end + def stub_lets_encrypt_client client = instance_double('Acme::Client') @@ -16,4 +36,24 @@ module LetsEncryptHelpers client end + + def acme_challenge_double + challenge = instance_double('Acme::Client::Resources::Challenges::HTTP01') + allow(challenge).to receive_messages(ACME_CHALLENGE_METHODS) + challenge + end + + def acme_authorization_double + authorization = instance_double('Acme::Client::Resources::Authorization') + allow(authorization).to receive(:http).and_return(acme_challenge_double) + authorization + end + + def acme_order_double(attributes = {}) + acme_order = instance_double('Acme::Client::Resources::Order') + allow(acme_order).to receive_messages(ACME_ORDER_METHODS.merge(attributes)) + allow(acme_order).to receive(:authorizations).and_return([acme_authorization_double]) + allow(acme_order).to receive(:finalize) + acme_order + end end diff --git a/spec/support/shared_context/policies/project_policy_shared_context.rb b/spec/support/shared_contexts/policies/project_policy_shared_context.rb index 54d9f5b15f2..54d9f5b15f2 100644 --- a/spec/support/shared_context/policies/project_policy_shared_context.rb +++ b/spec/support/shared_contexts/policies/project_policy_shared_context.rb |
