diff options
| author | Filipa Lacerda <filipa@gitlab.com> | 2018-02-26 10:30:18 +0000 |
|---|---|---|
| committer | Filipa Lacerda <filipa@gitlab.com> | 2018-02-26 10:30:18 +0000 |
| commit | bca29417ab4dcedfed23179c055fd0f3d5c5bce6 (patch) | |
| tree | ca441d0d7431a4f5cebf036758d580db97b48e43 | |
| parent | ebc2829added6a1ce61070da5f137d87472c5b12 (diff) | |
| parent | d1ceb60068c9c1a8a91e20ddffc108f82f313790 (diff) | |
| download | gitlab-ce-bca29417ab4dcedfed23179c055fd0f3d5c5bce6.tar.gz | |
[ci skip] Merge branch 'master' into 43315-gpg-popover
* master: (39 commits)
Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed
Add changelog entry
Use imported `DropdownUtils`
Update CHANGELOG.md for 10.5.2
Respond 404 when repo does not exist
Fix deletion attempts on dropped tables
Fix specs
prevent localStorage.clear from running when it would cause an error
Removed webpack tag
move webpack bundles we intend to keep below the list of bundles we're refactoring to reduce conflicts
Explicitly add uncovered files to troubleMakers
Backport custom metrics ce components
remove issue_show webpack bundle in favor of pages/projects/issues/show/index.js
Revert "Revert "Use Dir.mktmpdir""
Revert "Use Dir.mktmpdir"
Use Dir.mktmpdir
Refactored webpack bundle tag for deploy keys
Change FileUtils.cp to FileUtils.copy
Bring back the initial commit
Add a button to deploy a runner to a Kubernetes cluster in the settings page
...
145 files changed, 2310 insertions, 565 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 869884f8ca6..c8d399b2b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 10.5.2 (2018-02-25) + +### Fixed (7 changes) + +- Fix single digit value clipping for stacked progress bar. !17217 +- Fix issue with cache key being empty when variable used as the key. !17260 +- Enable Legacy Authorization by default on Cluster creations. !17302 +- Allow branch names to be named the same as the sha it points to. +- Fix 500 error when loading an invalid upload URL. +- Don't attempt to update user tracked fields if database is in read-only. +- Prevent MR Widget error when no CI configured. + +### Performance (5 changes) + +- Improve query performance for snippets dashboard. !17088 +- Only check LFS integrity for first ref in a push to avoid timeout. !17098 +- Improve query performance of MembersFinder. !17190 +- Increase feature flag cache TTL to one hour. +- Improve performance of searching for and autocompleting of users. + + ## 10.5.1 (2018-02-22) - No changes. diff --git a/app/assets/javascripts/boards/boards_bundle.js b/app/assets/javascripts/boards/index.js index 90166b3d3d1..06a8abea940 100644 --- a/app/assets/javascripts/boards/boards_bundle.js +++ b/app/assets/javascripts/boards/index.js @@ -24,7 +24,7 @@ import './components/new_list_dropdown'; import './components/modal/index'; import '../vue_shared/vue_resource_interceptor'; -$(() => { +export default () => { const $boardApp = document.getElementById('board-app'); const Store = gl.issueBoards.BoardsStore; const ModalStore = gl.issueBoards.ModalStore; @@ -236,4 +236,4 @@ $(() => { </div> `, }); -}); +}; diff --git a/app/assets/javascripts/deploy_keys/index.js b/app/assets/javascripts/deploy_keys/index.js index ca8798facc9..b727261648c 100644 --- a/app/assets/javascripts/deploy_keys/index.js +++ b/app/assets/javascripts/deploy_keys/index.js @@ -1,7 +1,7 @@ import Vue from 'vue'; import deployKeysApp from './components/app.vue'; -document.addEventListener('DOMContentLoaded', () => new Vue({ +export default () => new Vue({ el: document.getElementById('js-deploy-keys'), components: { deployKeysApp, @@ -18,4 +18,4 @@ document.addEventListener('DOMContentLoaded', () => new Vue({ }, }); }, -})); +}); diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index f66ce1c083b..acf0effa00d 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -42,31 +42,34 @@ var Dispatcher; }); }); - switch (page) { - case 'projects:merge_requests:index': - case 'projects:issues:index': - case 'projects:issues:show': - case 'projects:issues:new': - case 'projects:issues:edit': - case 'projects:merge_requests:creations:new': - case 'projects:merge_requests:creations:diffs': - case 'projects:merge_requests:edit': - case 'projects:merge_requests:show': - case 'projects:commit:show': - case 'projects:activity': - case 'projects:commits:show': - case 'projects:show': - case 'groups:show': - case 'projects:tree:show': - case 'projects:find_file:show': - case 'projects:blob:show': - case 'projects:blame:show': - case 'projects:network:show': - case 'projects:artifacts:browse': - case 'projects:artifacts:file': - shortcut_handler = true; - break; + const shortcutHandlerPages = [ + 'projects:activity', + 'projects:artifacts:browse', + 'projects:artifacts:file', + 'projects:blame:show', + 'projects:blob:show', + 'projects:commit:show', + 'projects:commits:show', + 'projects:find_file:show', + 'projects:issues:edit', + 'projects:issues:index', + 'projects:issues:new', + 'projects:issues:show', + 'projects:merge_requests:creations:diffs', + 'projects:merge_requests:creations:new', + 'projects:merge_requests:edit', + 'projects:merge_requests:index', + 'projects:merge_requests:show', + 'projects:network:show', + 'projects:show', + 'projects:tree:show', + 'groups:show', + ]; + + if (shortcutHandlerPages.indexOf(page) !== -1) { + shortcut_handler = true; } + switch (path[0]) { case 'admin': switch (path[1]) { diff --git a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js deleted file mode 100644 index b693084e434..00000000000 --- a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js +++ /dev/null @@ -1,102 +0,0 @@ -import eventHub from '../event_hub'; -import FilteredSearchTokenizer from '../filtered_search_tokenizer'; - -export default { - name: 'RecentSearchesDropdownContent', - - props: { - items: { - type: Array, - required: true, - }, - isLocalStorageAvailable: { - type: Boolean, - required: false, - default: true, - }, - allowedKeys: { - type: Array, - required: true, - }, - }, - - computed: { - processedItems() { - return this.items.map((item) => { - const { tokens, searchToken } - = FilteredSearchTokenizer.processTokens(item, this.allowedKeys); - - const resultantTokens = tokens.map(token => ({ - prefix: `${token.key}:`, - suffix: `${token.symbol}${token.value}`, - })); - - return { - text: item, - tokens: resultantTokens, - searchToken, - }; - }); - }, - hasItems() { - return this.items.length > 0; - }, - }, - - methods: { - onItemActivated(text) { - eventHub.$emit('recentSearchesItemSelected', text); - }, - onRequestClearRecentSearches(e) { - // Stop the dropdown from closing - e.stopPropagation(); - - eventHub.$emit('requestClearRecentSearches'); - }, - }, - - template: ` - <div> - <div - v-if="!isLocalStorageAvailable" - class="dropdown-info-note"> - This feature requires local storage to be enabled - </div> - <ul v-else-if="hasItems"> - <li - v-for="(item, index) in processedItems" - :key="index"> - <button - type="button" - class="filtered-search-history-dropdown-item" - @click="onItemActivated(item.text)"> - <span> - <span - v-for="(token, tokenIndex) in item.tokens" - class="filtered-search-history-dropdown-token"> - <span class="name">{{ token.prefix }}</span><span class="value">{{ token.suffix }}</span> - </span> - </span> - <span class="filtered-search-history-dropdown-search-token"> - {{ item.searchToken }} - </span> - </button> - </li> - <li class="divider"></li> - <li> - <button - type="button" - class="filtered-search-history-clear-button" - @click="onRequestClearRecentSearches($event)"> - Clear recent searches - </button> - </li> - </ul> - <div - v-else - class="dropdown-info-note"> - You don't have any recent searches - </div> - </div> - `, -}; diff --git a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue new file mode 100644 index 00000000000..26618af9515 --- /dev/null +++ b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue @@ -0,0 +1,104 @@ +<script> +import eventHub from '../event_hub'; +import FilteredSearchTokenizer from '../filtered_search_tokenizer'; + +export default { + name: 'RecentSearchesDropdownContent', + props: { + items: { + type: Array, + required: true, + }, + isLocalStorageAvailable: { + type: Boolean, + required: false, + default: true, + }, + allowedKeys: { + type: Array, + required: true, + }, + }, + computed: { + processedItems() { + return this.items.map((item) => { + const { tokens, searchToken } + = FilteredSearchTokenizer.processTokens(item, this.allowedKeys); + + const resultantTokens = tokens.map(token => ({ + prefix: `${token.key}:`, + suffix: `${token.symbol}${token.value}`, + })); + + return { + text: item, + tokens: resultantTokens, + searchToken, + }; + }); + }, + hasItems() { + return this.items.length > 0; + }, + }, + methods: { + onItemActivated(text) { + eventHub.$emit('recentSearchesItemSelected', text); + }, + onRequestClearRecentSearches(e) { + // Stop the dropdown from closing + e.stopPropagation(); + + eventHub.$emit('requestClearRecentSearches'); + }, + }, +}; +</script> +<template> + <div> + <div + v-if="!isLocalStorageAvailable" + class="dropdown-info-note"> + This feature requires local storage to be enabled + </div> + <ul v-else-if="hasItems"> + <li + v-for="(item, index) in processedItems" + :key="`processed-items-${index}`" + > + <button + type="button" + class="filtered-search-history-dropdown-item" + @click="onItemActivated(item.text)"> + <span> + <span + class="filtered-search-history-dropdown-token" + v-for="(token, index) in item.tokens" + :key="`dropdown-token-${index}`" + > + <span class="name">{{ token.prefix }}</span> + <span class="value">{{ token.suffix }}</span> + </span> + </span> + <span class="filtered-search-history-dropdown-search-token"> + {{ item.searchToken }} + </span> + </button> + </li> + <li class="divider"></li> + <li> + <button + type="button" + class="filtered-search-history-clear-button" + @click="onRequestClearRecentSearches($event)"> + Clear recent searches + </button> + </li> + </ul> + <div + v-else + class="dropdown-info-note"> + You don't have any recent searches + </div> + </div> +</template> diff --git a/app/assets/javascripts/filtered_search/recent_searches_root.js b/app/assets/javascripts/filtered_search/recent_searches_root.js index c99ed63c4af..f9338b82acf 100644 --- a/app/assets/javascripts/filtered_search/recent_searches_root.js +++ b/app/assets/javascripts/filtered_search/recent_searches_root.js @@ -1,5 +1,5 @@ import Vue from 'vue'; -import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content'; +import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content.vue'; import eventHub from './event_hub'; class RecentSearchesRoot { @@ -33,7 +33,7 @@ class RecentSearchesRoot { this.vm = new Vue({ el: this.wrapperElement, components: { - 'recent-searches-dropdown-content': RecentSearchesDropdownContent, + RecentSearchesDropdownContent, }, data() { return state; }, template: ` diff --git a/app/assets/javascripts/labels_select.js b/app/assets/javascripts/labels_select.js index 5de48aa49a9..7151ac05a09 100644 --- a/app/assets/javascripts/labels_select.js +++ b/app/assets/javascripts/labels_select.js @@ -213,7 +213,7 @@ export default class LabelsSelect { } } if (label.duplicate) { - color = gl.DropdownUtils.duplicateLabelColor(label.color); + color = DropdownUtils.duplicateLabelColor(label.color); } else { if (label.color != null) { diff --git a/app/assets/javascripts/pages/projects/boards/index.js b/app/assets/javascripts/pages/projects/boards/index.js index 3aeeedbb45d..5cfe8723204 100644 --- a/app/assets/javascripts/pages/projects/boards/index.js +++ b/app/assets/javascripts/pages/projects/boards/index.js @@ -1,7 +1,9 @@ import UsersSelect from '~/users_select'; import ShortcutsNavigation from '~/shortcuts_navigation'; +import initBoards from '~/boards'; document.addEventListener('DOMContentLoaded', () => { new UsersSelect(); // eslint-disable-line no-new new ShortcutsNavigation(); // eslint-disable-line no-new + initBoards(); }); diff --git a/app/assets/javascripts/pages/projects/issues/show/index.js b/app/assets/javascripts/pages/projects/issues/show/index.js index db064e3f801..1e56aa58da2 100644 --- a/app/assets/javascripts/pages/projects/issues/show/index.js +++ b/app/assets/javascripts/pages/projects/issues/show/index.js @@ -3,6 +3,7 @@ import Issue from '~/issue'; import ShortcutsIssuable from '~/shortcuts_issuable'; import ZenMode from '~/zen_mode'; import '~/notes/index'; +import '~/issue_show/index'; document.addEventListener('DOMContentLoaded', () => { new Issue(); // eslint-disable-line no-new diff --git a/app/assets/javascripts/pages/projects/settings/repository/show/index.js b/app/assets/javascripts/pages/projects/settings/repository/show/index.js index d88527351c1..5a6f4138b10 100644 --- a/app/assets/javascripts/pages/projects/settings/repository/show/index.js +++ b/app/assets/javascripts/pages/projects/settings/repository/show/index.js @@ -1,3 +1,7 @@ import initSettingsPanels from '~/settings_panels'; +import initDeployKeys from '~/deploy_keys'; -document.addEventListener('DOMContentLoaded', initSettingsPanels); +document.addEventListener('DOMContentLoaded', () => { + initDeployKeys(); + initSettingsPanels(); +}); diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js index 7ba6c29006a..162f048aac7 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js @@ -227,7 +227,8 @@ export default { @click="handleMergeButtonClick()" :disabled="isMergeButtonDisabled" :class="mergeButtonClass" - type="button"> + type="button" + class="qa-merge-button"> <i v-if="isMakingRequest" class="fa fa-spinner fa-spin" diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue index e9f23b0b113..143fd328d88 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue @@ -111,7 +111,7 @@ js-toggle-container accept-action media space-children" > <button type="button" - class="btn btn-sm btn-reopen btn-success" + class="btn btn-sm btn-reopen btn-success qa-mr-rebase-button" :disabled="isMakingRequest" @click="rebase" > diff --git a/app/controllers/projects/pages_domains_controller.rb b/app/controllers/projects/pages_domains_controller.rb index 15e77d854dc..b71f1e5fef4 100644 --- a/app/controllers/projects/pages_domains_controller.rb +++ b/app/controllers/projects/pages_domains_controller.rb @@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController before_action :require_pages_enabled! before_action :authorize_update_pages!, except: [:show] - before_action :domain, only: [:show, :destroy] + before_action :domain, only: [:show, :destroy, :verify] def show end @@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController @domain = @project.pages_domains.new end + def verify + result = VerifyPagesDomainService.new(@domain).execute + + if result[:status] == :success + flash[:notice] = 'Successfully verified domain ownership' + else + flash[:alert] = 'Failed to verify domain ownership' + end + + redirect_to project_pages_domain_path(@project, @domain) + end + def create @domain = @project.pages_domains.create(pages_domain_params) if @domain.valid? - redirect_to project_pages_path(@project) + redirect_to project_pages_domain_path(@project, @domain) else render 'new' end @@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController end def domain - @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s) + @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s) end end diff --git a/app/controllers/projects/prometheus/metrics_controller.rb b/app/controllers/projects/prometheus/metrics_controller.rb new file mode 100644 index 00000000000..b739d0f0f90 --- /dev/null +++ b/app/controllers/projects/prometheus/metrics_controller.rb @@ -0,0 +1,27 @@ +module Projects + module Prometheus + class MetricsController < Projects::ApplicationController + before_action :authorize_admin_project! + + def active_common + respond_to do |format| + format.json do + matched_metrics = prometheus_service.matched_metrics || {} + + if matched_metrics.any? + render json: matched_metrics + else + head :no_content + end + end + end + end + + private + + def prometheus_service + @prometheus_service ||= project.find_or_initialize_service('prometheus') + end + end + end +end diff --git a/app/controllers/projects/prometheus_controller.rb b/app/controllers/projects/prometheus_controller.rb deleted file mode 100644 index 507468d7102..00000000000 --- a/app/controllers/projects/prometheus_controller.rb +++ /dev/null @@ -1,24 +0,0 @@ -class Projects::PrometheusController < Projects::ApplicationController - before_action :authorize_read_project! - before_action :require_prometheus_metrics! - - def active_metrics - respond_to do |format| - format.json do - matched_metrics = project.prometheus_service.matched_metrics || {} - - if matched_metrics.any? - render json: matched_metrics - else - head :no_content - end - end - end - end - - private - - def require_prometheus_metrics! - render_404 unless project.prometheus_service.present? - end -end diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index e293b3ef329..ab68ecad2ba 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -199,6 +199,7 @@ module ApplicationSettingsHelper :metrics_port, :metrics_sample_interval, :metrics_timeout, + :pages_domain_verification_enabled, :password_authentication_enabled_for_web, :password_authentication_enabled_for_git, :performance_bar_allowed_group_id, diff --git a/app/mailers/emails/pages_domains.rb b/app/mailers/emails/pages_domains.rb new file mode 100644 index 00000000000..0027dfdc36b --- /dev/null +++ b/app/mailers/emails/pages_domains.rb @@ -0,0 +1,43 @@ +module Emails + module PagesDomains + def pages_domain_enabled_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled") + ) + end + + def pages_domain_disabled_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled") + ) + end + + def pages_domain_verification_succeeded_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'") + ) + end + + def pages_domain_verification_failed_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'") + ) + end + end +end diff --git a/app/mailers/notify.rb b/app/mailers/notify.rb index eade0fe278f..45d4fb451d8 100644 --- a/app/mailers/notify.rb +++ b/app/mailers/notify.rb @@ -5,6 +5,7 @@ class Notify < BaseMailer include Emails::Issues include Emails::MergeRequests include Emails::Notes + include Emails::PagesDomains include Emails::Projects include Emails::Profile include Emails::Pipelines diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index d8bf54e0c40..588bd50ed77 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -1,10 +1,14 @@ class PagesDomain < ActiveRecord::Base + VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze + VERIFICATION_THRESHOLD = 3.days.freeze + belongs_to :project validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } validates :certificate, certificate: true, allow_nil: true, allow_blank: true validates :key, certificate_key: true, allow_nil: true, allow_blank: true + validates :verification_code, presence: true, allow_blank: false validate :validate_pages_domain validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? } @@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base key: Gitlab::Application.secrets.db_key_base, algorithm: 'aes-256-cbc' + after_initialize :set_verification_code after_create :update_daemon - after_save :update_daemon + after_update :update_daemon, if: :pages_config_changed? after_destroy :update_daemon + scope :enabled, -> { where('enabled_until >= ?', Time.now ) } + scope :needs_verification, -> do + verified_at = arel_table[:verified_at] + enabled_until = arel_table[:enabled_until] + threshold = Time.now + VERIFICATION_THRESHOLD + + where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold)))) + end + + def verified? + !!verified_at + end + + def unverified? + !verified? + end + + def enabled? + !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present? + end + def to_param domain end @@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base @certificate_text ||= x509.try(:to_text) end + # Verification codes may be TXT records for domain or verification_domain, to + # support the use of CNAME records on domain. + def verification_domain + return unless domain.present? + + "_#{VERIFICATION_KEY}.#{domain}" + end + + def keyed_verification_code + return unless verification_code.present? + + "#{VERIFICATION_KEY}=#{verification_code}" + end + private + def set_verification_code + return if self.verification_code.present? + + self.verification_code = SecureRandom.hex(16) + end + def update_daemon ::Projects::UpdatePagesConfigurationService.new(project).execute end + def pages_config_changed? + project_id_changed? || + domain_changed? || + certificate_changed? || + key_changed? || + became_enabled? || + became_disabled? + end + + def became_enabled? + enabled_until.present? && !enabled_until_was.present? + end + + def became_disabled? + !enabled_until.present? && enabled_until_was.present? + end + def validate_matching_key unless has_matching_key? self.errors.add(:key, "doesn't match the certificate") diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 1bb576ff971..58731451429 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -69,16 +69,16 @@ class PrometheusService < MonitoringService client.ping { success: true, result: 'Checked API endpoint' } - rescue Gitlab::PrometheusError => err + rescue Gitlab::PrometheusClient::Error => err { success: false, result: err } end def environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics)) + with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics)) end def deployment_metrics(deployment) - metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics)) + metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics)) metrics&.merge(deployment_time: deployment.created_at.to_i) || {} end @@ -107,7 +107,7 @@ class PrometheusService < MonitoringService data: data, last_update: Time.now.utc } - rescue Gitlab::PrometheusError => err + rescue Gitlab::PrometheusClient::Error => err { success: false, result: err.message } end @@ -116,10 +116,10 @@ class PrometheusService < MonitoringService Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url)) else cluster = cluster_with_prometheus(environment_id) - raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster + raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster rest_client = client_from_cluster(cluster) - raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client + raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client Gitlab::PrometheusClient.new(rest_client) end @@ -152,9 +152,11 @@ class PrometheusService < MonitoringService cluster.application_prometheus.proxy_client end - def rename_data_to_metrics(metrics) - metrics[:metrics] = metrics.delete :data - metrics + def rename_field(old_field, new_field) + -> (metrics) do + metrics[new_field] = metrics.delete(old_field) + metrics + end end def synchronize_service_state! diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb index 280a2c3afa4..ffde824972c 100644 --- a/app/services/ci/create_trace_artifact_service.rb +++ b/app/services/ci/create_trace_artifact_service.rb @@ -4,13 +4,33 @@ module Ci return if job.job_artifacts_trace job.trace.read do |stream| - if stream.file? - job.create_job_artifacts_trace!( - project: job.project, - file_type: :trace, - file: stream) + break unless stream.file? + + clone_file!(stream.path, JobArtifactUploader.workhorse_upload_path) do |clone_path| + create_job_trace!(job, clone_path) + FileUtils.rm(stream.path) end end end + + private + + def create_job_trace!(job, path) + File.open(path) do |stream| + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream) + end + end + + def clone_file!(src_path, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.copy(src_path, temp_path) + yield(temp_path) + end + end end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 56e941d90ff..e07ecda27b5 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -339,6 +339,30 @@ class NotificationService end end + def pages_domain_verification_succeeded(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later + end + end + + def pages_domain_verification_failed(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_verification_failed_email(domain, user).deliver_later + end + end + + def pages_domain_enabled(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_enabled_email(domain, user).deliver_later + end + end + + def pages_domain_disabled(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_disabled_email(domain, user).deliver_later + end + end + protected def new_resource_email(target, method) @@ -433,6 +457,14 @@ class NotificationService private + def recipients_for_pages_domain(domain) + project = domain.project + + return [] unless project + + notifiable_users(project.team.masters, :watch, target: project) + end + def notifiable?(*args) NotificationRecipientService.notifiable?(*args) end diff --git a/app/services/projects/update_pages_configuration_service.rb b/app/services/projects/update_pages_configuration_service.rb index cacb74b1205..52ff64cc938 100644 --- a/app/services/projects/update_pages_configuration_service.rb +++ b/app/services/projects/update_pages_configuration_service.rb @@ -23,7 +23,7 @@ module Projects end def pages_domains_config - project.pages_domains.map do |domain| + enabled_pages_domains.map do |domain| { domain: domain.domain, certificate: domain.certificate, @@ -32,6 +32,14 @@ module Projects end end + def enabled_pages_domains + if Gitlab::CurrentSettings.pages_domain_verification_enabled? + project.pages_domains.enabled + else + project.pages_domains + end + end + def reload_daemon # GitLab Pages daemon constantly watches for modification time of `pages.path` # It reloads configuration when `pages.path` is modified diff --git a/app/services/verify_pages_domain_service.rb b/app/services/verify_pages_domain_service.rb new file mode 100644 index 00000000000..86166047302 --- /dev/null +++ b/app/services/verify_pages_domain_service.rb @@ -0,0 +1,107 @@ +require 'resolv' + +class VerifyPagesDomainService < BaseService + # The maximum number of seconds to be spent on each DNS lookup + RESOLVER_TIMEOUT_SECONDS = 15 + + # How long verification lasts for + VERIFICATION_PERIOD = 7.days + + attr_reader :domain + + def initialize(domain) + @domain = domain + end + + def execute + return error("No verification code set for #{domain.domain}") unless domain.verification_code.present? + + if !verification_enabled? || dns_record_present? + verify_domain! + elsif expired? + disable_domain! + else + unverify_domain! + end + end + + private + + def verify_domain! + was_disabled = !domain.enabled? + was_unverified = domain.unverified? + + # Prevent any pre-existing grace period from being truncated + reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max + + domain.update!(verified_at: Time.now, enabled_until: reverify) + + if was_disabled + notify(:enabled) + elsif was_unverified + notify(:verification_succeeded) + end + + success + end + + def unverify_domain! + if domain.verified? + domain.update!(verified_at: nil) + notify(:verification_failed) + end + + error("Couldn't verify #{domain.domain}") + end + + def disable_domain! + domain.update!(verified_at: nil, enabled_until: nil) + + notify(:disabled) + + error("Couldn't verify #{domain.domain}. It is now disabled.") + end + + # A domain is only expired until `disable!` has been called + def expired? + domain.enabled_until && domain.enabled_until < Time.now + end + + def dns_record_present? + Resolv::DNS.open do |resolver| + resolver.timeouts = RESOLVER_TIMEOUT_SECONDS + + check(domain.domain, resolver) || check(domain.verification_domain, resolver) + end + end + + def check(domain_name, resolver) + records = parse(txt_records(domain_name, resolver)) + + records.any? do |record| + record == domain.keyed_verification_code || record == domain.verification_code + end + rescue => err + log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}") + false + end + + def txt_records(domain_name, resolver) + resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT) + end + + def parse(records) + records.flat_map(&:strings).flat_map(&:split) + end + + def verification_enabled? + Gitlab::CurrentSettings.pages_domain_verification_enabled? + end + + def notify(type) + return unless verification_enabled? + + Gitlab::AppLogger.info("Pages domain '#{domain.domain}' changed state to '#{type}'") + notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 60f12030f98..20527d31870 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -237,6 +237,17 @@ .col-sm-10 = f.number_field :max_pages_size, class: 'form-control' .help-block 0 for unlimited + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :pages_domain_verification_enabled do + = f.check_box :pages_domain_verification_enabled + Require users to prove ownership of custom domains + .help-block + Domain verification is an essential security measure for public GitLab + sites. Users are required to demonstrate they control a domain before + it is enabled + = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') %fieldset %legend Continuous Integration and Deployment diff --git a/app/views/admin/runners/index.html.haml b/app/views/admin/runners/index.html.haml index 1e52646b1cc..abec3607cab 100644 --- a/app/views/admin/runners/index.html.haml +++ b/app/views/admin/runners/index.html.haml @@ -35,9 +35,8 @@ method: :put, class: 'btn btn-default', data: { confirm: _("Are you sure you want to reset registration token?") } - = render partial: 'ci/runner/how_to_setup_runner', - locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token, - type: 'shared' } + = render partial: 'ci/runner/how_to_setup_shared_runner', + locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token } .append-bottom-20.clearfix .pull-left diff --git a/app/views/ci/runner/_how_to_setup_runner.html.haml b/app/views/ci/runner/_how_to_setup_runner.html.haml index 8db7727b80c..37fb8fbab26 100644 --- a/app/views/ci/runner/_how_to_setup_runner.html.haml +++ b/app/views/ci/runner/_how_to_setup_runner.html.haml @@ -1,16 +1,16 @@ - link = link_to _("GitLab Runner section"), 'https://about.gitlab.com/gitlab-ci/#gitlab-runner', target: '_blank' -.bs-callout.help-callout - %h4= _("How to setup a #{type} Runner for a new project") +.append-bottom-10 + %h4= _("Setup a #{type} Runner manually") - %ol - %li - = _("Install a Runner compatible with GitLab CI") - = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe - %li - = _("Specify the following URL during the Runner setup:") - %code#coordinator_address= root_url(only_path: false) - %li - = _("Use the following registration token during setup:") - %code#registration_token= registration_token - %li - = _("Start the Runner!") +%ol + %li + = _("Install a Runner compatible with GitLab CI") + = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe + %li + = _("Specify the following URL during the Runner setup:") + %code#coordinator_address= root_url(only_path: false) + %li + = _("Use the following registration token during setup:") + %code#registration_token= registration_token + %li + = _("Start the Runner!") diff --git a/app/views/ci/runner/_how_to_setup_shared_runner.html.haml b/app/views/ci/runner/_how_to_setup_shared_runner.html.haml new file mode 100644 index 00000000000..2a190cb9250 --- /dev/null +++ b/app/views/ci/runner/_how_to_setup_shared_runner.html.haml @@ -0,0 +1,3 @@ +.bs-callout.help-callout + = render partial: 'ci/runner/how_to_setup_runner', + locals: { registration_token: registration_token, type: 'shared' } diff --git a/app/views/ci/runner/_how_to_setup_specific_runner.html.haml b/app/views/ci/runner/_how_to_setup_specific_runner.html.haml new file mode 100644 index 00000000000..e765a353fe4 --- /dev/null +++ b/app/views/ci/runner/_how_to_setup_specific_runner.html.haml @@ -0,0 +1,26 @@ +.bs-callout.help-callout + .append-bottom-10 + %h4= _('Setup a specific Runner automatically') + + %p + - link_to_help_page = link_to(_('Learn more about Kubernetes'), + help_page_path('user/project/clusters/index'), + target: '_blank', + rel: 'noopener noreferrer') + + = _('You can easily install a Runner on a Kubernetes cluster. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page } + + %ol + %li + = _('Click the button below to begin the install process by navigating to the Kubernetes page') + %li + = _('Select an existing Kubernetes cluster or create a new one') + %li + = _('From the Kubernetes cluster details view, install Runner from the applications list') + + = link_to _('Install Runner on Kubernetes'), + project_clusters_path(@project), + class: 'btn btn-info' + %hr + = render partial: 'ci/runner/how_to_setup_runner', + locals: { registration_token: registration_token, type: 'specific' } diff --git a/app/views/notify/pages_domain_disabled_email.html.haml b/app/views/notify/pages_domain_disabled_email.html.haml new file mode 100644 index 00000000000..34ce4238a12 --- /dev/null +++ b/app/views/notify/pages_domain_disabled_email.html.haml @@ -0,0 +1,15 @@ +%p + Following a verification check, your GitLab Pages custom domain has been + %strong disabled. + This means that your content is no longer visible at #{link_to @domain.url, @domain.url} +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + If this domain has been disabled in error, please follow + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + to verify and re-enable your domain. +%p + If you no longer wish to use this domain with GitLab Pages, please remove it + from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_disabled_email.text.haml b/app/views/notify/pages_domain_disabled_email.text.haml new file mode 100644 index 00000000000..4e81b054b1f --- /dev/null +++ b/app/views/notify/pages_domain_disabled_email.text.haml @@ -0,0 +1,13 @@ +Following a verification check, your GitLab Pages custom domain has been +**disabled**. This means that your content is no longer visible at #{@domain.url} + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +If this domain has been disabled in error, please follow these instructions +to verify and re-enable your domain: + += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + +If you no longer wish to use this domain with GitLab Pages, please remove it +from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_enabled_email.html.haml b/app/views/notify/pages_domain_enabled_email.html.haml new file mode 100644 index 00000000000..db09e503f65 --- /dev/null +++ b/app/views/notify/pages_domain_enabled_email.html.haml @@ -0,0 +1,11 @@ +%p + Following a verification check, your GitLab Pages custom domain has been + enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url} +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_enabled_email.text.haml b/app/views/notify/pages_domain_enabled_email.text.haml new file mode 100644 index 00000000000..1ed1dbb8315 --- /dev/null +++ b/app/views/notify/pages_domain_enabled_email.text.haml @@ -0,0 +1,9 @@ +Following a verification check, your GitLab Pages custom domain has been +enabled. You should now be able to view your content at #{@domain.url} + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_verification_failed_email.html.haml b/app/views/notify/pages_domain_verification_failed_email.html.haml new file mode 100644 index 00000000000..0bb0eb09fd5 --- /dev/null +++ b/app/views/notify/pages_domain_verification_failed_email.html.haml @@ -0,0 +1,17 @@ +%p + Verification has failed for one of your GitLab Pages custom domains! +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + Unless you take action, it will be disabled on + %strong= @domain.enabled_until.strftime('%F %T.') + Until then, you can view your content at #{link_to @domain.url, @domain.url} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. +%p + If you no longer wish to use this domain with GitLab Pages, please remove it + from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_verification_failed_email.text.haml b/app/views/notify/pages_domain_verification_failed_email.text.haml new file mode 100644 index 00000000000..c14e0e0c24d --- /dev/null +++ b/app/views/notify/pages_domain_verification_failed_email.text.haml @@ -0,0 +1,14 @@ +Verification has failed for one of your GitLab Pages custom domains! + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*. +Until then, you can view your content at #{@domain.url} + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. + +If you no longer wish to use this domain with GitLab Pages, please remove it +from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_verification_succeeded_email.html.haml b/app/views/notify/pages_domain_verification_succeeded_email.html.haml new file mode 100644 index 00000000000..2ead3187b10 --- /dev/null +++ b/app/views/notify/pages_domain_verification_succeeded_email.html.haml @@ -0,0 +1,13 @@ +%p + One of your GitLab Pages custom domains has been successfully verified! +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + This is a notification. No action is required on your part. You can view your + content at #{link_to @domain.url, @domain.url} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_verification_succeeded_email.text.haml b/app/views/notify/pages_domain_verification_succeeded_email.text.haml new file mode 100644 index 00000000000..e7cdbdee420 --- /dev/null +++ b/app/views/notify/pages_domain_verification_succeeded_email.text.haml @@ -0,0 +1,10 @@ +One of your GitLab Pages custom domains has been successfully verified! + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +No action is required on your part. You can view your content at #{@domain.url} + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. diff --git a/app/views/projects/_merge_request_fast_forward_settings.html.haml b/app/views/projects/_merge_request_fast_forward_settings.html.haml index 8129c72feb2..f455522d17c 100644 --- a/app/views/projects/_merge_request_fast_forward_settings.html.haml +++ b/app/views/projects/_merge_request_fast_forward_settings.html.haml @@ -3,7 +3,7 @@ .radio = label_tag :project_merge_method_ff do - = form.radio_button :merge_method, :ff, class: "js-merge-method-radio" + = form.radio_button :merge_method, :ff, class: "js-merge-method-radio qa-radio-button-merge-ff" %strong Fast-forward merge %br %span.descr diff --git a/app/views/projects/clusters/_empty_state.html.haml b/app/views/projects/clusters/_empty_state.html.haml index 600d679b60c..112dde66ff7 100644 --- a/app/views/projects/clusters/_empty_state.html.haml +++ b/app/views/projects/clusters/_empty_state.html.haml @@ -4,7 +4,7 @@ .col-xs-12 .text-content %h4.text-center= s_('ClusterIntegration|Integrate Kubernetes cluster automation') - - link_to_help_page = link_to(s_('ClusterIntegration|Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') + - link_to_help_page = link_to(_('Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') %p= s_('ClusterIntegration|Kubernetes clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page} .text-center diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index 0931ceb1512..b947b91322d 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -85,7 +85,7 @@ .settings-content = form_for [@project.namespace.becomes(Namespace), @project], remote: true, html: { multipart: true, class: "merge-request-settings-form" }, authenticity_token: true do |f| = render 'merge_request_settings', form: f - = f.submit 'Save changes', class: "btn btn-save" + = f.submit 'Save changes', class: "btn btn-save qa-save-merge-request-changes" = render 'export', project: @project diff --git a/app/views/projects/issues/show.html.haml b/app/views/projects/issues/show.html.haml index 91f68d8c419..7bc5c46d64a 100644 --- a/app/views/projects/issues/show.html.haml +++ b/app/views/projects/issues/show.html.haml @@ -82,6 +82,3 @@ = render 'projects/issues/discussion' = render 'shared/issuable/sidebar', issuable: @issue - -= webpack_bundle_tag('common_vue') -= webpack_bundle_tag('issue_show') diff --git a/app/views/projects/pages/_list.html.haml b/app/views/projects/pages/_list.html.haml index a85cda407af..75df92b05a7 100644 --- a/app/views/projects/pages/_list.html.haml +++ b/app/views/projects/pages/_list.html.haml @@ -3,15 +3,26 @@ .panel-heading Domains (#{@domains.count}) %ul.well-list + - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled? - @domains.each do |domain| %li .pull-right = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped" = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped" .clearfix - %span= link_to domain.domain, domain.url + - if verification_enabled + - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success'] + = link_to domain.url, title: tooltip, class: 'has-tooltip' do + = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}") + = domain.domain + - else + = link_to domain.domain, domain.url %p - if domain.subject %span.label.label-gray Certificate: #{domain.subject} - if domain.expired? %span.label.label-danger Expired + - if verification_enabled && domain.unverified? + %li.warning-row + #{domain.domain} is not verified. To learn how to verify ownership, visit your + = link_to 'domain details', project_pages_domain_path(@project, domain) diff --git a/app/views/projects/pages_domains/show.html.haml b/app/views/projects/pages_domains/show.html.haml index 876cac0dacb..72e9203bdb0 100644 --- a/app/views/projects/pages_domains/show.html.haml +++ b/app/views/projects/pages_domains/show.html.haml @@ -1,4 +1,10 @@ - page_title "#{@domain.domain}", 'Pages Domains' +- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled? +- if verification_enabled && @domain.unverified? + %p.alert.alert-warning + %strong + This domain is not verified. You will need to verify ownership before + access is enabled. %h3.page-title Pages Domain @@ -15,9 +21,26 @@ DNS %td %p - To access the domain create a new DNS record: + To access this domain create a new DNS record: %pre #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}. + - if verification_enabled + %tr + %td + Verification status + %td + %p + - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + To #{link_to 'verify ownership', help_link} of your domain, create + this DNS record: + %pre + #{@domain.verification_domain} TXT #{@domain.keyed_verification_code} + %p + - if @domain.verified? + #{@domain.domain} has been successfully verified. + - else + = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm' + %tr %td Certificate diff --git a/app/views/projects/runners/_shared_runners.html.haml b/app/views/projects/runners/_shared_runners.html.haml index b037b57e78a..4fd4ca355a8 100644 --- a/app/views/projects/runners/_shared_runners.html.haml +++ b/app/views/projects/runners/_shared_runners.html.haml @@ -1,6 +1,6 @@ %h3 Shared Runners -.bs-callout.bs-callout-warning.shared-runners-description +.bs-callout.shared-runners-description - if Gitlab::CurrentSettings.shared_runners_text.present? = markdown_field(Gitlab::CurrentSettings.current_application_settings, :shared_runners_text) - else @@ -9,7 +9,7 @@ on GitLab.com). %hr - if @project.shared_runners_enabled? - = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do + = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do Disable shared Runners - else = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do diff --git a/app/views/projects/runners/_specific_runners.html.haml b/app/views/projects/runners/_specific_runners.html.haml index 28ccbf7eb15..f0813e56b71 100644 --- a/app/views/projects/runners/_specific_runners.html.haml +++ b/app/views/projects/runners/_specific_runners.html.haml @@ -1,8 +1,7 @@ %h3 Specific Runners -= render partial: 'ci/runner/how_to_setup_runner', - locals: { registration_token: @project.runners_token, - type: 'specific' } += render partial: 'ci/runner/how_to_setup_specific_runner', + locals: { registration_token: @project.runners_token } - if @project_runners.any? %h4.underlined-title Runners activated for this project diff --git a/app/views/projects/services/prometheus/_show.html.haml b/app/views/projects/services/prometheus/_show.html.haml index 5f38ecd6820..6dc2b85fd32 100644 --- a/app/views/projects/services/prometheus/_show.html.haml +++ b/app/views/projects/services/prometheus/_show.html.haml @@ -7,7 +7,7 @@ = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus') .col-lg-9 - .panel.panel-default.js-panel-monitored-metrics{ data: { "active-metrics" => "#{project_prometheus_active_metrics_path(@project, :json)}" } } + .panel.panel-default.js-panel-monitored-metrics{ data: { active_metrics: active_common_project_prometheus_metrics_path(@project, :json) } } .panel-heading %h3.panel-title = s_('PrometheusService|Monitored') diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index 3077203c2a6..235d532bf98 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -4,7 +4,6 @@ - content_for :page_specific_javascripts do = webpack_bundle_tag('common_vue') - = webpack_bundle_tag('deploy_keys') -# Protected branches & tags use a lot of nested partials. -# The shared parts of the views can be found in the `shared` directory. diff --git a/app/views/shared/boards/_show.html.haml b/app/views/shared/boards/_show.html.haml index a10fc42b82d..3312254f5fb 100644 --- a/app/views/shared/boards/_show.html.haml +++ b/app/views/shared/boards/_show.html.haml @@ -6,7 +6,6 @@ - content_for :page_specific_javascripts do = webpack_bundle_tag 'common_vue' - = webpack_bundle_tag 'boards' %script#js-board-template{ type: "text/x-template" }= render "shared/boards/components/board" %script#js-board-modal-filter{ type: "text/x-template" }= render "shared/issuable/search_bar", type: :boards_modal diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f2c20114534..28a5e5da037 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -3,6 +3,7 @@ - cronjob:expire_build_artifacts - cronjob:gitlab_usage_ping - cronjob:import_export_project_cleanup +- cronjob:pages_domain_verification_cron - cronjob:pipeline_schedule - cronjob:prune_old_events - cronjob:remove_expired_group_links @@ -82,6 +83,7 @@ - new_merge_request - new_note - pages +- pages_domain_verification - post_receive - process_commit - project_cache diff --git a/app/workers/pages_domain_verification_cron_worker.rb b/app/workers/pages_domain_verification_cron_worker.rb new file mode 100644 index 00000000000..a3ff4bd2101 --- /dev/null +++ b/app/workers/pages_domain_verification_cron_worker.rb @@ -0,0 +1,10 @@ +class PagesDomainVerificationCronWorker + include ApplicationWorker + include CronjobQueue + + def perform + PagesDomain.needs_verification.find_each do |domain| + PagesDomainVerificationWorker.perform_async(domain.id) + end + end +end diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb new file mode 100644 index 00000000000..2e93489113c --- /dev/null +++ b/app/workers/pages_domain_verification_worker.rb @@ -0,0 +1,11 @@ +class PagesDomainVerificationWorker + include ApplicationWorker + + def perform(domain_id) + domain = PagesDomain.find_by(id: domain_id) + + return unless domain + + VerifyPagesDomainService.new(domain).execute + end +end diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index e0e6d1418de..fbb14efc525 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -16,43 +16,41 @@ class StuckImportJobsWorker private def mark_projects_without_jid_as_failed! - started_projects_without_jid.each do |project| + enqueued_projects_without_jid.each do |project| project.mark_import_as_failed(error_message) end.count end def mark_projects_with_jid_as_failed! - completed_jids_count = 0 + jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h - started_projects_with_jid.find_in_batches(batch_size: 500) do |group| - jids = group.map(&:import_jid) + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return unless completed_jids.any? - # Find the jobs that aren't currently running or that exceeded the threshold. - completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set + completed_project_ids = jids_and_ids.values_at(*completed_jids) - if completed_jids.any? - completed_jids_count += completed_jids.count - group.each do |project| - project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid) - end + # We select the projects again, because they may have transitioned from + # scheduled/started to finished/failed while we were looking up their Sidekiq status. + completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids) - Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}") - end - end + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}") - completed_jids_count + completed_projects.each do |project| + project.mark_import_as_failed(error_message) + end.count end - def started_projects - Project.with_import_status(:started) + def enqueued_projects + Project.with_import_status(:scheduled, :started) end - def started_projects_with_jid - started_projects.where.not(import_jid: nil) + def enqueued_projects_with_jid + enqueued_projects.where.not(import_jid: nil) end - def started_projects_without_jid - started_projects.where(import_jid: nil) + def enqueued_projects_without_jid + enqueued_projects.where(import_jid: nil) end def error_message diff --git a/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml b/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml new file mode 100644 index 00000000000..f958f3f1272 --- /dev/null +++ b/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml @@ -0,0 +1,5 @@ +--- +title: Add verification for GitLab Pages custom domains +merge_request: +author: +type: security diff --git a/changelogs/unreleased/34130-null-pipes.yml b/changelogs/unreleased/34130-null-pipes.yml deleted file mode 100644 index a56e5cf8db2..00000000000 --- a/changelogs/unreleased/34130-null-pipes.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Prevent MR Widget error when no CI configured -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml b/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml deleted file mode 100644 index 27eee7d943b..00000000000 --- a/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Improve query performance of MembersFinder. -merge_request: 17190 -author: -type: performance diff --git a/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml b/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml deleted file mode 100644 index 507367c98c4..00000000000 --- a/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Enable Legacy Authorization by default on Cluster creations -merge_request: 17302 -author: -type: fixed diff --git a/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml b/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml new file mode 100644 index 00000000000..6cf0de5b3fa --- /dev/null +++ b/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml @@ -0,0 +1,5 @@ +--- +title: Add a button to deploy a runner to a Kubernetes cluster in the settings page +merge_request: 17278 +author: +type: changed diff --git a/changelogs/unreleased/42877-snippets-dashboard-slow.yml b/changelogs/unreleased/42877-snippets-dashboard-slow.yml deleted file mode 100644 index 839b44ad272..00000000000 --- a/changelogs/unreleased/42877-snippets-dashboard-slow.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Improve query performance for snippets dashboard. -merge_request: 17088 -author: -type: performance diff --git a/changelogs/unreleased/43373-fix-cache-index-appending.yml b/changelogs/unreleased/43373-fix-cache-index-appending.yml deleted file mode 100644 index fdb293ea04d..00000000000 --- a/changelogs/unreleased/43373-fix-cache-index-appending.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix issue with cache key being empty when variable used as the key -merge_request: 17260 -author: -type: fixed diff --git a/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml b/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml new file mode 100644 index 00000000000..bda4ec84e5c --- /dev/null +++ b/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml @@ -0,0 +1,5 @@ +--- +title: Fix Group labels load failure when there are duplicate labels present +merge_request: 17353 +author: +type: fixed diff --git a/changelogs/unreleased/dm-stuck-import-jobs-verify.yml b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml new file mode 100644 index 00000000000..ed2c2d30f0d --- /dev/null +++ b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml @@ -0,0 +1,5 @@ +--- +title: Verify project import status again before marking as failed +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/fix-500-for-invalid-upload-path.yml b/changelogs/unreleased/fix-500-for-invalid-upload-path.yml deleted file mode 100644 index a4ce00c64c4..00000000000 --- a/changelogs/unreleased/fix-500-for-invalid-upload-path.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix 500 error when loading an invalid upload URL -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/flipper-caching.yml b/changelogs/unreleased/flipper-caching.yml deleted file mode 100644 index 6db27fd579e..00000000000 --- a/changelogs/unreleased/flipper-caching.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Increase feature flag cache TTL to one hour -merge_request: -author: -type: performance diff --git a/changelogs/unreleased/grpc-unavailable-restart.yml b/changelogs/unreleased/grpc-unavailable-restart.yml new file mode 100644 index 00000000000..5ce08d66004 --- /dev/null +++ b/changelogs/unreleased/grpc-unavailable-restart.yml @@ -0,0 +1,5 @@ +--- +title: Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed +merge_request: 17293 +author: +type: fixed diff --git a/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml b/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml deleted file mode 100644 index 09112fba85e..00000000000 --- a/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Only check LFS integrity for first ref in a push to avoid timeout -merge_request: 17098 -author: -type: performance diff --git a/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml b/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml deleted file mode 100644 index 690536a533b..00000000000 --- a/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix single digit value clipping for stacked progress bar -merge_request: 17217 -author: -type: fixed diff --git a/changelogs/unreleased/minimal-fix-for-artifacts-service.yml b/changelogs/unreleased/minimal-fix-for-artifacts-service.yml new file mode 100644 index 00000000000..11f5bc17759 --- /dev/null +++ b/changelogs/unreleased/minimal-fix-for-artifacts-service.yml @@ -0,0 +1,5 @@ +--- +title: Prevent trace artifact migration to incur data loss +merge_request: 17313 +author: +type: fixed diff --git a/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml b/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml new file mode 100644 index 00000000000..a761d610da1 --- /dev/null +++ b/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml @@ -0,0 +1,5 @@ +--- +title: Return a 404 instead of 403 if the repository does not exist on disk +merge_request: 17341 +author: +type: fixed diff --git a/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml b/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml new file mode 100644 index 00000000000..d65318d7ba1 --- /dev/null +++ b/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml @@ -0,0 +1,5 @@ +--- +title: Move RecentSearchesDropdownContent vue component +merge_request: 16951 +author: George Tsiolis +type: performance diff --git a/changelogs/unreleased/sh-guard-read-only-user-updates.yml b/changelogs/unreleased/sh-guard-read-only-user-updates.yml deleted file mode 100644 index b8dbd840ed9..00000000000 --- a/changelogs/unreleased/sh-guard-read-only-user-updates.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Don't attempt to update user tracked fields if database is in read-only -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/users-autocomplete.yml b/changelogs/unreleased/users-autocomplete.yml deleted file mode 100644 index 2cb078a3a7c..00000000000 --- a/changelogs/unreleased/users-autocomplete.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Improve performance of searching for and autocompleting of users -merge_request: -author: -type: performance diff --git a/changelogs/unreleased/zj-branch-contains-git-message.yml b/changelogs/unreleased/zj-branch-contains-git-message.yml deleted file mode 100644 index ce034e7ec87..00000000000 --- a/changelogs/unreleased/zj-branch-contains-git-message.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Allow branch names to be named the same as the sha it points to -merge_request: -author: -type: fixed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index bbc2bcfb0cc..bd696a7f2c5 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -214,6 +214,10 @@ production: &base repository_archive_cache_worker: cron: "0 * * * *" + # Verify custom GitLab Pages domains + pages_domain_verification_cron_worker: + cron: "*/15 * * * *" + registry: # enabled: true # host: registry.example.com diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 17a8801f7bc..ea0dee7af53 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *' Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker' +Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *' +Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker' + # # GitLab Shell # diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 0f164e628f9..161fb185c9b 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -10,7 +10,7 @@ Sidekiq.configure_server do |config| config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] - chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] + chain.add Gitlab::SidekiqMiddleware::Shutdown chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0' chain.add Gitlab::SidekiqStatus::ServerMiddleware end diff --git a/config/routes/project.rb b/config/routes/project.rb index 1912808f9c0..8fe545b721e 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do end resource :pages, only: [:show, :destroy] do - resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } + resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do + member do + post :verify + end + end end resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do @@ -74,7 +78,9 @@ constraints(ProjectUrlConstrainer.new) do resource :mattermost, only: [:new, :create] namespace :prometheus do - get :active_metrics + resources :metrics, constraints: { id: %r{[^\/]+} }, only: [] do + get :active_common, on: :collection + end end resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 31a38f2b508..f037e3d1221 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -67,3 +67,4 @@ - [gcp_cluster, 1] - [project_migrate_hashed_storage, 1] - [storage_migrator, 1] + - [pages_domain_verification, 1] diff --git a/config/webpack.config.js b/config/webpack.config.js index 94ff39485fb..be827903a6a 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -52,20 +52,15 @@ var config = { entry: { balsamiq_viewer: './blob/balsamiq_viewer.js', blob: './blob_edit/blob_bundle.js', - boards: './boards/boards_bundle.js', common: './commons/index.js', common_vue: './vue_shared/vue_resource_interceptor.js', cycle_analytics: './cycle_analytics/cycle_analytics_bundle.js', commit_pipelines: './commit/pipelines/pipelines_bundle.js', - deploy_keys: './deploy_keys/index.js', diff_notes: './diff_notes/diff_notes_bundle.js', environments: './environments/environments_bundle.js', environments_folder: './environments/folder/environments_folder_bundle.js', filtered_search: './filtered_search/filtered_search_bundle.js', help: './help/help.js', - issue_show: './issue_show/index.js', - locale: './locale/index.js', - main: './main.js', merge_conflicts: './merge_conflicts/merge_conflicts_bundle.js', monitoring: './monitoring/monitoring_bundle.js', network: './network/network_bundle.js', @@ -78,18 +73,24 @@ var config = { protected_branches: './protected_branches', protected_tags: './protected_tags', registry_list: './registry/index.js', - ide: './ide/index.js', sidebar: './sidebar/sidebar_bundle.js', snippet: './snippet/snippet_bundle.js', sketch_viewer: './blob/sketch_viewer.js', stl_viewer: './blob/stl_viewer.js', terminal: './terminal/terminal_bundle.js', - u2f: ['vendor/u2f'], ui_development_kit: './ui_development_kit.js', - raven: './raven/index.js', vue_merge_request_widget: './vue_merge_request_widget/index.js', - test: './test.js', two_factor_auth: './two_factor_auth.js', + + + common: './commons/index.js', + common_vue: './vue_shared/vue_resource_interceptor.js', + locale: './locale/index.js', + main: './main.js', + ide: './ide/index.js', + raven: './raven/index.js', + test: './test.js', + u2f: ['vendor/u2f'], webpack_runtime: './webpack.js', }, @@ -252,7 +253,6 @@ var config = { 'environments_folder', 'filtered_search', 'groups', - 'issue_show', 'merge_conflicts', 'monitoring', 'notebook_viewer', diff --git a/db/migrate/20180216120000_add_pages_domain_verification.rb b/db/migrate/20180216120000_add_pages_domain_verification.rb new file mode 100644 index 00000000000..8b7cae92285 --- /dev/null +++ b/db/migrate/20180216120000_add_pages_domain_verification.rb @@ -0,0 +1,8 @@ +class AddPagesDomainVerification < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :verified_at, :datetime_with_timezone + add_column :pages_domains, :verification_code, :string + end +end diff --git a/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb new file mode 100644 index 00000000000..825dfb52dce --- /dev/null +++ b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb @@ -0,0 +1,15 @@ +class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, :verified_at + end + + def down + remove_concurrent_index :pages_domains, :verified_at + end +end diff --git a/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb new file mode 100644 index 00000000000..06d458028b3 --- /dev/null +++ b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb @@ -0,0 +1,7 @@ +class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false + end +end diff --git a/db/migrate/20180216120030_add_pages_domain_enabled_until.rb b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb new file mode 100644 index 00000000000..b40653044dd --- /dev/null +++ b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb @@ -0,0 +1,7 @@ +class AddPagesDomainEnabledUntil < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :enabled_until, :datetime_with_timezone + end +end diff --git a/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb new file mode 100644 index 00000000000..00f6e4979da --- /dev/null +++ b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb @@ -0,0 +1,17 @@ +class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, [:project_id, :enabled_until] + add_concurrent_index :pages_domains, [:verified_at, :enabled_until] + end + + def down + remove_concurrent_index :pages_domains, [:verified_at, :enabled_until] + remove_concurrent_index :pages_domains, [:project_id, :enabled_until] + end +end diff --git a/db/migrate/20180216120050_pages_domains_verification_grace_period.rb b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb new file mode 100644 index 00000000000..d7f8634b536 --- /dev/null +++ b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb @@ -0,0 +1,26 @@ +class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + now = Time.now + grace = now + 30.days + + PagesDomain.each_batch do |relation| + relation.update_all(verified_at: now, enabled_until: grace) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb new file mode 100644 index 00000000000..d423673d2a5 --- /dev/null +++ b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb @@ -0,0 +1,41 @@ +class FillPagesDomainVerificationCode < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + PagesDomain.where(verification_code: [nil, '']).each_batch do |relation| + connection.execute(set_codes_sql(relation)) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + + change_column_null(:pages_domains, :verification_code, false) + end + + def down + change_column_null(:pages_domains, :verification_code, true) + end + + private + + def set_codes_sql(relation) + ids = relation.pluck(:id) + whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" } + + <<~SQL + UPDATE pages_domains + SET verification_code = + CASE id + #{whens.join("\n")} + END + WHERE id IN(#{ids.join(',')}) + SQL + end +end diff --git a/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb new file mode 100644 index 00000000000..bf9bf4e660f --- /dev/null +++ b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb @@ -0,0 +1,16 @@ +class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + def up + PagesDomain.each_batch do |relation| + ids = relation.pluck(:id).map { |id| [id] } + PagesDomainVerificationWorker.bulk_perform_async(ids) + end + end + + def down + # no-op + end +end diff --git a/db/schema.rb b/db/schema.rb index c0ce87302cf..5bb461169f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180215181245) do +ActiveRecord::Schema.define(version: 20180216121030) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180215181245) do t.integer "gitaly_timeout_fast", default: 10, null: false t.boolean "authorized_keys_enabled", default: true, null: false t.string "auto_devops_domain" + t.boolean "pages_domain_verification_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -1313,10 +1314,16 @@ ActiveRecord::Schema.define(version: 20180215181245) do t.string "encrypted_key_iv" t.string "encrypted_key_salt" t.string "domain" + t.datetime_with_timezone "verified_at" + t.string "verification_code", null: false + t.datetime_with_timezone "enabled_until" end add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree + add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree + add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree + add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree create_table "personal_access_tokens", force: :cascade do |t| t.integer "user_id", null: false diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index edb3e4c961e..00c631fdaae 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -226,6 +226,18 @@ world. Custom domains and TLS are supported. 1. [Reconfigure GitLab][reconfigure] +### Custom domain verification + +To prevent malicious users from hijacking domains that don't belong to them, +GitLab supports [custom domain verification](../../user/project/pages/getting_started_part_three.md#dns-txt-record). +When adding a custom domain, users will be required to prove they own it by +adding a GitLab-controlled verification code to the DNS records for that domain. + +If your userbase is private or otherwise trusted, you can disable the +verification requirement. Navigate to `Admin area âž” Settings` and uncheck +**Require users to prove ownership of custom domains** in the Pages section. +This setting is enabled by default. + ## Change storage path Follow the steps below to change the default path where GitLab Pages' contents diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index d3e9bf9e6a8..10e6321eb82 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -8,8 +8,8 @@ Milestones allow you to organize issues and merge requests into a cohesive group ## Project milestones and group milestones -- **Project milestones** can be assigned to issues or merge requests in that project only. -- **Group milestones** can be assigned to any issue or merge request of any project in that group. +- **Project milestones** can be assigned to issues or merge requests in that project only. +- **Group milestones** can be assigned to any issue or merge request of any project in that group. - In the [future](https://gitlab.com/gitlab-org/gitlab-ce/issues/36862), you will be able to assign group milestones to issues and merge reqeusts of projects in [subgroups](../../group/subgroups/index.md). ## Creating milestones @@ -108,4 +108,4 @@ The milestone sidebar on the milestone view shows the following: For project milestones only, the milestone sidebar shows the total issue weight of all issues that have the milestone assigned. -
\ No newline at end of file + diff --git a/doc/user/project/pages/getting_started_part_three.md b/doc/user/project/pages/getting_started_part_three.md index b6cf68a02a2..430fe3af1f8 100644 --- a/doc/user/project/pages/getting_started_part_three.md +++ b/doc/user/project/pages/getting_started_part_three.md @@ -62,7 +62,7 @@ for the most popular hosting services: - [Microsoft](https://msdn.microsoft.com/en-us/library/bb727018.aspx) If your hosting service is not listed above, you can just try to -search the web for "how to add dns record on <my hosting service>". +search the web for `how to add dns record on <my hosting service>`. ### DNS A record @@ -95,12 +95,32 @@ without any `/project-name`.  -### TL;DR +#### DNS TXT record + +Unless your GitLab administrator has [disabled custom domain verification](../../../administration/pages/index.md#custom-domain-verification), +you'll have to prove that you own the domain by creating a `TXT` record +containing a verification code. The code will be displayed after you +[add your custom domain to GitLab Pages settings](#add-your-custom-domain-to-gitlab-pages-settings). + +If using a [DNS A record](#dns-a-record), you can place the TXT record directly +under the domain. If using a [DNS CNAME record](#dns-cname-record), the two record types won't +co-exist, so you need to place the TXT record in a special subdomain of its own. + +#### TL;DR + +If the domain has multiple uses (e.g., you host email on it as well): | From | DNS Record | To | | ---- | ---------- | -- | | domain.com | A | 52.167.214.135 | -| subdomain.domain.com | CNAME | namespace.gitlab.io | +| domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff | + +If the domain is dedicated to GitLab Pages use and no other services run on it: + +| From | DNS Record | To | +| ---- | ---------- | -- | +| subdomain.domain.com | CNAME | gitlab.io | +| _gitlab-pages-verification-code.subdomain.domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff | > **Notes**: > @@ -121,6 +141,17 @@ your site will be accessible only via HTTP:  +Once you have added a new domain, you will need to **verify your ownership** +(unless the GitLab administrator has disabled this feature). A verification code +will be shown to you; add it as a [DNS TXT record](#dns-txt-record), then press +the "Verify ownership" button to activate your new domain: + + + +Once your domain has been verified, leave the verification record in place - +your domain will be periodically reverified, and may be disabled if the record +is removed. + You can add more than one alias (custom domains and subdomains) to the same project. An alias can be understood as having many doors leading to the same room. @@ -128,8 +159,8 @@ All the aliases you've set to your site will be listed on **Setting > Pages**. From that page, you can view, add, and remove them. Note that [DNS propagation may take some time (up to 24h)](http://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/domain-names-dns-changes), -although it's usually a matter of minutes to complete. Until it does, visit attempts -to your domain will respond with a 404. +although it's usually a matter of minutes to complete. Until it does, verification +will fail and attempts to visit your domain will respond with a 404. Read through the [general documentation on GitLab Pages](introduction.md#add-a-custom-domain-to-your-pages-website) to learn more about adding custom domains to GitLab Pages sites. diff --git a/doc/user/project/pages/img/verify_your_domain.png b/doc/user/project/pages/img/verify_your_domain.png Binary files differnew file mode 100644 index 00000000000..89c69cac9a5 --- /dev/null +++ b/doc/user/project/pages/img/verify_your_domain.png diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 45c737c6c29..167878ba600 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1154,6 +1154,10 @@ module API expose :domain expose :url expose :project_id + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, @@ -1165,6 +1169,10 @@ module API class PagesDomain < Grape::Entity expose :domain expose :url + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, if: ->(pages_domain, _) { pages_domain.certificate? }, using: PagesDomainCertificate do |pages_domain| diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bbdb593d4e2..6400089a22f 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -199,7 +199,7 @@ module Gitlab def check_repository_existence! unless repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + raise NotFoundError, ERROR_MESSAGES[:no_repo] end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c5d3e944f7d..9cd76630484 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -125,6 +125,8 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + rescue GRPC::Unavailable => ex + handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -135,6 +137,27 @@ module Gitlab duration) end + def self.handle_grpc_unavailable!(ex) + status = ex.to_status + raise ex unless status.details == 'Endpoint read failed' + + # There is a bug in grpc 1.8.x that causes a client process to get stuck + # always raising '14:Endpoint read failed'. The only thing that we can + # do to recover is to restart the process. + # + # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + + if Sidekiq.server? + raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) + else + # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. + Process.kill('QUIT', Process.pid) + end + + raise ex + end + private_class_method :handle_grpc_unavailable! + def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 729fef34b35..e91c6fb2e27 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -6,9 +6,14 @@ module Gitlab attr_accessor :name, :priority, :metrics validates :name, :priority, :metrics, presence: true - def self.all + def self.common_metrics AdditionalMetricsParser.load_groups_from_yaml end + + # EE only + def self.for_project(_) + common_metrics + end end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 294a6ae34ca..972ab75d1d5 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( + deployment.project, common_query_context( deployment.environment, timeframe_start: (deployment.created_at - 30.minutes).to_f, diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index 32fe8201a8d..9273e69e158 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( + environment.project, common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) ) end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 4c3edccc71a..5710ad47c1a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -18,7 +18,7 @@ module Gitlab private def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) lookup = active_series_lookup(metrics_groups) groups = {} diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 5cddc96a643..0c280dc9a3c 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -2,10 +2,10 @@ module Gitlab module Prometheus module Queries module QueryAdditionalMetrics - def query_metrics(query_context) + def query_metrics(project, query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics.map do |group| + groups = matched_metrics(project).map do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -60,8 +60,8 @@ module Gitlab @available_metrics ||= client_label_values || [] end - def matched_metrics - result = Gitlab::Prometheus::MetricGroup.all.map do |group| + def matched_metrics(project) + result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group| group.metrics.select! do |metric| metric.required_metrics.all?(&available_metrics.method(:include?)) end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 10527972663..659021c9ac9 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -1,8 +1,9 @@ module Gitlab - PrometheusError = Class.new(StandardError) - # Helper methods to interact with Prometheus network services & resources class PrometheusClient + Error = Class.new(StandardError) + QueryError = Class.new(Gitlab::PrometheusClient::Error) + attr_reader :rest_client, :headers def initialize(rest_client) @@ -22,10 +23,10 @@ module Gitlab def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', - query: query, - start: start.to_f, - end: stop.to_f, - step: 1.minute.to_i) + query: query, + start: start.to_f, + end: stop.to_f, + step: 1.minute.to_i) end end @@ -43,22 +44,22 @@ module Gitlab path = ['api', 'v1', type].join('/') get(path, args) rescue JSON::ParserError - raise PrometheusError, 'Parsing response failed' + raise PrometheusClient::Error, 'Parsing response failed' rescue Errno::ECONNREFUSED - raise PrometheusError, 'Connection refused' + raise PrometheusClient::Error, 'Connection refused' end def get(path, args) response = rest_client[path].get(params: args) handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex handle_exception_response(ex.response) rescue RestClient::Exception - raise PrometheusError, "Network connection error" + raise PrometheusClient::Error, "Network connection error" end def handle_response(response) @@ -66,16 +67,18 @@ module Gitlab if response.code == 200 && json_data['status'] == 'success' json_data['data'] || {} else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end def handle_exception_response(response) - if response.code == 400 + if response.code == 200 && response['status'] == 'success' + response['data'] || {} + elsif response.code == 400 json_data = JSON.parse(response.body) - raise PrometheusError, json_data['error'] || 'Bad data received' + raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index b89ae2505c9..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - def call(worker, job, queue) - yield - - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - return unless MUTEX.try_lock - - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - Process.kill(signal, pid) - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb new file mode 100644 index 00000000000..c2b8d6de66e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -0,0 +1,133 @@ +require 'mutex_m' + +module Gitlab + module SidekiqMiddleware + class Shutdown + extend Mutex_m + + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # This exception can be used to request that the middleware start shutting down Sidekiq + WantShutdown = Class.new(StandardError) + + ShutdownWithoutRaise = Class.new(WantShutdown) + private_constant :ShutdownWithoutRaise + + # For testing only, to avoid race conditions (?) in Rspec mocks. + attr_reader :trace + + # We store the shutdown thread in a class variable to ensure that there + # can be only one shutdown thread in the process. + def self.create_shutdown_thread + mu_synchronize do + return unless @shutdown_thread.nil? + + @shutdown_thread = Thread.new { yield } + end + end + + # For testing only: so we can wait for the shutdown thread to finish. + def self.shutdown_thread + mu_synchronize { @shutdown_thread } + end + + # For testing only: so that we can reset the global state before each test. + def self.clear_shutdown_thread + mu_synchronize { @shutdown_thread = nil } + end + + def initialize + @trace = Queue.new if Rails.env.test? + end + + def call(worker, job, queue) + shutdown_exception = nil + + begin + yield + check_rss! + rescue WantShutdown => ex + shutdown_exception = ex + end + + return unless shutdown_exception + + self.class.create_shutdown_thread do + do_shutdown(worker, job, shutdown_exception) + end + + raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) + end + + private + + def do_shutdown(worker, job, shutdown_exception) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ + "#{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + + def check_rss! + return unless MAX_RSS > 0 + + current_rss = get_rss + return unless current_rss > MAX_RSS + + raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + kill(signal, pid) + end + + def pid + Process.pid + end + + def sleep(time) + if Rails.env.test? + @trace << [:sleep, time] + else + Kernel.sleep(time) + end + end + + def kill(signal, pid) + if Rails.env.test? + @trace << [:kill, signal, pid] + else + Process.kill(signal, pid) + end + end + end + end +end @@ -130,6 +130,7 @@ module QA autoload :DeployKeys, 'qa/page/project/settings/deploy_keys' autoload :SecretVariables, 'qa/page/project/settings/secret_variables' autoload :Runners, 'qa/page/project/settings/runners' + autoload :MergeRequest, 'qa/page/project/settings/merge_request' end module Issue @@ -145,6 +146,7 @@ module QA module MergeRequest autoload :New, 'qa/page/merge_request/new' + autoload :Show, 'qa/page/merge_request/show' end module Admin diff --git a/qa/qa/factory/base.rb b/qa/qa/factory/base.rb index bd66b74a164..afaa96b4541 100644 --- a/qa/qa/factory/base.rb +++ b/qa/qa/factory/base.rb @@ -22,7 +22,7 @@ module QA factory.fabricate!(*args) - return Factory::Product.populate!(self) + return Factory::Product.populate!(factory) end end diff --git a/qa/qa/factory/product.rb b/qa/qa/factory/product.rb index d004e642f9b..996b7f14f61 100644 --- a/qa/qa/factory/product.rb +++ b/qa/qa/factory/product.rb @@ -17,8 +17,9 @@ module QA def self.populate!(factory) new.tap do |product| - factory.attributes.each_value do |attribute| - product.instance_exec(&attribute.block).tap do |value| + factory.class.attributes.each_value do |attribute| + product.instance_exec(factory, attribute.block) do |factory, block| + value = block.call(factory) product.define_singleton_method(attribute.name) { value } end end diff --git a/qa/qa/factory/repository/push.rb b/qa/qa/factory/repository/push.rb index 2f4de4173d4..6e8905cde78 100644 --- a/qa/qa/factory/repository/push.rb +++ b/qa/qa/factory/repository/push.rb @@ -2,7 +2,7 @@ module QA module Factory module Repository class Push < Factory::Base - attr_writer :file_name, :file_content, :commit_message, :branch_name + attr_writer :file_name, :file_content, :commit_message, :branch_name, :new_branch dependency Factory::Resource::Project, as: :project do |project| project.name = 'project-with-code' @@ -14,6 +14,7 @@ module QA @file_content = '# This is test project' @commit_message = "Add #{@file_name}" @branch_name = 'master' + @new_branch = true end def fabricate! @@ -29,6 +30,7 @@ module QA repository.clone repository.configure_identity('GitLab QA', 'root@gitlab.com') + repository.checkout(@branch_name) unless @new_branch repository.add_file(@file_name, @file_content) repository.commit(@commit_message) repository.push_changes(@branch_name) diff --git a/qa/qa/factory/resource/merge_request.rb b/qa/qa/factory/resource/merge_request.rb index ce04e904aaf..539fe6b8a70 100644 --- a/qa/qa/factory/resource/merge_request.rb +++ b/qa/qa/factory/resource/merge_request.rb @@ -9,11 +9,20 @@ module QA :source_branch, :target_branch + product :project do |factory| + factory.project + end + + product :source_branch do |factory| + factory.source_branch + end + dependency Factory::Resource::Project, as: :project do |project| project.name = 'project-with-merge-request' end dependency Factory::Repository::Push, as: :target do |push, factory| + factory.project.visit! push.project = factory.project push.branch_name = "master:#{factory.target_branch}" end diff --git a/qa/qa/git/repository.rb b/qa/qa/git/repository.rb index 4c4ef3ef477..b3150e8f3fa 100644 --- a/qa/qa/git/repository.rb +++ b/qa/qa/git/repository.rb @@ -36,6 +36,10 @@ module QA `git clone #{opts} #{@uri.to_s} ./ #{suppress_output}` end + def checkout(branch_name) + `git checkout "#{branch_name}"` + end + def shallow_clone clone('--depth 1') end diff --git a/qa/qa/page/merge_request/show.rb b/qa/qa/page/merge_request/show.rb new file mode 100644 index 00000000000..35875487da8 --- /dev/null +++ b/qa/qa/page/merge_request/show.rb @@ -0,0 +1,46 @@ +module QA + module Page + module MergeRequest + class Show < Page::Base + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js' do + element :merge_button + end + + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_merged.vue' do + element :merged_status, 'The changes were merged into' + end + + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue' do + element :mr_rebase_button + element :fast_forward_nessage, "Fast-forward merge is not possible" + end + + def rebase! + wait(reload: false) do + click_element :mr_rebase_button + + has_text?("The source branch HEAD has recently changed.") + end + end + + def fast_forward_possible? + !has_text?("Fast-forward merge is not possible") + end + + def has_merge_button? + refresh + + has_selector?('.accept-merge-request') + end + + def merge! + wait(reload: false) do + click_element :merge_button + + has_text?("The changes were merged into") + end + end + end + end + end +end diff --git a/qa/qa/page/project/settings/merge_request.rb b/qa/qa/page/project/settings/merge_request.rb new file mode 100644 index 00000000000..b147c91b467 --- /dev/null +++ b/qa/qa/page/project/settings/merge_request.rb @@ -0,0 +1,27 @@ +module QA + module Page + module Project + module Settings + class MergeRequest < QA::Page::Base + include Common + + view 'app/views/projects/_merge_request_fast_forward_settings.html.haml' do + element :radio_button_merge_ff + end + + view 'app/views/projects/edit.html.haml' do + element :merge_request_settings, 'Merge request settings' + element :save_merge_request_changes + end + + def enable_ff_only + expand_section('Merge request settings') do + click_element :radio_button_merge_ff + click_element :save_merge_request_changes + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/merge_request/rebase_spec.rb b/qa/qa/specs/features/merge_request/rebase_spec.rb new file mode 100644 index 00000000000..2a44d42af6f --- /dev/null +++ b/qa/qa/specs/features/merge_request/rebase_spec.rb @@ -0,0 +1,39 @@ +module QA + feature 'merge request rebase', :core do + scenario 'rebases source branch of merge request' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + project = Factory::Resource::Project.fabricate! do |project| + project.name = "only-fast-forward" + end + + Page::Menu::Side.act { go_to_settings } + Page::Project::Settings::MergeRequest.act { enable_ff_only } + + merge_request = Factory::Resource::MergeRequest.fabricate! do |merge_request| + merge_request.project = project + merge_request.title = 'Needs rebasing' + end + + Factory::Repository::Push.fabricate! do |push| + push.project = project + push.file_name = "other.txt" + push.file_content = "New file added!" + end + + merge_request.visit! + + Page::MergeRequest::Show.perform do |merge_request| + expect(merge_request).to have_content('Needs rebasing') + expect(merge_request).not_to be_fast_forward_possible + expect(merge_request).not_to have_merge_button + + merge_request.rebase! + + expect(merge_request).to have_merge_button + expect(merge_request.fast_forward_possible?).to be_truthy + end + end + end +end diff --git a/qa/spec/factory/base_spec.rb b/qa/spec/factory/base_spec.rb index c5663049be8..04e04886699 100644 --- a/qa/spec/factory/base_spec.rb +++ b/qa/spec/factory/base_spec.rb @@ -7,6 +7,7 @@ describe QA::Factory::Base do before do allow(QA::Factory::Product).to receive(:new).and_return(product) + allow(QA::Factory::Product).to receive(:populate!).and_return(product) end it 'instantiates the factory and calls factory method' do @@ -76,6 +77,7 @@ describe QA::Factory::Base do allow(subject).to receive(:new).and_return(instance) allow(instance).to receive(:mydep).and_return(nil) allow(QA::Factory::Product).to receive(:new) + allow(QA::Factory::Product).to receive(:populate!) end it 'builds all dependencies first' do @@ -89,8 +91,16 @@ describe QA::Factory::Base do describe '.product' do subject do Class.new(described_class) do + def fabricate! + "any" + end + + # Defined only to be stubbed + def self.find_page + end + product :token do - page.do_something_on_page! + find_page.do_something_on_page! 'resulting value' end end @@ -105,16 +115,17 @@ describe QA::Factory::Base do let(:page) { spy('page') } before do - allow(subject).to receive(:new).and_return(factory) + allow(factory).to receive(:class).and_return(subject) allow(QA::Factory::Product).to receive(:new).and_return(product) allow(product).to receive(:page).and_return(page) + allow(subject).to receive(:find_page).and_return(page) end it 'populates product after fabrication' do subject.fabricate! - expect(page).to have_received(:do_something_on_page!) expect(product.token).to eq 'resulting value' + expect(page).to have_received(:do_something_on_page!) end end end diff --git a/qa/spec/factory/product_spec.rb b/qa/spec/factory/product_spec.rb index fdfb1ec90cc..f245aabbf43 100644 --- a/qa/spec/factory/product_spec.rb +++ b/qa/spec/factory/product_spec.rb @@ -1,9 +1,20 @@ describe QA::Factory::Product do - let(:factory) { spy('factory') } + let(:factory) do + QA::Factory::Base.new + end + + let(:attributes) do + { test: QA::Factory::Product::Attribute.new(:test, proc { 'returned' }) } + end + let(:product) { spy('product') } + before do + allow(QA::Factory::Base).to receive(:attributes).and_return(attributes) + end + describe '.populate!' do - it 'returns a fabrication product' do + it 'returns a fabrication product and define factory attributes as its methods' do expect(described_class).to receive(:new).and_return(product) result = described_class.populate!(factory) do |instance| @@ -11,6 +22,7 @@ describe QA::Factory::Product do end expect(result).to be product + expect(result.test).to eq('returned') end end diff --git a/spec/controllers/projects/pages_domains_controller_spec.rb b/spec/controllers/projects/pages_domains_controller_spec.rb index e9e7d357d9c..2192fd5cae2 100644 --- a/spec/controllers/projects/pages_domains_controller_spec.rb +++ b/spec/controllers/projects/pages_domains_controller_spec.rb @@ -46,7 +46,46 @@ describe Projects::PagesDomainsController do post(:create, request_params.merge(pages_domain: pages_domain_params)) end.to change { PagesDomain.count }.by(1) - expect(response).to redirect_to(project_pages_path(project)) + created_domain = PagesDomain.reorder(:id).last + + expect(created_domain).to be_present + expect(response).to redirect_to(project_pages_domain_path(project, created_domain)) + end + end + + describe 'POST verify' do + let(:params) { request_params.merge(id: pages_domain.domain) } + + def stub_service + service = double(:service) + + expect(VerifyPagesDomainService).to receive(:new) { service } + + service + end + + it 'handles verification success' do + expect(stub_service).to receive(:execute).and_return(status: :success) + + post :verify, params + + expect(response).to redirect_to project_pages_domain_path(project, pages_domain) + expect(flash[:notice]).to eq('Successfully verified domain ownership') + end + + it 'handles verification failure' do + expect(stub_service).to receive(:execute).and_return(status: :failed) + + post :verify, params + + expect(response).to redirect_to project_pages_domain_path(project, pages_domain) + expect(flash[:alert]).to eq('Failed to verify domain ownership') + end + + it 'returns a 404 response for an unknown domain' do + post :verify, request_params.merge(id: 'unknown-domain') + + expect(response).to have_gitlab_http_status(404) end end diff --git a/spec/controllers/projects/prometheus_controller_spec.rb b/spec/controllers/projects/prometheus/metrics_controller_spec.rb index bbfe78d305a..f17f819feee 100644 --- a/spec/controllers/projects/prometheus_controller_spec.rb +++ b/spec/controllers/projects/prometheus/metrics_controller_spec.rb @@ -1,20 +1,20 @@ -require('spec_helper') +require 'spec_helper' -describe Projects::PrometheusController do +describe Projects::Prometheus::MetricsController do let(:user) { create(:user) } - let!(:project) { create(:project) } + let(:project) { create(:project) } let(:prometheus_service) { double('prometheus_service') } before do allow(controller).to receive(:project).and_return(project) - allow(project).to receive(:prometheus_service).and_return(prometheus_service) + allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return(prometheus_service) project.add_master(user) sign_in(user) end - describe 'GET #active_metrics' do + describe 'GET #active_common' do context 'when prometheus metrics are enabled' do context 'when data is not present' do before do @@ -22,7 +22,7 @@ describe Projects::PrometheusController do end it 'returns no content response' do - get :active_metrics, project_params(format: :json) + get :active_common, project_params(format: :json) expect(response).to have_gitlab_http_status(204) end @@ -36,7 +36,7 @@ describe Projects::PrometheusController do end it 'returns no content response' do - get :active_metrics, project_params(format: :json) + get :active_common, project_params(format: :json) expect(response).to have_gitlab_http_status(200) expect(json_response).to eq(sample_response.deep_stringify_keys) @@ -45,7 +45,7 @@ describe Projects::PrometheusController do context 'when requesting non json response' do it 'returns not found response' do - get :active_metrics, project_params + get :active_common, project_params expect(response).to have_gitlab_http_status(404) end diff --git a/spec/factories/pages_domains.rb b/spec/factories/pages_domains.rb index 61b04708da2..35b44e1c52e 100644 --- a/spec/factories/pages_domains.rb +++ b/spec/factories/pages_domains.rb @@ -1,6 +1,25 @@ FactoryBot.define do factory :pages_domain, class: 'PagesDomain' do - domain 'my.domain.com' + sequence(:domain) { |n| "my#{n}.domain.com" } + verified_at { Time.now } + enabled_until { 1.week.from_now } + + trait :disabled do + verified_at nil + enabled_until nil + end + + trait :unverified do + verified_at nil + end + + trait :reverify do + enabled_until { 1.hour.from_now } + end + + trait :expired do + enabled_until { 1.hour.ago } + end trait :with_certificate do certificate '-----BEGIN CERTIFICATE----- diff --git a/spec/features/admin/admin_runners_spec.rb b/spec/features/admin/admin_runners_spec.rb index a01c129defd..7eeed7da998 100644 --- a/spec/features/admin/admin_runners_spec.rb +++ b/spec/features/admin/admin_runners_spec.rb @@ -19,7 +19,7 @@ describe "Admin Runners" do end it 'has all necessary texts' do - expect(page).to have_text "How to setup" + expect(page).to have_text "Setup a shared Runner manually" expect(page).to have_text "Runners with last contact more than a minute ago: 1" end @@ -54,7 +54,7 @@ describe "Admin Runners" do end it 'has all necessary texts including no runner message' do - expect(page).to have_text "How to setup" + expect(page).to have_text "Setup a shared Runner manually" expect(page).to have_text "Runners with last contact more than a minute ago: 0" expect(page).to have_text 'No runners found' end diff --git a/spec/features/issues/filtered_search/recent_searches_spec.rb b/spec/features/issues/filtered_search/recent_searches_spec.rb index f355cec3ba9..41b9ada988a 100644 --- a/spec/features/issues/filtered_search/recent_searches_spec.rb +++ b/spec/features/issues/filtered_search/recent_searches_spec.rb @@ -39,8 +39,8 @@ describe 'Recent searches', :js do items = all('.filtered-search-history-dropdown-item', visible: false, count: 2) - expect(items[0].text).to eq('label:~qux garply') - expect(items[1].text).to eq('label:~foo bar') + expect(items[0].text).to eq('label: ~qux garply') + expect(items[1].text).to eq('label: ~foo bar') end it 'saved recent searches are restored last on the list' do diff --git a/spec/features/projects/pages_spec.rb b/spec/features/projects/pages_spec.rb index 3f1ef0b2a47..a96f2c186a4 100644 --- a/spec/features/projects/pages_spec.rb +++ b/spec/features/projects/pages_spec.rb @@ -60,7 +60,6 @@ feature 'Pages' do fill_in 'Domain', with: 'my.test.domain.com' click_button 'Create New Domain' - expect(page).to have_content('Domains (1)') expect(page).to have_content('my.test.domain.com') end end @@ -159,7 +158,6 @@ feature 'Pages' do fill_in 'Key (PEM)', with: certificate_key click_button 'Create New Domain' - expect(page).to have_content('Domains (1)') expect(page).to have_content('my.test.domain.com') end end diff --git a/spec/features/runners_spec.rb b/spec/features/runners_spec.rb index aec9de6c7ca..df65c2d2f83 100644 --- a/spec/features/runners_spec.rb +++ b/spec/features/runners_spec.rb @@ -7,6 +7,20 @@ feature 'Runners' do sign_in(user) end + context 'when user opens runners page' do + given(:project) { create(:project) } + + background do + project.add_master(user) + end + + scenario 'user can see a button to install runners on kubernetes clusters' do + visit runners_path(project) + + expect(page).to have_link('Install Runner on Kubernetes', href: project_clusters_path(project)) + end + end + context 'when a project has enabled shared_runners' do given(:project) { create(:project) } diff --git a/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json b/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json index e8c17298b43..ed8ed9085c0 100644 --- a/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json +++ b/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json @@ -4,6 +4,9 @@ "domain": { "type": "string" }, "url": { "type": "uri" }, "project_id": { "type": "integer" }, + "verified": { "type": "boolean" }, + "verification_code": { "type": ["string", "null"] }, + "enabled_until": { "type": ["date", "null"] }, "certificate_expiration": { "type": "object", "properties": { @@ -14,6 +17,6 @@ "additionalProperties": false } }, - "required": ["domain", "url", "project_id"], + "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until"], "additionalProperties": false } diff --git a/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json b/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json index 08db8d47050..b57d544f896 100644 --- a/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json +++ b/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json @@ -3,6 +3,9 @@ "properties": { "domain": { "type": "string" }, "url": { "type": "uri" }, + "verified": { "type": "boolean" }, + "verification_code": { "type": ["string", "null"] }, + "enabled_until": { "type": ["date", "null"] }, "certificate": { "type": "object", "properties": { @@ -15,6 +18,6 @@ "additionalProperties": false } }, - "required": ["domain", "url"], + "required": ["domain", "url", "verified", "verification_code", "enabled_until"], "additionalProperties": false } diff --git a/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js index 4a516c517ef..59bd2650081 100644 --- a/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js +++ b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js @@ -1,7 +1,6 @@ import Vue from 'vue'; import eventHub from '~/filtered_search/event_hub'; -import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content'; - +import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content.vue'; import FilteredSearchTokenKeys from '~/filtered_search/filtered_search_token_keys'; const createComponent = (propsData) => { diff --git a/spec/javascripts/test_bundle.js b/spec/javascripts/test_bundle.js index 323b8a9572d..94fcc6c7f2b 100644 --- a/spec/javascripts/test_bundle.js +++ b/spec/javascripts/test_bundle.js @@ -113,7 +113,9 @@ if (process.env.BABEL_ENV === 'coverage') { // exempt these files from the coverage report const troubleMakers = [ './blob_edit/blob_bundle.js', - './boards/boards_bundle.js', + './boards/components/modal/empty_state.js', + './boards/components/modal/footer.js', + './boards/components/modal/header.js', './cycle_analytics/cycle_analytics_bundle.js', './cycle_analytics/components/stage_plan_component.js', './cycle_analytics/components/stage_staging_component.js', diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 19d3f55501e..6f07e423c1b 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -534,6 +534,19 @@ describe Gitlab::GitAccess do expect { pull_access_check }.to raise_unauthorized('Your account has been blocked.') end + context 'when the project repository does not exist' do + it 'returns not found' do + project.add_guest(user) + repo = project.repository + FileUtils.rm_rf(repo.path) + + # Sanity check for rm_rf + expect(repo.exists?).to eq(false) + + expect { pull_access_check }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.') + end + end + describe 'without access to project' do context 'pull code' do it { expect { pull_access_check }.to raise_not_found } diff --git a/spec/lib/gitlab/git_access_wiki_spec.rb b/spec/lib/gitlab/git_access_wiki_spec.rb index 215f1ecc9c5..730ede99fc9 100644 --- a/spec/lib/gitlab/git_access_wiki_spec.rb +++ b/spec/lib/gitlab/git_access_wiki_spec.rb @@ -57,7 +57,7 @@ describe Gitlab::GitAccessWiki do # Sanity check for rm_rf expect(wiki_repo.exists?).to eq(false) - expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'A repository for this project does not exist yet.') + expect { subject }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.') end end end diff --git a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb b/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb index 2b488101496..c95719eff1d 100644 --- a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb @@ -24,7 +24,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do context 'with one group where two metrics is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) allow(client).to receive(:label_values).and_return(metric_names) end @@ -70,7 +70,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do context 'with one group where only one metric is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) allow(client).to receive(:label_values).and_return('metric_a') end @@ -99,7 +99,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do let(:second_metric_group) { simple_metric_group(name: 'nameb', metrics: simple_metrics(added_metric_name: 'metric_c')) } before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group, second_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group, second_metric_group]) allow(client).to receive(:label_values).and_return('metric_c') end diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 5d86007f71f..4c3b8deefb9 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -19,41 +19,41 @@ describe Gitlab::PrometheusClient do # - execute_query: A query call shared_examples 'failure response' do context 'when request returns 400 with an error message' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 400, body: { error: 'bar!' }) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'bar!') + .to raise_error(Gitlab::PrometheusClient::Error, 'bar!') expect(req_stub).to have_been_requested end end context 'when request returns 400 without an error message' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 400) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'Bad data received') + .to raise_error(Gitlab::PrometheusClient::Error, 'Bad data received') expect(req_stub).to have_been_requested end end context 'when request returns 500' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 500, body: { message: 'FAIL!' }) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, '500 - {"message":"FAIL!"}') + .to raise_error(Gitlab::PrometheusClient::Error, '500 - {"message":"FAIL!"}') expect(req_stub).to have_been_requested end end context 'when request returns non json data' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 200, body: 'not json') expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'Parsing response failed') + .to raise_error(Gitlab::PrometheusClient::Error, 'Parsing response failed') expect(req_stub).to have_been_requested end end @@ -65,27 +65,27 @@ describe Gitlab::PrometheusClient do subject { described_class.new(RestClient::Resource.new(prometheus_url)) } context 'exceptions are raised' do - it 'raises a Gitlab::PrometheusError error when a SocketError is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a SocketError is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, SocketError) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "Can't connect to #{prometheus_url}") + .to raise_error(Gitlab::PrometheusClient::Error, "Can't connect to #{prometheus_url}") expect(req_stub).to have_been_requested end - it 'raises a Gitlab::PrometheusError error when a SSLError is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a SSLError is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, OpenSSL::SSL::SSLError) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "#{prometheus_url} contains invalid SSL data") + .to raise_error(Gitlab::PrometheusClient::Error, "#{prometheus_url} contains invalid SSL data") expect(req_stub).to have_been_requested end - it 'raises a Gitlab::PrometheusError error when a RestClient::Exception is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a RestClient::Exception is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, RestClient::Exception) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "Network connection error") + .to raise_error(Gitlab::PrometheusClient::Error, "Network connection error") expect(req_stub).to have_been_requested end end diff --git a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb deleted file mode 100644 index 8fdbbacd04d..00000000000 --- a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'spec_helper' - -describe Gitlab::SidekiqMiddleware::MemoryKiller do - subject { described_class.new } - let(:pid) { 999 } - - let(:worker) { double(:worker, class: 'TestWorker') } - let(:job) { { 'jid' => 123 } } - let(:queue) { 'test_queue' } - - def run - thread = subject.call(worker, job, queue) { nil } - thread&.join - end - - before do - allow(subject).to receive(:get_rss).and_return(10.kilobytes) - allow(subject).to receive(:pid).and_return(pid) - end - - context 'when MAX_RSS is set to 0' do - before do - stub_const("#{described_class}::MAX_RSS", 0) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end - - context 'when MAX_RSS is exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 5.kilobytes) - end - - it 'sends the STP, TERM and KILL signals at expected times' do - expect(subject).to receive(:sleep).with(15 * 60).ordered - expect(Process).to receive(:kill).with('SIGSTP', pid).ordered - - expect(subject).to receive(:sleep).with(30).ordered - expect(Process).to receive(:kill).with('SIGTERM', pid).ordered - - expect(subject).to receive(:sleep).with(10).ordered - expect(Process).to receive(:kill).with('SIGKILL', pid).ordered - - run - end - end - - context 'when MAX_RSS is not exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 15.kilobytes) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end -end diff --git a/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb b/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb new file mode 100644 index 00000000000..0001795c3f0 --- /dev/null +++ b/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb @@ -0,0 +1,88 @@ +require 'spec_helper' + +describe Gitlab::SidekiqMiddleware::Shutdown do + subject { described_class.new } + + let(:pid) { Process.pid } + let(:worker) { double(:worker, class: 'TestWorker') } + let(:job) { { 'jid' => 123 } } + let(:queue) { 'test_queue' } + let(:block) { proc { nil } } + + def run + subject.call(worker, job, queue) { block.call } + described_class.shutdown_thread&.join + end + + def pop_trace + subject.trace.pop(true) + end + + before do + allow(subject).to receive(:get_rss).and_return(10.kilobytes) + described_class.clear_shutdown_thread + end + + context 'when MAX_RSS is set to 0' do + before do + stub_const("#{described_class}::MAX_RSS", 0) + end + + it 'does nothing' do + expect(subject).not_to receive(:sleep) + + run + end + end + + def expect_shutdown_sequence + expect(pop_trace).to eq([:sleep, 15 * 60]) + expect(pop_trace).to eq([:kill, 'SIGTSTP', pid]) + + expect(pop_trace).to eq([:sleep, 30]) + expect(pop_trace).to eq([:kill, 'SIGTERM', pid]) + + expect(pop_trace).to eq([:sleep, 10]) + expect(pop_trace).to eq([:kill, 'SIGKILL', pid]) + end + + context 'when MAX_RSS is exceeded' do + before do + stub_const("#{described_class}::MAX_RSS", 5.kilobytes) + end + + it 'sends the TSTP, TERM and KILL signals at expected times' do + run + + expect_shutdown_sequence + end + end + + context 'when MAX_RSS is not exceeded' do + before do + stub_const("#{described_class}::MAX_RSS", 15.kilobytes) + end + + it 'does nothing' do + expect(subject).not_to receive(:sleep) + + run + end + end + + context 'when WantShutdown is raised' do + let(:block) { proc { raise described_class::WantShutdown } } + + it 'starts the shutdown sequence and re-raises the exception' do + expect { run }.to raise_exception(described_class::WantShutdown) + + # We can't expect 'run' to have joined on the shutdown thread, because + # it hit an exception. + shutdown_thread = described_class.shutdown_thread + expect(shutdown_thread).not_to be_nil + shutdown_thread.join + + expect_shutdown_sequence + end + end +end diff --git a/spec/mailers/emails/pages_domains_spec.rb b/spec/mailers/emails/pages_domains_spec.rb new file mode 100644 index 00000000000..fe428ea657d --- /dev/null +++ b/spec/mailers/emails/pages_domains_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' +require 'email_spec' + +describe Emails::PagesDomains do + include EmailSpec::Matchers + include_context 'gitlab email notification' + + set(:project) { create(:project) } + set(:domain) { create(:pages_domain, project: project) } + set(:user) { project.owner } + + shared_examples 'a pages domain email' do + it_behaves_like 'an email sent from GitLab' + it_behaves_like 'it should not have Gmail Actions links' + it_behaves_like 'a user cannot unsubscribe through footer link' + + it 'has the expected content' do + aggregate_failures do + is_expected.to have_subject(email_subject) + is_expected.to have_body_text(project.human_name) + is_expected.to have_body_text(domain.domain) + is_expected.to have_body_text domain.url + is_expected.to have_body_text project_pages_domain_url(project, domain) + is_expected.to have_body_text help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + end + end + end + + describe '#pages_domain_enabled_email' do + let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been enabled" } + + subject { Notify.pages_domain_enabled_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'has been enabled' } + end + + describe '#pages_domain_disabled_email' do + let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been disabled" } + + subject { Notify.pages_domain_disabled_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'has been disabled' } + end + + describe '#pages_domain_verification_succeeded_email' do + let(:email_subject) { "#{project.path} | Verification succeeded for GitLab Pages domain '#{domain.domain}'" } + + subject { Notify.pages_domain_verification_succeeded_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'successfully verified' } + end + + describe '#pages_domain_verification_failed_email' do + let(:email_subject) { "#{project.path} | ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'" } + + subject { Notify.pages_domain_verification_failed_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it 'says verification has failed and when the domain is enabled until' do + is_expected.to have_body_text 'Verification has failed' + is_expected.to have_body_text domain.enabled_until.strftime('%F %T') + end + end +end diff --git a/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb new file mode 100644 index 00000000000..afcaefa0591 --- /dev/null +++ b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180216121030_enqueue_verify_pages_domain_workers') + +describe EnqueueVerifyPagesDomainWorkers, :sidekiq, :migration do + around do |example| + Sidekiq::Testing.fake! do + example.run + end + end + + describe '#up' do + it 'enqueues a verification worker for every domain' do + domains = 1.upto(3).map { |i| PagesDomain.create!(domain: "my#{i}.domain.com") } + + expect { migrate! }.to change(PagesDomainVerificationWorker.jobs, :size).by(3) + + enqueued_ids = PagesDomainVerificationWorker.jobs.map { |job| job['args'] } + expected_ids = domains.map { |domain| [domain.id] } + + expect(enqueued_ids).to match_array(expected_ids) + end + end +end diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index 9d12f96c642..95713d8b85b 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -1,6 +1,10 @@ require 'spec_helper' describe PagesDomain do + using RSpec::Parameterized::TableSyntax + + subject(:pages_domain) { described_class.new } + describe 'associations' do it { is_expected.to belong_to(:project) } end @@ -64,19 +68,51 @@ describe PagesDomain do end end + describe 'validations' do + it { is_expected.to validate_presence_of(:verification_code) } + end + + describe '#verification_code' do + subject { pages_domain.verification_code } + + it 'is set automatically with 128 bits of SecureRandom data' do + expect(SecureRandom).to receive(:hex).with(16) { 'verification code' } + + is_expected.to eq('verification code') + end + end + + describe '#keyed_verification_code' do + subject { pages_domain.keyed_verification_code } + + it { is_expected.to eq("gitlab-pages-verification-code=#{pages_domain.verification_code}") } + end + + describe '#verification_domain' do + subject { pages_domain.verification_domain } + + it { is_expected.to be_nil } + + it 'is a well-known subdomain if the domain is present' do + pages_domain.domain = 'example.com' + + is_expected.to eq('_gitlab-pages-verification-code.example.com') + end + end + describe '#url' do subject { domain.url } context 'without the certificate' do let(:domain) { build(:pages_domain, certificate: '') } - it { is_expected.to eq('http://my.domain.com') } + it { is_expected.to eq("http://#{domain.domain}") } end context 'with a certificate' do let(:domain) { build(:pages_domain, :with_certificate) } - it { is_expected.to eq('https://my.domain.com') } + it { is_expected.to eq("https://#{domain.domain}") } end end @@ -154,4 +190,108 @@ describe PagesDomain do # We test only existence of output, since the output is long it { is_expected.not_to be_empty } end + + describe '#update_daemon' do + it 'runs when the domain is created' do + domain = build(:pages_domain) + + expect(domain).to receive(:update_daemon) + + domain.save! + end + + it 'runs when the domain is destroyed' do + domain = create(:pages_domain) + + expect(domain).to receive(:update_daemon) + + domain.destroy! + end + + it 'delegates to Projects::UpdatePagesConfigurationService' do + service = instance_double('Projects::UpdatePagesConfigurationService') + expect(Projects::UpdatePagesConfigurationService).to receive(:new) { service } + expect(service).to receive(:execute) + + create(:pages_domain) + end + + context 'configuration updates when attributes change' do + set(:project1) { create(:project) } + set(:project2) { create(:project) } + set(:domain) { create(:pages_domain) } + + where(:attribute, :old_value, :new_value, :update_expected) do + now = Time.now + future = now + 1.day + + :project | nil | :project1 | true + :project | :project1 | :project1 | false + :project | :project1 | :project2 | true + :project | :project1 | nil | true + + # domain can't be set to nil + :domain | 'a.com' | 'a.com' | false + :domain | 'a.com' | 'b.com' | true + + # verification_code can't be set to nil + :verification_code | 'foo' | 'foo' | false + :verification_code | 'foo' | 'bar' | false + + :verified_at | nil | now | false + :verified_at | now | now | false + :verified_at | now | future | false + :verified_at | now | nil | false + + :enabled_until | nil | now | true + :enabled_until | now | now | false + :enabled_until | now | future | false + :enabled_until | now | nil | true + end + + with_them do + it 'runs if a relevant attribute has changed' do + a = old_value.is_a?(Symbol) ? send(old_value) : old_value + b = new_value.is_a?(Symbol) ? send(new_value) : new_value + + domain.update!(attribute => a) + + if update_expected + expect(domain).to receive(:update_daemon) + else + expect(domain).not_to receive(:update_daemon) + end + + domain.update!(attribute => b) + end + end + + context 'TLS configuration' do + set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) } + + let(:cert1) { domain_with_tls.certificate } + let(:cert2) { cert1 + ' ' } + let(:key1) { domain_with_tls.key } + let(:key2) { key1 + ' ' } + + it 'updates when added' do + expect(domain).to receive(:update_daemon) + + domain.update!(key: key1, certificate: cert1) + end + + it 'updates when changed' do + expect(domain_with_tls).to receive(:update_daemon) + + domain_with_tls.update!(key: key2, certificate: cert2) + end + + it 'updates when removed' do + expect(domain_with_tls).to receive(:update_daemon) + + domain_with_tls.update!(key: nil, certificate: nil) + end + end + end + end end diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index ed17e019d42..6693e5783a5 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -223,8 +223,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do context 'with cluster for all environments without prometheus installed' do context 'without environment supplied' do - it 'raises PrometheusError because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/) + it 'raises PrometheusClient::Error because cluster was not found' do + expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) end end @@ -242,8 +242,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do context 'with prod environment supplied' do let!(:environment) { create(:environment, project: project, name: 'prod') } - it 'raises PrometheusError because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/) + it 'raises PrometheusClient::Error because cluster was not found' do + expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) end end end diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index c6fdda203ad..1b0a5eac9b0 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -597,7 +597,7 @@ describe 'Git HTTP requests' do context "when a gitlab ci token is provided" do let(:project) { create(:project, :repository) } let(:build) { create(:ci_build, :running) } - let(:other_project) { create(:project) } + let(:other_project) { create(:project, :repository) } before do build.update!(project: project) # can't associate it on factory create @@ -648,10 +648,10 @@ describe 'Git HTTP requests' do context 'when the repo does not exist' do let(:project) { create(:project) } - it 'rejects pulls with 403 Forbidden' do + it 'rejects pulls with 404 Not Found' do clone_get path, env - expect(response).to have_gitlab_http_status(:forbidden) + expect(response).to have_gitlab_http_status(:not_found) expect(response.body).to eq(git_access_error(:no_repo)) end end diff --git a/spec/services/ci/create_trace_artifact_service_spec.rb b/spec/services/ci/create_trace_artifact_service_spec.rb index 847a88920fe..8c5e8e438c7 100644 --- a/spec/services/ci/create_trace_artifact_service_spec.rb +++ b/spec/services/ci/create_trace_artifact_service_spec.rb @@ -4,40 +4,60 @@ describe Ci::CreateTraceArtifactService do describe '#execute' do subject { described_class.new(nil, nil).execute(job) } - let(:job) { create(:ci_build) } - context 'when the job does not have trace artifact' do context 'when the job has a trace file' do - before do - allow_any_instance_of(Gitlab::Ci::Trace) - .to receive(:default_path) { expand_fixture_path('trace/sample_trace') } + let!(:job) { create(:ci_build, :trace_live) } + let!(:legacy_path) { job.trace.read { |stream| return stream.path } } + let!(:legacy_checksum) { Digest::SHA256.file(legacy_path).hexdigest } + let(:new_path) { job.job_artifacts_trace.file.path } + let(:new_checksum) { Digest::SHA256.file(new_path).hexdigest } - allow_any_instance_of(JobArtifactUploader).to receive(:move_to_cache) { false } - allow_any_instance_of(JobArtifactUploader).to receive(:move_to_store) { false } - end + it { expect(File.exist?(legacy_path)).to be_truthy } it 'creates trace artifact' do expect { subject }.to change { Ci::JobArtifact.count }.by(1) - expect(job.job_artifacts_trace.read_attribute(:file)).to eq('sample_trace') + expect(File.exist?(legacy_path)).to be_falsy + expect(File.exist?(new_path)).to be_truthy + expect(new_checksum).to eq(legacy_checksum) + expect(job.job_artifacts_trace.file.exists?).to be_truthy + expect(job.job_artifacts_trace.file.filename).to eq('job.log') end - context 'when the job has already had trace artifact' do + context 'when failed to create trace artifact record' do before do - create(:ci_job_artifact, :trace, job: job) + # When ActiveRecord error happens + allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false) + allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages) + .and_return("Error") + + subject rescue nil + + job.reload end - it 'does not create trace artifact' do - expect { subject }.not_to change { Ci::JobArtifact.count } + it 'keeps legacy trace and removes trace artifact' do + expect(File.exist?(legacy_path)).to be_truthy + expect(job.job_artifacts_trace).to be_nil end end end context 'when the job does not have a trace file' do + let!(:job) { create(:ci_build) } + it 'does not create trace artifact' do expect { subject }.not_to change { Ci::JobArtifact.count } end end end + + context 'when the job has already had trace artifact' do + let!(:job) { create(:ci_build, :trace_artifact) } + + it 'does not create trace artifact' do + expect { subject }.not_to change { Ci::JobArtifact.count } + end + end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index 836ffb7cea0..62fdf870090 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -1678,6 +1678,78 @@ describe NotificationService, :mailer do end end + describe 'Pages domains' do + set(:project) { create(:project) } + set(:domain) { create(:pages_domain, project: project) } + set(:u_blocked) { create(:user, :blocked) } + set(:u_silence) { create_user_with_notification(:disabled, 'silent', project) } + set(:u_owner) { project.owner } + set(:u_master1) { create(:user) } + set(:u_master2) { create(:user) } + set(:u_developer) { create(:user) } + + before do + project.add_master(u_blocked) + project.add_master(u_silence) + project.add_master(u_master1) + project.add_master(u_master2) + project.add_developer(u_developer) + + reset_delivered_emails! + end + + %i[ + pages_domain_enabled + pages_domain_disabled + pages_domain_verification_succeeded + pages_domain_verification_failed + ].each do |sym| + describe "##{sym}" do + subject(:notify!) { notification.send(sym, domain) } + + it 'emails current watching masters' do + expect(Notify).to receive(:"#{sym}_email").at_least(:once).and_call_original + + notify! + + should_only_email(u_master1, u_master2, u_owner) + end + + it 'emails nobody if the project is missing' do + domain.project = nil + + notify! + + should_not_email_anyone + end + end + end + + describe '#pages_domain_verification_failed' do + it 'emails current watching masters' do + notification.pages_domain_verification_failed(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + + describe '#pages_domain_enabled' do + it 'emails current watching masters' do + notification.pages_domain_enabled(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + + describe '#pages_domain_disabled' do + it 'emails current watching masters' do + notification.pages_domain_disabled(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + end + def build_team(project) @u_watcher = create_global_setting_for(create(:user), :watch) @u_participating = create_global_setting_for(create(:user), :participating) diff --git a/spec/services/verify_pages_domain_service_spec.rb b/spec/services/verify_pages_domain_service_spec.rb new file mode 100644 index 00000000000..576db1dde2d --- /dev/null +++ b/spec/services/verify_pages_domain_service_spec.rb @@ -0,0 +1,270 @@ +require 'spec_helper' + +describe VerifyPagesDomainService do + using RSpec::Parameterized::TableSyntax + include EmailHelpers + + let(:error_status) { { status: :error, message: "Couldn't verify #{domain.domain}" } } + + subject(:service) { described_class.new(domain) } + + describe '#execute' do + context 'verification code recognition (verified domain)' do + where(:domain_sym, :code_sym) do + :domain | :verification_code + :domain | :keyed_verification_code + + :verification_domain | :verification_code + :verification_domain | :keyed_verification_code + end + + with_them do + set(:domain) { create(:pages_domain) } + + let(:domain_name) { domain.send(domain_sym) } + let(:verification_code) { domain.send(code_sym) } + + it 'verifies and enables the domain' do + stub_resolver(domain_name => ['something else', verification_code]) + + expect(service.execute).to eq(status: :success) + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'verifies and enables when the code is contained partway through a TXT record' do + stub_resolver(domain_name => "something #{verification_code} else") + + expect(service.execute).to eq(status: :success) + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'does not verify when the code is not present' do + stub_resolver(domain_name => 'something else') + + expect(service.execute).to eq(error_status) + + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + end + + context 'verified domain' do + set(:domain) { create(:pages_domain) } + + it 'unverifies (but does not disable) when the right code is not present' do + stub_resolver(domain.domain => 'something else') + + expect(service.execute).to eq(error_status) + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + + it 'unverifies (but does not disable) when no records are present' do + stub_resolver + + expect(service.execute).to eq(error_status) + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + end + + context 'expired domain' do + set(:domain) { create(:pages_domain, :expired) } + + it 'verifies and enables when the right code is present' do + stub_resolver(domain.domain => domain.keyed_verification_code) + + expect(service.execute).to eq(status: :success) + + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'disables when the right code is not present' do + error_status[:message] += '. It is now disabled.' + + stub_resolver + + expect(service.execute).to eq(error_status) + + expect(domain).not_to be_verified + expect(domain).not_to be_enabled + end + end + end + + context 'timeout behaviour' do + let(:domain) { create(:pages_domain) } + + it 'sets a timeout on the DNS query' do + expect(stub_resolver).to receive(:timeouts=).with(described_class::RESOLVER_TIMEOUT_SECONDS) + + service.execute + end + end + + context 'email notifications' do + let(:notification_service) { instance_double('NotificationService') } + + where(:factory, :verification_succeeds, :expected_notification) do + nil | true | nil + nil | false | :verification_failed + :reverify | true | nil + :reverify | false | :verification_failed + :unverified | true | :verification_succeeded + :unverified | false | nil + :expired | true | nil + :expired | false | :disabled + :disabled | true | :enabled + :disabled | false | nil + end + + with_them do + let(:domain) { create(:pages_domain, *[factory].compact) } + + before do + allow(service).to receive(:notification_service) { notification_service } + + if verification_succeeds + stub_resolver(domain.domain => domain.verification_code) + else + stub_resolver + end + end + + it 'sends a notification if appropriate' do + if expected_notification + expect(notification_service).to receive(:"pages_domain_#{expected_notification}").with(domain) + end + + service.execute + end + end + + context 'pages verification disabled' do + let(:domain) { create(:pages_domain, :disabled) } + + before do + stub_application_setting(pages_domain_verification_enabled: false) + allow(service).to receive(:notification_service) { notification_service } + end + + it 'skips email notifications' do + expect(notification_service).not_to receive(:pages_domain_enabled) + + service.execute + end + end + end + + context 'pages configuration updates' do + context 'enabling a disabled domain' do + let(:domain) { create(:pages_domain, :disabled) } + + it 'schedules an update' do + stub_resolver(domain.domain => domain.verification_code) + + expect(domain).to receive(:update_daemon) + + service.execute + end + end + + context 'verifying an enabled domain' do + let(:domain) { create(:pages_domain) } + + it 'schedules an update' do + stub_resolver(domain.domain => domain.verification_code) + + expect(domain).not_to receive(:update_daemon) + + service.execute + end + end + + context 'disabling an expired domain' do + let(:domain) { create(:pages_domain, :expired) } + + it 'schedules an update' do + stub_resolver + + expect(domain).to receive(:update_daemon) + + service.execute + end + end + + context 'failing to verify a disabled domain' do + let(:domain) { create(:pages_domain, :disabled) } + + it 'does not schedule an update' do + stub_resolver + + expect(domain).not_to receive(:update_daemon) + + service.execute + end + end + end + + context 'no verification code' do + let(:domain) { create(:pages_domain) } + + it 'returns an error' do + domain.verification_code = '' + + disallow_resolver! + + expect(service.execute).to eq(status: :error, message: "No verification code set for #{domain.domain}") + end + end + + context 'pages domain verification is disabled' do + let(:domain) { create(:pages_domain, :disabled) } + + before do + stub_application_setting(pages_domain_verification_enabled: false) + end + + it 'extends domain validity by unconditionally reverifying' do + disallow_resolver! + + service.execute + + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'does not shorten any grace period' do + grace = Time.now + 1.year + domain.update!(enabled_until: grace) + disallow_resolver! + + service.execute + + expect(domain.enabled_until).to be_like_time(grace) + end + end + end + + def disallow_resolver! + expect(Resolv::DNS).not_to receive(:open) + end + + def stub_resolver(stubbed_lookups = {}) + resolver = instance_double('Resolv::DNS') + allow(resolver).to receive(:timeouts=) + + expect(Resolv::DNS).to receive(:open).and_yield(resolver) + + allow(resolver).to receive(:getresources) { [] } + stubbed_lookups.each do |domain, records| + records = Array(records).map { |txt| Resolv::DNS::Resource::IN::TXT.new(txt) } + allow(resolver).to receive(:getresources).with(domain, Resolv::DNS::Resource::IN::TXT) { records } + end + + resolver + end +end diff --git a/spec/support/capybara.rb b/spec/support/capybara.rb index 5189c57b7db..8603b7f3e2c 100644 --- a/spec/support/capybara.rb +++ b/spec/support/capybara.rb @@ -78,8 +78,10 @@ RSpec.configure do |config| end config.after(:example, :js) do |example| - # prevent localstorage from introducing side effects based on test order - execute_script("localStorage.clear();") + # prevent localStorage from introducing side effects based on test order + unless ['', 'about:blank', 'data:,'].include? Capybara.current_session.driver.browser.current_url + execute_script("localStorage.clear();") + end # capybara/rspec already calls Capybara.reset_sessions! in an `after` hook, # but `block_and_wait_for_requests_complete` is called before it so by diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 1809ae1d141..5edc5de2a09 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -28,11 +28,11 @@ RSpec.configure do |config| end config.before(:each, :js) do - DatabaseCleaner.strategy = :deletion + DatabaseCleaner.strategy = :deletion, { cache_tables: false } end config.before(:each, :delete) do - DatabaseCleaner.strategy = :deletion + DatabaseCleaner.strategy = :deletion, { cache_tables: false } end config.before(:each, :migration) do diff --git a/spec/support/prometheus/additional_metrics_shared_examples.rb b/spec/support/prometheus/additional_metrics_shared_examples.rb index dbbd4ad4d40..c7c3346d39e 100644 --- a/spec/support/prometheus/additional_metrics_shared_examples.rb +++ b/spec/support/prometheus/additional_metrics_shared_examples.rb @@ -12,11 +12,12 @@ RSpec.shared_examples 'additional metrics query' do let(:client) { double('prometheus_client') } let(:query_result) { described_class.new(client).query(*query_params) } - let(:environment) { create(:environment, slug: 'environment-slug') } + let(:project) { create(:project) } + let(:environment) { create(:environment, slug: 'environment-slug', project: project) } before do allow(client).to receive(:label_values).and_return(metric_names) - allow(metric_group_class).to receive(:all).and_return([simple_metric_group(metrics: [simple_metric])]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group(metrics: [simple_metric])]) end context 'metrics query context' do @@ -24,13 +25,14 @@ RSpec.shared_examples 'additional metrics query' do shared_examples 'query context containing environment slug and filter' do it 'contains ci_environment_slug' do - expect(subject).to receive(:query_metrics).with(hash_including(ci_environment_slug: environment.slug)) + expect(subject).to receive(:query_metrics).with(project, hash_including(ci_environment_slug: environment.slug)) subject.query(*query_params) end it 'contains environment filter' do expect(subject).to receive(:query_metrics).with( + project, hash_including( environment_filter: "container_name!=\"POD\",environment=\"#{environment.slug}\"" ) @@ -48,7 +50,7 @@ RSpec.shared_examples 'additional metrics query' do it_behaves_like 'query context containing environment slug and filter' it 'query context contains kube_namespace' do - expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace)) + expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: kube_namespace)) subject.query(*query_params) end @@ -72,7 +74,7 @@ RSpec.shared_examples 'additional metrics query' do it_behaves_like 'query context containing environment slug and filter' it 'query context contains empty kube_namespace' do - expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: '')) + expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: '')) subject.query(*query_params) end @@ -81,7 +83,7 @@ RSpec.shared_examples 'additional metrics query' do context 'with one group where two metrics is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) end context 'some queries return results' do @@ -117,7 +119,7 @@ RSpec.shared_examples 'additional metrics query' do let(:metrics) { [simple_metric(queries: [simple_query])] } before do - allow(metric_group_class).to receive(:all).and_return( + allow(metric_group_class).to receive(:common_metrics).and_return( [ simple_metric_group(name: 'group_a', metrics: [simple_metric(queries: [simple_query])]), simple_metric_group(name: 'group_b', metrics: [simple_metric(title: 'title_b', queries: [simple_query('b')])]) diff --git a/spec/workers/pages_domain_verification_cron_worker_spec.rb b/spec/workers/pages_domain_verification_cron_worker_spec.rb new file mode 100644 index 00000000000..8f780428c82 --- /dev/null +++ b/spec/workers/pages_domain_verification_cron_worker_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe PagesDomainVerificationCronWorker do + subject(:worker) { described_class.new } + + describe '#perform' do + it 'enqueues a PagesDomainVerificationWorker for domains needing verification' do + verified = create(:pages_domain) + reverify = create(:pages_domain, :reverify) + disabled = create(:pages_domain, :disabled) + + [reverify, disabled].each do |domain| + expect(PagesDomainVerificationWorker).to receive(:perform_async).with(domain.id) + end + + expect(PagesDomainVerificationWorker).not_to receive(:perform_async).with(verified.id) + + worker.perform + end + end +end diff --git a/spec/workers/pages_domain_verification_worker_spec.rb b/spec/workers/pages_domain_verification_worker_spec.rb new file mode 100644 index 00000000000..372fc95ab4a --- /dev/null +++ b/spec/workers/pages_domain_verification_worker_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe PagesDomainVerificationWorker do + subject(:worker) { described_class.new } + + let(:domain) { create(:pages_domain) } + + describe '#perform' do + it 'does nothing for a non-existent domain' do + domain.destroy + + expect(VerifyPagesDomainService).not_to receive(:new) + + expect { worker.perform(domain.id) }.not_to raise_error + end + + it 'delegates to VerifyPagesDomainService' do + service = double(:service) + expected_domain = satisfy { |obj| obj == domain } + + expect(VerifyPagesDomainService).to receive(:new).with(expected_domain) { service } + expect(service).to receive(:execute) + + worker.perform(domain.id) + end + end +end diff --git a/spec/workers/stuck_import_jobs_worker_spec.rb b/spec/workers/stuck_import_jobs_worker_spec.rb index a82eb54ffe4..069514552b1 100644 --- a/spec/workers/stuck_import_jobs_worker_spec.rb +++ b/spec/workers/stuck_import_jobs_worker_spec.rb @@ -2,35 +2,59 @@ require 'spec_helper' describe StuckImportJobsWorker do let(:worker) { described_class.new } - let(:exclusive_lease_uuid) { SecureRandom.uuid } - before do - allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid) - end + shared_examples 'project import job detection' do + context 'when the job has completed' do + context 'when the import status was already updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids) do + project.import_start + project.import_finish - describe 'with started import_status' do - let(:project) { create(:project, :import_started, import_jid: '123') } + [project.import_jid] + end + end + + it 'does not mark the project as failed' do + worker.perform + + expect(project.reload.import_status).to eq('finished') + end + end + + context 'when the import status was not updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([project.import_jid]) + end - describe 'long running import' do - it 'marks the project as failed' do - allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return(['123']) + it 'marks the project as failed' do + worker.perform - expect { worker.perform }.to change { project.reload.import_status }.to('failed') + expect(project.reload.import_status).to eq('failed') + end end end - describe 'running import' do - it 'does not mark the project as failed' do + context 'when the job is still in Sidekiq' do + before do allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([]) + end + it 'does not mark the project as failed' do expect { worker.perform }.not_to change { project.reload.import_status } end + end + end - describe 'import without import_jid' do - it 'marks the project as failed' do - expect { worker.perform }.to change { project.reload.import_status }.to('failed') - end - end + describe 'with scheduled import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_scheduled, import_jid: '123') } + end + end + + describe 'with started import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_started, import_jid: '123') } end end end |
