diff options
176 files changed, 2656 insertions, 791 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 869884f8ca6..c8d399b2b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 10.5.2 (2018-02-25) + +### Fixed (7 changes) + +- Fix single digit value clipping for stacked progress bar. !17217 +- Fix issue with cache key being empty when variable used as the key. !17260 +- Enable Legacy Authorization by default on Cluster creations. !17302 +- Allow branch names to be named the same as the sha it points to. +- Fix 500 error when loading an invalid upload URL. +- Don't attempt to update user tracked fields if database is in read-only. +- Prevent MR Widget error when no CI configured. + +### Performance (5 changes) + +- Improve query performance for snippets dashboard. !17088 +- Only check LFS integrity for first ref in a push to avoid timeout. !17098 +- Improve query performance of MembersFinder. !17190 +- Increase feature flag cache TTL to one hour. +- Improve performance of searching for and autocompleting of users. + + ## 10.5.1 (2018-02-22) - No changes. diff --git a/app/assets/javascripts/boards/boards_bundle.js b/app/assets/javascripts/boards/index.js index 90166b3d3d1..06a8abea940 100644 --- a/app/assets/javascripts/boards/boards_bundle.js +++ b/app/assets/javascripts/boards/index.js @@ -24,7 +24,7 @@ import './components/new_list_dropdown'; import './components/modal/index'; import '../vue_shared/vue_resource_interceptor'; -$(() => { +export default () => { const $boardApp = document.getElementById('board-app'); const Store = gl.issueBoards.BoardsStore; const ModalStore = gl.issueBoards.ModalStore; @@ -236,4 +236,4 @@ $(() => { </div> `, }); -}); +}; diff --git a/app/assets/javascripts/deploy_keys/index.js b/app/assets/javascripts/deploy_keys/index.js index ca8798facc9..b727261648c 100644 --- a/app/assets/javascripts/deploy_keys/index.js +++ b/app/assets/javascripts/deploy_keys/index.js @@ -1,7 +1,7 @@ import Vue from 'vue'; import deployKeysApp from './components/app.vue'; -document.addEventListener('DOMContentLoaded', () => new Vue({ +export default () => new Vue({ el: document.getElementById('js-deploy-keys'), components: { deployKeysApp, @@ -18,4 +18,4 @@ document.addEventListener('DOMContentLoaded', () => new Vue({ }, }); }, -})); +}); diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index f66ce1c083b..acf0effa00d 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -42,31 +42,34 @@ var Dispatcher; }); }); - switch (page) { - case 'projects:merge_requests:index': - case 'projects:issues:index': - case 'projects:issues:show': - case 'projects:issues:new': - case 'projects:issues:edit': - case 'projects:merge_requests:creations:new': - case 'projects:merge_requests:creations:diffs': - case 'projects:merge_requests:edit': - case 'projects:merge_requests:show': - case 'projects:commit:show': - case 'projects:activity': - case 'projects:commits:show': - case 'projects:show': - case 'groups:show': - case 'projects:tree:show': - case 'projects:find_file:show': - case 'projects:blob:show': - case 'projects:blame:show': - case 'projects:network:show': - case 'projects:artifacts:browse': - case 'projects:artifacts:file': - shortcut_handler = true; - break; + const shortcutHandlerPages = [ + 'projects:activity', + 'projects:artifacts:browse', + 'projects:artifacts:file', + 'projects:blame:show', + 'projects:blob:show', + 'projects:commit:show', + 'projects:commits:show', + 'projects:find_file:show', + 'projects:issues:edit', + 'projects:issues:index', + 'projects:issues:new', + 'projects:issues:show', + 'projects:merge_requests:creations:diffs', + 'projects:merge_requests:creations:new', + 'projects:merge_requests:edit', + 'projects:merge_requests:index', + 'projects:merge_requests:show', + 'projects:network:show', + 'projects:show', + 'projects:tree:show', + 'groups:show', + ]; + + if (shortcutHandlerPages.indexOf(page) !== -1) { + shortcut_handler = true; } + switch (path[0]) { case 'admin': switch (path[1]) { diff --git a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js deleted file mode 100644 index b693084e434..00000000000 --- a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js +++ /dev/null @@ -1,102 +0,0 @@ -import eventHub from '../event_hub'; -import FilteredSearchTokenizer from '../filtered_search_tokenizer'; - -export default { - name: 'RecentSearchesDropdownContent', - - props: { - items: { - type: Array, - required: true, - }, - isLocalStorageAvailable: { - type: Boolean, - required: false, - default: true, - }, - allowedKeys: { - type: Array, - required: true, - }, - }, - - computed: { - processedItems() { - return this.items.map((item) => { - const { tokens, searchToken } - = FilteredSearchTokenizer.processTokens(item, this.allowedKeys); - - const resultantTokens = tokens.map(token => ({ - prefix: `${token.key}:`, - suffix: `${token.symbol}${token.value}`, - })); - - return { - text: item, - tokens: resultantTokens, - searchToken, - }; - }); - }, - hasItems() { - return this.items.length > 0; - }, - }, - - methods: { - onItemActivated(text) { - eventHub.$emit('recentSearchesItemSelected', text); - }, - onRequestClearRecentSearches(e) { - // Stop the dropdown from closing - e.stopPropagation(); - - eventHub.$emit('requestClearRecentSearches'); - }, - }, - - template: ` - <div> - <div - v-if="!isLocalStorageAvailable" - class="dropdown-info-note"> - This feature requires local storage to be enabled - </div> - <ul v-else-if="hasItems"> - <li - v-for="(item, index) in processedItems" - :key="index"> - <button - type="button" - class="filtered-search-history-dropdown-item" - @click="onItemActivated(item.text)"> - <span> - <span - v-for="(token, tokenIndex) in item.tokens" - class="filtered-search-history-dropdown-token"> - <span class="name">{{ token.prefix }}</span><span class="value">{{ token.suffix }}</span> - </span> - </span> - <span class="filtered-search-history-dropdown-search-token"> - {{ item.searchToken }} - </span> - </button> - </li> - <li class="divider"></li> - <li> - <button - type="button" - class="filtered-search-history-clear-button" - @click="onRequestClearRecentSearches($event)"> - Clear recent searches - </button> - </li> - </ul> - <div - v-else - class="dropdown-info-note"> - You don't have any recent searches - </div> - </div> - `, -}; diff --git a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue new file mode 100644 index 00000000000..26618af9515 --- /dev/null +++ b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.vue @@ -0,0 +1,104 @@ +<script> +import eventHub from '../event_hub'; +import FilteredSearchTokenizer from '../filtered_search_tokenizer'; + +export default { + name: 'RecentSearchesDropdownContent', + props: { + items: { + type: Array, + required: true, + }, + isLocalStorageAvailable: { + type: Boolean, + required: false, + default: true, + }, + allowedKeys: { + type: Array, + required: true, + }, + }, + computed: { + processedItems() { + return this.items.map((item) => { + const { tokens, searchToken } + = FilteredSearchTokenizer.processTokens(item, this.allowedKeys); + + const resultantTokens = tokens.map(token => ({ + prefix: `${token.key}:`, + suffix: `${token.symbol}${token.value}`, + })); + + return { + text: item, + tokens: resultantTokens, + searchToken, + }; + }); + }, + hasItems() { + return this.items.length > 0; + }, + }, + methods: { + onItemActivated(text) { + eventHub.$emit('recentSearchesItemSelected', text); + }, + onRequestClearRecentSearches(e) { + // Stop the dropdown from closing + e.stopPropagation(); + + eventHub.$emit('requestClearRecentSearches'); + }, + }, +}; +</script> +<template> + <div> + <div + v-if="!isLocalStorageAvailable" + class="dropdown-info-note"> + This feature requires local storage to be enabled + </div> + <ul v-else-if="hasItems"> + <li + v-for="(item, index) in processedItems" + :key="`processed-items-${index}`" + > + <button + type="button" + class="filtered-search-history-dropdown-item" + @click="onItemActivated(item.text)"> + <span> + <span + class="filtered-search-history-dropdown-token" + v-for="(token, index) in item.tokens" + :key="`dropdown-token-${index}`" + > + <span class="name">{{ token.prefix }}</span> + <span class="value">{{ token.suffix }}</span> + </span> + </span> + <span class="filtered-search-history-dropdown-search-token"> + {{ item.searchToken }} + </span> + </button> + </li> + <li class="divider"></li> + <li> + <button + type="button" + class="filtered-search-history-clear-button" + @click="onRequestClearRecentSearches($event)"> + Clear recent searches + </button> + </li> + </ul> + <div + v-else + class="dropdown-info-note"> + You don't have any recent searches + </div> + </div> +</template> diff --git a/app/assets/javascripts/filtered_search/dropdown_user.js b/app/assets/javascripts/filtered_search/dropdown_user.js index 22421fc4868..d36f38a70b5 100644 --- a/app/assets/javascripts/filtered_search/dropdown_user.js +++ b/app/assets/javascripts/filtered_search/dropdown_user.js @@ -14,7 +14,6 @@ export default class DropdownUser extends FilteredSearchDropdown { endpoint: `${gon.relative_url_root || ''}/autocomplete/users.json`, searchKey: 'search', params: { - per_page: 20, active: true, group_id: this.getGroupId(), project_id: this.getProjectId(), diff --git a/app/assets/javascripts/filtered_search/recent_searches_root.js b/app/assets/javascripts/filtered_search/recent_searches_root.js index c99ed63c4af..f9338b82acf 100644 --- a/app/assets/javascripts/filtered_search/recent_searches_root.js +++ b/app/assets/javascripts/filtered_search/recent_searches_root.js @@ -1,5 +1,5 @@ import Vue from 'vue'; -import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content'; +import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content.vue'; import eventHub from './event_hub'; class RecentSearchesRoot { @@ -33,7 +33,7 @@ class RecentSearchesRoot { this.vm = new Vue({ el: this.wrapperElement, components: { - 'recent-searches-dropdown-content': RecentSearchesDropdownContent, + RecentSearchesDropdownContent, }, data() { return state; }, template: ` diff --git a/app/assets/javascripts/labels_select.js b/app/assets/javascripts/labels_select.js index 5de48aa49a9..7151ac05a09 100644 --- a/app/assets/javascripts/labels_select.js +++ b/app/assets/javascripts/labels_select.js @@ -213,7 +213,7 @@ export default class LabelsSelect { } } if (label.duplicate) { - color = gl.DropdownUtils.duplicateLabelColor(label.color); + color = DropdownUtils.duplicateLabelColor(label.color); } else { if (label.color != null) { diff --git a/app/assets/javascripts/pages/projects/boards/index.js b/app/assets/javascripts/pages/projects/boards/index.js index 3aeeedbb45d..5cfe8723204 100644 --- a/app/assets/javascripts/pages/projects/boards/index.js +++ b/app/assets/javascripts/pages/projects/boards/index.js @@ -1,7 +1,9 @@ import UsersSelect from '~/users_select'; import ShortcutsNavigation from '~/shortcuts_navigation'; +import initBoards from '~/boards'; document.addEventListener('DOMContentLoaded', () => { new UsersSelect(); // eslint-disable-line no-new new ShortcutsNavigation(); // eslint-disable-line no-new + initBoards(); }); diff --git a/app/assets/javascripts/pages/projects/issues/show/index.js b/app/assets/javascripts/pages/projects/issues/show/index.js index db064e3f801..1e56aa58da2 100644 --- a/app/assets/javascripts/pages/projects/issues/show/index.js +++ b/app/assets/javascripts/pages/projects/issues/show/index.js @@ -3,6 +3,7 @@ import Issue from '~/issue'; import ShortcutsIssuable from '~/shortcuts_issuable'; import ZenMode from '~/zen_mode'; import '~/notes/index'; +import '~/issue_show/index'; document.addEventListener('DOMContentLoaded', () => { new Issue(); // eslint-disable-line no-new diff --git a/app/assets/javascripts/pages/projects/settings/repository/show/index.js b/app/assets/javascripts/pages/projects/settings/repository/show/index.js index d88527351c1..5a6f4138b10 100644 --- a/app/assets/javascripts/pages/projects/settings/repository/show/index.js +++ b/app/assets/javascripts/pages/projects/settings/repository/show/index.js @@ -1,3 +1,7 @@ import initSettingsPanels from '~/settings_panels'; +import initDeployKeys from '~/deploy_keys'; -document.addEventListener('DOMContentLoaded', initSettingsPanels); +document.addEventListener('DOMContentLoaded', () => { + initDeployKeys(); + initSettingsPanels(); +}); diff --git a/app/assets/javascripts/users_select.js b/app/assets/javascripts/users_select.js index 8958534689c..3385aba0279 100644 --- a/app/assets/javascripts/users_select.js +++ b/app/assets/javascripts/users_select.js @@ -39,7 +39,6 @@ function UsersSelect(currentUser, els, options = {}) { options.showCurrentUser = $dropdown.data('currentUser'); options.todoFilter = $dropdown.data('todoFilter'); options.todoStateFilter = $dropdown.data('todoStateFilter'); - options.perPage = $dropdown.data('perPage'); showNullUser = $dropdown.data('nullUser'); defaultNullUser = $dropdown.data('nullUserDefault'); showMenuAbove = $dropdown.data('showMenuAbove'); @@ -669,7 +668,6 @@ UsersSelect.prototype.users = function(query, options, callback) { const url = this.buildUrl(this.usersPath); const params = { search: query, - per_page: options.perPage || 20, active: true, project_id: options.projectId || null, group_id: options.groupId || null, diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js index 7ba6c29006a..162f048aac7 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js @@ -227,7 +227,8 @@ export default { @click="handleMergeButtonClick()" :disabled="isMergeButtonDisabled" :class="mergeButtonClass" - type="button"> + type="button" + class="qa-merge-button"> <i v-if="isMakingRequest" class="fa fa-spinner fa-spin" diff --git a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue index e9f23b0b113..143fd328d88 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue @@ -111,7 +111,7 @@ js-toggle-container accept-action media space-children" > <button type="button" - class="btn btn-sm btn-reopen btn-success" + class="btn btn-sm btn-reopen btn-success qa-mr-rebase-button" :disabled="isMakingRequest" @click="rebase" > diff --git a/app/controllers/projects/clusters/gcp_controller.rb b/app/controllers/projects/clusters/gcp_controller.rb index 0f41af7d87b..6b0b22f8e73 100644 --- a/app/controllers/projects/clusters/gcp_controller.rb +++ b/app/controllers/projects/clusters/gcp_controller.rb @@ -40,9 +40,9 @@ class Projects::Clusters::GcpController < Projects::ApplicationController def verify_billing case google_project_billing_status when nil - flash[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.') + flash.now[:alert] = _('We could not verify that one of your projects on GCP has billing enabled. Please try again.') when false - flash[:alert] = _('Please <a href=%{link_to_billing} target="_blank" rel="noopener noreferrer">enable billing for one of your projects to be able to create a Kubernetes cluster</a>, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } + flash.now[:alert] = _('Please <a href=%{link_to_billing} target="_blank" rel="noopener noreferrer">enable billing for one of your projects to be able to create a Kubernetes cluster</a>, then try again.').html_safe % { link_to_billing: "https://console.cloud.google.com/freetrial?utm_campaign=2018_cpanel&utm_source=gitlab&utm_medium=referral" } when true return end diff --git a/app/controllers/projects/pages_domains_controller.rb b/app/controllers/projects/pages_domains_controller.rb index 15e77d854dc..b71f1e5fef4 100644 --- a/app/controllers/projects/pages_domains_controller.rb +++ b/app/controllers/projects/pages_domains_controller.rb @@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController before_action :require_pages_enabled! before_action :authorize_update_pages!, except: [:show] - before_action :domain, only: [:show, :destroy] + before_action :domain, only: [:show, :destroy, :verify] def show end @@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController @domain = @project.pages_domains.new end + def verify + result = VerifyPagesDomainService.new(@domain).execute + + if result[:status] == :success + flash[:notice] = 'Successfully verified domain ownership' + else + flash[:alert] = 'Failed to verify domain ownership' + end + + redirect_to project_pages_domain_path(@project, @domain) + end + def create @domain = @project.pages_domains.create(pages_domain_params) if @domain.valid? - redirect_to project_pages_path(@project) + redirect_to project_pages_domain_path(@project, @domain) else render 'new' end @@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController end def domain - @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s) + @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s) end end diff --git a/app/controllers/projects/prometheus/metrics_controller.rb b/app/controllers/projects/prometheus/metrics_controller.rb new file mode 100644 index 00000000000..b739d0f0f90 --- /dev/null +++ b/app/controllers/projects/prometheus/metrics_controller.rb @@ -0,0 +1,27 @@ +module Projects + module Prometheus + class MetricsController < Projects::ApplicationController + before_action :authorize_admin_project! + + def active_common + respond_to do |format| + format.json do + matched_metrics = prometheus_service.matched_metrics || {} + + if matched_metrics.any? + render json: matched_metrics + else + head :no_content + end + end + end + end + + private + + def prometheus_service + @prometheus_service ||= project.find_or_initialize_service('prometheus') + end + end + end +end diff --git a/app/controllers/projects/prometheus_controller.rb b/app/controllers/projects/prometheus_controller.rb deleted file mode 100644 index 507468d7102..00000000000 --- a/app/controllers/projects/prometheus_controller.rb +++ /dev/null @@ -1,24 +0,0 @@ -class Projects::PrometheusController < Projects::ApplicationController - before_action :authorize_read_project! - before_action :require_prometheus_metrics! - - def active_metrics - respond_to do |format| - format.json do - matched_metrics = project.prometheus_service.matched_metrics || {} - - if matched_metrics.any? - render json: matched_metrics - else - head :no_content - end - end - end - end - - private - - def require_prometheus_metrics! - render_404 unless project.prometheus_service.present? - end -end diff --git a/app/finders/autocomplete_users_finder.rb b/app/finders/autocomplete_users_finder.rb index c3f5358b577..e8a03947f59 100644 --- a/app/finders/autocomplete_users_finder.rb +++ b/app/finders/autocomplete_users_finder.rb @@ -1,6 +1,12 @@ class AutocompleteUsersFinder + # The number of users to display in the results is hardcoded to 20, and + # pagination is not supported. This ensures that performance remains + # consistent and removes the need for implementing keyset pagination to ensure + # good performance. + LIMIT = 20 + attr_reader :current_user, :project, :group, :search, :skip_users, - :page, :per_page, :author_id, :params + :author_id, :params def initialize(params:, current_user:, project:, group:) @current_user = current_user @@ -8,8 +14,6 @@ class AutocompleteUsersFinder @group = group @search = params[:search] @skip_users = params[:skip_users] - @page = params[:page] - @per_page = params[:per_page] @author_id = params[:author_id] @params = params end @@ -20,7 +24,7 @@ class AutocompleteUsersFinder items = items.reorder(:name) items = items.search(search) if search.present? items = items.where.not(id: skip_users) if skip_users.present? - items = items.page(page).per(per_page) + items = items.limit(LIMIT) if params[:todo_filter].present? && current_user items = items.todo_authors(current_user.id, params[:todo_state_filter]) @@ -52,9 +56,13 @@ class AutocompleteUsersFinder end def users_from_project - user_ids = project.team.users.pluck(:id) - user_ids << author_id if author_id.present? + if author_id.present? + union = Gitlab::SQL::Union + .new([project.authorized_users, User.where(id: author_id)]) - User.where(id: user_ids) + User.from("(#{union.to_sql}) #{User.table_name}") + else + project.authorized_users + end end end diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index e293b3ef329..ab68ecad2ba 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -199,6 +199,7 @@ module ApplicationSettingsHelper :metrics_port, :metrics_sample_interval, :metrics_timeout, + :pages_domain_verification_enabled, :password_authentication_enabled_for_web, :password_authentication_enabled_for_git, :performance_bar_allowed_group_id, diff --git a/app/helpers/issuables_helper.rb b/app/helpers/issuables_helper.rb index 7cd84fe69c9..44ecc2212f2 100644 --- a/app/helpers/issuables_helper.rb +++ b/app/helpers/issuables_helper.rb @@ -234,7 +234,7 @@ module IssuablesHelper data.merge!(updated_at_by(issuable)) - data.to_json + data end def updated_at_by(issuable) diff --git a/app/mailers/emails/pages_domains.rb b/app/mailers/emails/pages_domains.rb new file mode 100644 index 00000000000..0027dfdc36b --- /dev/null +++ b/app/mailers/emails/pages_domains.rb @@ -0,0 +1,43 @@ +module Emails + module PagesDomains + def pages_domain_enabled_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled") + ) + end + + def pages_domain_disabled_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled") + ) + end + + def pages_domain_verification_succeeded_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'") + ) + end + + def pages_domain_verification_failed_email(domain, recipient) + @domain = domain + @project = domain.project + + mail( + to: recipient.notification_email, + subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'") + ) + end + end +end diff --git a/app/mailers/notify.rb b/app/mailers/notify.rb index eade0fe278f..45d4fb451d8 100644 --- a/app/mailers/notify.rb +++ b/app/mailers/notify.rb @@ -5,6 +5,7 @@ class Notify < BaseMailer include Emails::Issues include Emails::MergeRequests include Emails::Notes + include Emails::PagesDomains include Emails::Projects include Emails::Profile include Emails::Pipelines diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index d8bf54e0c40..588bd50ed77 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -1,10 +1,14 @@ class PagesDomain < ActiveRecord::Base + VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze + VERIFICATION_THRESHOLD = 3.days.freeze + belongs_to :project validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } validates :certificate, certificate: true, allow_nil: true, allow_blank: true validates :key, certificate_key: true, allow_nil: true, allow_blank: true + validates :verification_code, presence: true, allow_blank: false validate :validate_pages_domain validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? } @@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base key: Gitlab::Application.secrets.db_key_base, algorithm: 'aes-256-cbc' + after_initialize :set_verification_code after_create :update_daemon - after_save :update_daemon + after_update :update_daemon, if: :pages_config_changed? after_destroy :update_daemon + scope :enabled, -> { where('enabled_until >= ?', Time.now ) } + scope :needs_verification, -> do + verified_at = arel_table[:verified_at] + enabled_until = arel_table[:enabled_until] + threshold = Time.now + VERIFICATION_THRESHOLD + + where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold)))) + end + + def verified? + !!verified_at + end + + def unverified? + !verified? + end + + def enabled? + !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present? + end + def to_param domain end @@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base @certificate_text ||= x509.try(:to_text) end + # Verification codes may be TXT records for domain or verification_domain, to + # support the use of CNAME records on domain. + def verification_domain + return unless domain.present? + + "_#{VERIFICATION_KEY}.#{domain}" + end + + def keyed_verification_code + return unless verification_code.present? + + "#{VERIFICATION_KEY}=#{verification_code}" + end + private + def set_verification_code + return if self.verification_code.present? + + self.verification_code = SecureRandom.hex(16) + end + def update_daemon ::Projects::UpdatePagesConfigurationService.new(project).execute end + def pages_config_changed? + project_id_changed? || + domain_changed? || + certificate_changed? || + key_changed? || + became_enabled? || + became_disabled? + end + + def became_enabled? + enabled_until.present? && !enabled_until_was.present? + end + + def became_disabled? + !enabled_until.present? && enabled_until_was.present? + end + def validate_matching_key unless has_matching_key? self.errors.add(:key, "doesn't match the certificate") diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 1bb576ff971..58731451429 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -69,16 +69,16 @@ class PrometheusService < MonitoringService client.ping { success: true, result: 'Checked API endpoint' } - rescue Gitlab::PrometheusError => err + rescue Gitlab::PrometheusClient::Error => err { success: false, result: err } end def environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics)) + with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics)) end def deployment_metrics(deployment) - metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics)) + metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics)) metrics&.merge(deployment_time: deployment.created_at.to_i) || {} end @@ -107,7 +107,7 @@ class PrometheusService < MonitoringService data: data, last_update: Time.now.utc } - rescue Gitlab::PrometheusError => err + rescue Gitlab::PrometheusClient::Error => err { success: false, result: err.message } end @@ -116,10 +116,10 @@ class PrometheusService < MonitoringService Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url)) else cluster = cluster_with_prometheus(environment_id) - raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster + raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster rest_client = client_from_cluster(cluster) - raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client + raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client Gitlab::PrometheusClient.new(rest_client) end @@ -152,9 +152,11 @@ class PrometheusService < MonitoringService cluster.application_prometheus.proxy_client end - def rename_data_to_metrics(metrics) - metrics[:metrics] = metrics.delete :data - metrics + def rename_field(old_field, new_field) + -> (metrics) do + metrics[new_field] = metrics.delete(old_field) + metrics + end end def synchronize_service_state! diff --git a/app/models/user.rb b/app/models/user.rb index f5eeba27572..8610ca27b7f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -327,8 +327,8 @@ class User < ActiveRecord::Base SQL where( - fuzzy_arel_match(:name, query) - .or(fuzzy_arel_match(:username, query)) + fuzzy_arel_match(:name, query, lower_exact_match: true) + .or(fuzzy_arel_match(:username, query, lower_exact_match: true)) .or(arel_table[:email].eq(query)) ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb index 280a2c3afa4..ffde824972c 100644 --- a/app/services/ci/create_trace_artifact_service.rb +++ b/app/services/ci/create_trace_artifact_service.rb @@ -4,13 +4,33 @@ module Ci return if job.job_artifacts_trace job.trace.read do |stream| - if stream.file? - job.create_job_artifacts_trace!( - project: job.project, - file_type: :trace, - file: stream) + break unless stream.file? + + clone_file!(stream.path, JobArtifactUploader.workhorse_upload_path) do |clone_path| + create_job_trace!(job, clone_path) + FileUtils.rm(stream.path) end end end + + private + + def create_job_trace!(job, path) + File.open(path) do |stream| + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream) + end + end + + def clone_file!(src_path, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.copy(src_path, temp_path) + yield(temp_path) + end + end end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 56e941d90ff..e07ecda27b5 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -339,6 +339,30 @@ class NotificationService end end + def pages_domain_verification_succeeded(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later + end + end + + def pages_domain_verification_failed(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_verification_failed_email(domain, user).deliver_later + end + end + + def pages_domain_enabled(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_enabled_email(domain, user).deliver_later + end + end + + def pages_domain_disabled(domain) + recipients_for_pages_domain(domain).each do |user| + mailer.pages_domain_disabled_email(domain, user).deliver_later + end + end + protected def new_resource_email(target, method) @@ -433,6 +457,14 @@ class NotificationService private + def recipients_for_pages_domain(domain) + project = domain.project + + return [] unless project + + notifiable_users(project.team.masters, :watch, target: project) + end + def notifiable?(*args) NotificationRecipientService.notifiable?(*args) end diff --git a/app/services/projects/autocomplete_service.rb b/app/services/projects/autocomplete_service.rb index 1ae2c40872a..e61ecb696d0 100644 --- a/app/services/projects/autocomplete_service.rb +++ b/app/services/projects/autocomplete_service.rb @@ -50,16 +50,7 @@ module Projects return [] unless noteable&.is_a?(Issuable) - opts = { - project: project, - issuable: noteable, - current_user: current_user - } - QuickActions::InterpretService.command_definitions.map do |definition| - next unless definition.available?(opts) - - definition.to_h(opts) - end.compact + QuickActions::InterpretService.new(project, current_user).available_commands(noteable) end end end diff --git a/app/services/projects/update_pages_configuration_service.rb b/app/services/projects/update_pages_configuration_service.rb index cacb74b1205..52ff64cc938 100644 --- a/app/services/projects/update_pages_configuration_service.rb +++ b/app/services/projects/update_pages_configuration_service.rb @@ -23,7 +23,7 @@ module Projects end def pages_domains_config - project.pages_domains.map do |domain| + enabled_pages_domains.map do |domain| { domain: domain.domain, certificate: domain.certificate, @@ -32,6 +32,14 @@ module Projects end end + def enabled_pages_domains + if Gitlab::CurrentSettings.pages_domain_verification_enabled? + project.pages_domains.enabled + else + project.pages_domains + end + end + def reload_daemon # GitLab Pages daemon constantly watches for modification time of `pages.path` # It reloads configuration when `pages.path` is modified diff --git a/app/services/quick_actions/interpret_service.rb b/app/services/quick_actions/interpret_service.rb index 669c1ba0a22..1e9bd84e749 100644 --- a/app/services/quick_actions/interpret_service.rb +++ b/app/services/quick_actions/interpret_service.rb @@ -7,6 +7,18 @@ module QuickActions SHRUG = '¯\\＿(ツ)＿/¯'.freeze TABLEFLIP = '(╯°□°)╯︵ ┻━┻'.freeze + # Takes an issuable and returns an array of all the available commands + # represented with .to_h + def available_commands(issuable) + @issuable = issuable + + self.class.command_definitions.map do |definition| + next unless definition.available?(self) + + definition.to_h(self) + end.compact + end + # Takes a text and interprets the commands that are extracted from it. # Returns the content without commands, and hash of changes to be applied to a record. def execute(content, issuable) @@ -15,8 +27,8 @@ module QuickActions @issuable = issuable @updates = {} - content, commands = extractor.extract_commands(content, context) - extract_updates(commands, context) + content, commands = extractor.extract_commands(content) + extract_updates(commands) [content, @updates] end @@ -28,8 +40,8 @@ module QuickActions @issuable = issuable - content, commands = extractor.extract_commands(content, context) - commands = explain_commands(commands, context) + content, commands = extractor.extract_commands(content) + commands = explain_commands(commands) [content, commands] end @@ -157,11 +169,11 @@ module QuickActions params '%"milestone"' condition do current_user.can?(:"admin_#{issuable.to_ability_name}", project) && - project.milestones.active.any? + find_milestones(project, state: 'active').any? end parse_params do |milestone_param| extract_references(milestone_param, :milestone).first || - project.milestones.find_by(title: milestone_param.strip) + find_milestones(project, title: milestone_param.strip).first end command :milestone do |milestone| @updates[:milestone_id] = milestone.id if milestone @@ -544,6 +556,10 @@ module QuickActions users end + def find_milestones(project, params = {}) + MilestonesFinder.new(params.merge(project_ids: [project.id], group_ids: [project.group&.id])).execute + end + def find_labels(labels_param) extract_references(labels_param, :label) | LabelsFinder.new(current_user, project_id: project.id, name: labels_param.split).execute @@ -557,21 +573,21 @@ module QuickActions find_labels(labels_param).map(&:id) end - def explain_commands(commands, opts) + def explain_commands(commands) commands.map do |name, arg| definition = self.class.definition_by_name(name) next unless definition - definition.explain(self, opts, arg) + definition.explain(self, arg) end.compact end - def extract_updates(commands, opts) + def extract_updates(commands) commands.each do |name, arg| definition = self.class.definition_by_name(name) next unless definition - definition.execute(self, opts, arg) + definition.execute(self, arg) end end @@ -581,14 +597,5 @@ module QuickActions ext.references(type) end - - def context - { - issuable: issuable, - current_user: current_user, - project: project, - params: params - } - end end end diff --git a/app/services/verify_pages_domain_service.rb b/app/services/verify_pages_domain_service.rb new file mode 100644 index 00000000000..86166047302 --- /dev/null +++ b/app/services/verify_pages_domain_service.rb @@ -0,0 +1,107 @@ +require 'resolv' + +class VerifyPagesDomainService < BaseService + # The maximum number of seconds to be spent on each DNS lookup + RESOLVER_TIMEOUT_SECONDS = 15 + + # How long verification lasts for + VERIFICATION_PERIOD = 7.days + + attr_reader :domain + + def initialize(domain) + @domain = domain + end + + def execute + return error("No verification code set for #{domain.domain}") unless domain.verification_code.present? + + if !verification_enabled? || dns_record_present? + verify_domain! + elsif expired? + disable_domain! + else + unverify_domain! + end + end + + private + + def verify_domain! + was_disabled = !domain.enabled? + was_unverified = domain.unverified? + + # Prevent any pre-existing grace period from being truncated + reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max + + domain.update!(verified_at: Time.now, enabled_until: reverify) + + if was_disabled + notify(:enabled) + elsif was_unverified + notify(:verification_succeeded) + end + + success + end + + def unverify_domain! + if domain.verified? + domain.update!(verified_at: nil) + notify(:verification_failed) + end + + error("Couldn't verify #{domain.domain}") + end + + def disable_domain! + domain.update!(verified_at: nil, enabled_until: nil) + + notify(:disabled) + + error("Couldn't verify #{domain.domain}. It is now disabled.") + end + + # A domain is only expired until `disable!` has been called + def expired? + domain.enabled_until && domain.enabled_until < Time.now + end + + def dns_record_present? + Resolv::DNS.open do |resolver| + resolver.timeouts = RESOLVER_TIMEOUT_SECONDS + + check(domain.domain, resolver) || check(domain.verification_domain, resolver) + end + end + + def check(domain_name, resolver) + records = parse(txt_records(domain_name, resolver)) + + records.any? do |record| + record == domain.keyed_verification_code || record == domain.verification_code + end + rescue => err + log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}") + false + end + + def txt_records(domain_name, resolver) + resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT) + end + + def parse(records) + records.flat_map(&:strings).flat_map(&:split) + end + + def verification_enabled? + Gitlab::CurrentSettings.pages_domain_verification_enabled? + end + + def notify(type) + return unless verification_enabled? + + Gitlab::AppLogger.info("Pages domain '#{domain.domain}' changed state to '#{type}'") + notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 60f12030f98..20527d31870 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -237,6 +237,17 @@ .col-sm-10 = f.number_field :max_pages_size, class: 'form-control' .help-block 0 for unlimited + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :pages_domain_verification_enabled do + = f.check_box :pages_domain_verification_enabled + Require users to prove ownership of custom domains + .help-block + Domain verification is an essential security measure for public GitLab + sites. Users are required to demonstrate they control a domain before + it is enabled + = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') %fieldset %legend Continuous Integration and Deployment diff --git a/app/views/admin/runners/index.html.haml b/app/views/admin/runners/index.html.haml index 1e52646b1cc..abec3607cab 100644 --- a/app/views/admin/runners/index.html.haml +++ b/app/views/admin/runners/index.html.haml @@ -35,9 +35,8 @@ method: :put, class: 'btn btn-default', data: { confirm: _("Are you sure you want to reset registration token?") } - = render partial: 'ci/runner/how_to_setup_runner', - locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token, - type: 'shared' } + = render partial: 'ci/runner/how_to_setup_shared_runner', + locals: { registration_token: Gitlab::CurrentSettings.runners_registration_token } .append-bottom-20.clearfix .pull-left diff --git a/app/views/ci/runner/_how_to_setup_runner.html.haml b/app/views/ci/runner/_how_to_setup_runner.html.haml index 8db7727b80c..37fb8fbab26 100644 --- a/app/views/ci/runner/_how_to_setup_runner.html.haml +++ b/app/views/ci/runner/_how_to_setup_runner.html.haml @@ -1,16 +1,16 @@ - link = link_to _("GitLab Runner section"), 'https://about.gitlab.com/gitlab-ci/#gitlab-runner', target: '_blank' -.bs-callout.help-callout - %h4= _("How to setup a #{type} Runner for a new project") +.append-bottom-10 + %h4= _("Setup a #{type} Runner manually") - %ol - %li - = _("Install a Runner compatible with GitLab CI") - = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe - %li - = _("Specify the following URL during the Runner setup:") - %code#coordinator_address= root_url(only_path: false) - %li - = _("Use the following registration token during setup:") - %code#registration_token= registration_token - %li - = _("Start the Runner!") +%ol + %li + = _("Install a Runner compatible with GitLab CI") + = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe + %li + = _("Specify the following URL during the Runner setup:") + %code#coordinator_address= root_url(only_path: false) + %li + = _("Use the following registration token during setup:") + %code#registration_token= registration_token + %li + = _("Start the Runner!") diff --git a/app/views/ci/runner/_how_to_setup_shared_runner.html.haml b/app/views/ci/runner/_how_to_setup_shared_runner.html.haml new file mode 100644 index 00000000000..2a190cb9250 --- /dev/null +++ b/app/views/ci/runner/_how_to_setup_shared_runner.html.haml @@ -0,0 +1,3 @@ +.bs-callout.help-callout + = render partial: 'ci/runner/how_to_setup_runner', + locals: { registration_token: registration_token, type: 'shared' } diff --git a/app/views/ci/runner/_how_to_setup_specific_runner.html.haml b/app/views/ci/runner/_how_to_setup_specific_runner.html.haml new file mode 100644 index 00000000000..e765a353fe4 --- /dev/null +++ b/app/views/ci/runner/_how_to_setup_specific_runner.html.haml @@ -0,0 +1,26 @@ +.bs-callout.help-callout + .append-bottom-10 + %h4= _('Setup a specific Runner automatically') + + %p + - link_to_help_page = link_to(_('Learn more about Kubernetes'), + help_page_path('user/project/clusters/index'), + target: '_blank', + rel: 'noopener noreferrer') + + = _('You can easily install a Runner on a Kubernetes cluster. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page } + + %ol + %li + = _('Click the button below to begin the install process by navigating to the Kubernetes page') + %li + = _('Select an existing Kubernetes cluster or create a new one') + %li + = _('From the Kubernetes cluster details view, install Runner from the applications list') + + = link_to _('Install Runner on Kubernetes'), + project_clusters_path(@project), + class: 'btn btn-info' + %hr + = render partial: 'ci/runner/how_to_setup_runner', + locals: { registration_token: registration_token, type: 'specific' } diff --git a/app/views/notify/pages_domain_disabled_email.html.haml b/app/views/notify/pages_domain_disabled_email.html.haml new file mode 100644 index 00000000000..34ce4238a12 --- /dev/null +++ b/app/views/notify/pages_domain_disabled_email.html.haml @@ -0,0 +1,15 @@ +%p + Following a verification check, your GitLab Pages custom domain has been + %strong disabled. + This means that your content is no longer visible at #{link_to @domain.url, @domain.url} +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + If this domain has been disabled in error, please follow + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + to verify and re-enable your domain. +%p + If you no longer wish to use this domain with GitLab Pages, please remove it + from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_disabled_email.text.haml b/app/views/notify/pages_domain_disabled_email.text.haml new file mode 100644 index 00000000000..4e81b054b1f --- /dev/null +++ b/app/views/notify/pages_domain_disabled_email.text.haml @@ -0,0 +1,13 @@ +Following a verification check, your GitLab Pages custom domain has been +**disabled**. This means that your content is no longer visible at #{@domain.url} + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +If this domain has been disabled in error, please follow these instructions +to verify and re-enable your domain: + += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + +If you no longer wish to use this domain with GitLab Pages, please remove it +from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_enabled_email.html.haml b/app/views/notify/pages_domain_enabled_email.html.haml new file mode 100644 index 00000000000..db09e503f65 --- /dev/null +++ b/app/views/notify/pages_domain_enabled_email.html.haml @@ -0,0 +1,11 @@ +%p + Following a verification check, your GitLab Pages custom domain has been + enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url} +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_enabled_email.text.haml b/app/views/notify/pages_domain_enabled_email.text.haml new file mode 100644 index 00000000000..1ed1dbb8315 --- /dev/null +++ b/app/views/notify/pages_domain_enabled_email.text.haml @@ -0,0 +1,9 @@ +Following a verification check, your GitLab Pages custom domain has been +enabled. You should now be able to view your content at #{@domain.url} + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_verification_failed_email.html.haml b/app/views/notify/pages_domain_verification_failed_email.html.haml new file mode 100644 index 00000000000..0bb0eb09fd5 --- /dev/null +++ b/app/views/notify/pages_domain_verification_failed_email.html.haml @@ -0,0 +1,17 @@ +%p + Verification has failed for one of your GitLab Pages custom domains! +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + Unless you take action, it will be disabled on + %strong= @domain.enabled_until.strftime('%F %T.') + Until then, you can view your content at #{link_to @domain.url, @domain.url} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. +%p + If you no longer wish to use this domain with GitLab Pages, please remove it + from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_verification_failed_email.text.haml b/app/views/notify/pages_domain_verification_failed_email.text.haml new file mode 100644 index 00000000000..c14e0e0c24d --- /dev/null +++ b/app/views/notify/pages_domain_verification_failed_email.text.haml @@ -0,0 +1,14 @@ +Verification has failed for one of your GitLab Pages custom domains! + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*. +Until then, you can view your content at #{@domain.url} + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. + +If you no longer wish to use this domain with GitLab Pages, please remove it +from your GitLab project and delete any related DNS records. diff --git a/app/views/notify/pages_domain_verification_succeeded_email.html.haml b/app/views/notify/pages_domain_verification_succeeded_email.html.haml new file mode 100644 index 00000000000..2ead3187b10 --- /dev/null +++ b/app/views/notify/pages_domain_verification_succeeded_email.html.haml @@ -0,0 +1,13 @@ +%p + One of your GitLab Pages custom domains has been successfully verified! +%p + Project: #{link_to @project.human_name, project_url(@project)} +%p + Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)} +%p + This is a notification. No action is required on your part. You can view your + content at #{link_to @domain.url, @domain.url} +%p + Please visit + = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + for more information about custom domain verification. diff --git a/app/views/notify/pages_domain_verification_succeeded_email.text.haml b/app/views/notify/pages_domain_verification_succeeded_email.text.haml new file mode 100644 index 00000000000..e7cdbdee420 --- /dev/null +++ b/app/views/notify/pages_domain_verification_succeeded_email.text.haml @@ -0,0 +1,10 @@ +One of your GitLab Pages custom domains has been successfully verified! + +Project: #{@project.human_name} (#{project_url(@project)}) +Domain: #{@domain.domain} (#{project_pages_domain_url(@project, @domain)}) + +No action is required on your part. You can view your content at #{@domain.url} + +Please visit += help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') +for more information about custom domain verification. diff --git a/app/views/projects/_merge_request_fast_forward_settings.html.haml b/app/views/projects/_merge_request_fast_forward_settings.html.haml index 8129c72feb2..f455522d17c 100644 --- a/app/views/projects/_merge_request_fast_forward_settings.html.haml +++ b/app/views/projects/_merge_request_fast_forward_settings.html.haml @@ -3,7 +3,7 @@ .radio = label_tag :project_merge_method_ff do - = form.radio_button :merge_method, :ff, class: "js-merge-method-radio" + = form.radio_button :merge_method, :ff, class: "js-merge-method-radio qa-radio-button-merge-ff" %strong Fast-forward merge %br %span.descr diff --git a/app/views/projects/clusters/_empty_state.html.haml b/app/views/projects/clusters/_empty_state.html.haml index 600d679b60c..112dde66ff7 100644 --- a/app/views/projects/clusters/_empty_state.html.haml +++ b/app/views/projects/clusters/_empty_state.html.haml @@ -4,7 +4,7 @@ .col-xs-12 .text-content %h4.text-center= s_('ClusterIntegration|Integrate Kubernetes cluster automation') - - link_to_help_page = link_to(s_('ClusterIntegration|Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') + - link_to_help_page = link_to(_('Learn more about Kubernetes'), help_page_path('user/project/clusters/index'), target: '_blank', rel: 'noopener noreferrer') %p= s_('ClusterIntegration|Kubernetes clusters allow you to use review apps, deploy your applications, run your pipelines, and much more in an easy way. %{link_to_help_page}').html_safe % { link_to_help_page: link_to_help_page} .text-center diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index 0931ceb1512..b947b91322d 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -85,7 +85,7 @@ .settings-content = form_for [@project.namespace.becomes(Namespace), @project], remote: true, html: { multipart: true, class: "merge-request-settings-form" }, authenticity_token: true do |f| = render 'merge_request_settings', form: f - = f.submit 'Save changes', class: "btn btn-save" + = f.submit 'Save changes', class: "btn btn-save qa-save-merge-request-changes" = render 'export', project: @project diff --git a/app/views/projects/issues/show.html.haml b/app/views/projects/issues/show.html.haml index 91f68d8c419..d63443c9da5 100644 --- a/app/views/projects/issues/show.html.haml +++ b/app/views/projects/issues/show.html.haml @@ -55,7 +55,7 @@ .issue-details.issuable-details .detail-page-description.content-block - %script#js-issuable-app-initial-data{ type: "application/json" }= issuable_initial_data(@issue) + %script#js-issuable-app-initial-data{ type: "application/json" }= issuable_initial_data(@issue).to_json #js-issuable-app %h2.title= markdown_field(@issue, :title) - if @issue.description.present? @@ -82,6 +82,3 @@ = render 'projects/issues/discussion' = render 'shared/issuable/sidebar', issuable: @issue - -= webpack_bundle_tag('common_vue') -= webpack_bundle_tag('issue_show') diff --git a/app/views/projects/pages/_list.html.haml b/app/views/projects/pages/_list.html.haml index a85cda407af..75df92b05a7 100644 --- a/app/views/projects/pages/_list.html.haml +++ b/app/views/projects/pages/_list.html.haml @@ -3,15 +3,26 @@ .panel-heading Domains (#{@domains.count}) %ul.well-list + - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled? - @domains.each do |domain| %li .pull-right = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped" = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped" .clearfix - %span= link_to domain.domain, domain.url + - if verification_enabled + - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success'] + = link_to domain.url, title: tooltip, class: 'has-tooltip' do + = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}") + = domain.domain + - else + = link_to domain.domain, domain.url %p - if domain.subject %span.label.label-gray Certificate: #{domain.subject} - if domain.expired? %span.label.label-danger Expired + - if verification_enabled && domain.unverified? + %li.warning-row + #{domain.domain} is not verified. To learn how to verify ownership, visit your + = link_to 'domain details', project_pages_domain_path(@project, domain) diff --git a/app/views/projects/pages_domains/show.html.haml b/app/views/projects/pages_domains/show.html.haml index 876cac0dacb..72e9203bdb0 100644 --- a/app/views/projects/pages_domains/show.html.haml +++ b/app/views/projects/pages_domains/show.html.haml @@ -1,4 +1,10 @@ - page_title "#{@domain.domain}", 'Pages Domains' +- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled? +- if verification_enabled && @domain.unverified? + %p.alert.alert-warning + %strong + This domain is not verified. You will need to verify ownership before + access is enabled. %h3.page-title Pages Domain @@ -15,9 +21,26 @@ DNS %td %p - To access the domain create a new DNS record: + To access this domain create a new DNS record: %pre #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}. + - if verification_enabled + %tr + %td + Verification status + %td + %p + - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + To #{link_to 'verify ownership', help_link} of your domain, create + this DNS record: + %pre + #{@domain.verification_domain} TXT #{@domain.keyed_verification_code} + %p + - if @domain.verified? + #{@domain.domain} has been successfully verified. + - else + = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm' + %tr %td Certificate diff --git a/app/views/projects/runners/_shared_runners.html.haml b/app/views/projects/runners/_shared_runners.html.haml index b037b57e78a..4fd4ca355a8 100644 --- a/app/views/projects/runners/_shared_runners.html.haml +++ b/app/views/projects/runners/_shared_runners.html.haml @@ -1,6 +1,6 @@ %h3 Shared Runners -.bs-callout.bs-callout-warning.shared-runners-description +.bs-callout.shared-runners-description - if Gitlab::CurrentSettings.shared_runners_text.present? = markdown_field(Gitlab::CurrentSettings.current_application_settings, :shared_runners_text) - else @@ -9,7 +9,7 @@ on GitLab.com). %hr - if @project.shared_runners_enabled? - = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do + = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-close', method: :post do Disable shared Runners - else = link_to toggle_shared_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do diff --git a/app/views/projects/runners/_specific_runners.html.haml b/app/views/projects/runners/_specific_runners.html.haml index 28ccbf7eb15..f0813e56b71 100644 --- a/app/views/projects/runners/_specific_runners.html.haml +++ b/app/views/projects/runners/_specific_runners.html.haml @@ -1,8 +1,7 @@ %h3 Specific Runners -= render partial: 'ci/runner/how_to_setup_runner', - locals: { registration_token: @project.runners_token, - type: 'specific' } += render partial: 'ci/runner/how_to_setup_specific_runner', + locals: { registration_token: @project.runners_token } - if @project_runners.any? %h4.underlined-title Runners activated for this project diff --git a/app/views/projects/services/prometheus/_show.html.haml b/app/views/projects/services/prometheus/_show.html.haml index 5f38ecd6820..6dc2b85fd32 100644 --- a/app/views/projects/services/prometheus/_show.html.haml +++ b/app/views/projects/services/prometheus/_show.html.haml @@ -7,7 +7,7 @@ = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus') .col-lg-9 - .panel.panel-default.js-panel-monitored-metrics{ data: { "active-metrics" => "#{project_prometheus_active_metrics_path(@project, :json)}" } } + .panel.panel-default.js-panel-monitored-metrics{ data: { active_metrics: active_common_project_prometheus_metrics_path(@project, :json) } } .panel-heading %h3.panel-title = s_('PrometheusService|Monitored') diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index 3077203c2a6..235d532bf98 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -4,7 +4,6 @@ - content_for :page_specific_javascripts do = webpack_bundle_tag('common_vue') - = webpack_bundle_tag('deploy_keys') -# Protected branches & tags use a lot of nested partials. -# The shared parts of the views can be found in the `shared` directory. diff --git a/app/views/shared/boards/_show.html.haml b/app/views/shared/boards/_show.html.haml index a10fc42b82d..3312254f5fb 100644 --- a/app/views/shared/boards/_show.html.haml +++ b/app/views/shared/boards/_show.html.haml @@ -6,7 +6,6 @@ - content_for :page_specific_javascripts do = webpack_bundle_tag 'common_vue' - = webpack_bundle_tag 'boards' %script#js-board-template{ type: "text/x-template" }= render "shared/boards/components/board" %script#js-board-modal-filter{ type: "text/x-template" }= render "shared/issuable/search_bar", type: :boards_modal diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f2c20114534..28a5e5da037 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -3,6 +3,7 @@ - cronjob:expire_build_artifacts - cronjob:gitlab_usage_ping - cronjob:import_export_project_cleanup +- cronjob:pages_domain_verification_cron - cronjob:pipeline_schedule - cronjob:prune_old_events - cronjob:remove_expired_group_links @@ -82,6 +83,7 @@ - new_merge_request - new_note - pages +- pages_domain_verification - post_receive - process_commit - project_cache diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb index 09559e3b696..d7e24491516 100644 --- a/app/workers/authorized_projects_worker.rb +++ b/app/workers/authorized_projects_worker.rb @@ -1,42 +1,10 @@ class AuthorizedProjectsWorker include ApplicationWorker + prepend WaitableWorker - # Schedules multiple jobs and waits for them to be completed. - def self.bulk_perform_and_wait(args_list) - # Short-circuit: it's more efficient to do small numbers of jobs inline - return bulk_perform_inline(args_list) if args_list.size <= 3 - - waiter = Gitlab::JobWaiter.new(args_list.size) - - # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]] - # into [[1, "key"], [2, "key"], [3, "key"]] - waiting_args_list = args_list.map { |args| [*args, waiter.key] } - bulk_perform_async(waiting_args_list) - - waiter.wait - end - - # Performs multiple jobs directly. Failed jobs will be put into sidekiq so - # they can benefit from retries - def self.bulk_perform_inline(args_list) - failed = [] - - args_list.each do |args| - begin - new.perform(*args) - rescue - failed << args - end - end - - bulk_perform_async(failed) if failed.present? - end - - def perform(user_id, notify_key = nil) + def perform(user_id) user = User.find_by(id: user_id) user&.refresh_authorized_projects - ensure - Gitlab::JobWaiter.notify(notify_key, jid) if notify_key end end diff --git a/app/workers/concerns/waitable_worker.rb b/app/workers/concerns/waitable_worker.rb new file mode 100644 index 00000000000..48ebe862248 --- /dev/null +++ b/app/workers/concerns/waitable_worker.rb @@ -0,0 +1,44 @@ +module WaitableWorker + extend ActiveSupport::Concern + + module ClassMethods + # Schedules multiple jobs and waits for them to be completed. + def bulk_perform_and_wait(args_list, timeout: 10) + # Short-circuit: it's more efficient to do small numbers of jobs inline + return bulk_perform_inline(args_list) if args_list.size <= 3 + + waiter = Gitlab::JobWaiter.new(args_list.size) + + # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]] + # into [[1, "key"], [2, "key"], [3, "key"]] + waiting_args_list = args_list.map { |args| [*args, waiter.key] } + bulk_perform_async(waiting_args_list) + + waiter.wait(timeout) + end + + # Performs multiple jobs directly. Failed jobs will be put into sidekiq so + # they can benefit from retries + def bulk_perform_inline(args_list) + failed = [] + + args_list.each do |args| + begin + new.perform(*args) + rescue + failed << args + end + end + + bulk_perform_async(failed) if failed.present? + end + end + + def perform(*args) + notify_key = args.pop if Gitlab::JobWaiter.key?(args.last) + + super(*args) + ensure + Gitlab::JobWaiter.notify(notify_key, jid) if notify_key + end +end diff --git a/app/workers/pages_domain_verification_cron_worker.rb b/app/workers/pages_domain_verification_cron_worker.rb new file mode 100644 index 00000000000..a3ff4bd2101 --- /dev/null +++ b/app/workers/pages_domain_verification_cron_worker.rb @@ -0,0 +1,10 @@ +class PagesDomainVerificationCronWorker + include ApplicationWorker + include CronjobQueue + + def perform + PagesDomain.needs_verification.find_each do |domain| + PagesDomainVerificationWorker.perform_async(domain.id) + end + end +end diff --git a/app/workers/pages_domain_verification_worker.rb b/app/workers/pages_domain_verification_worker.rb new file mode 100644 index 00000000000..2e93489113c --- /dev/null +++ b/app/workers/pages_domain_verification_worker.rb @@ -0,0 +1,11 @@ +class PagesDomainVerificationWorker + include ApplicationWorker + + def perform(domain_id) + domain = PagesDomain.find_by(id: domain_id) + + return unless domain + + VerifyPagesDomainService.new(domain).execute + end +end diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index e0e6d1418de..fbb14efc525 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -16,43 +16,41 @@ class StuckImportJobsWorker private def mark_projects_without_jid_as_failed! - started_projects_without_jid.each do |project| + enqueued_projects_without_jid.each do |project| project.mark_import_as_failed(error_message) end.count end def mark_projects_with_jid_as_failed! - completed_jids_count = 0 + jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h - started_projects_with_jid.find_in_batches(batch_size: 500) do |group| - jids = group.map(&:import_jid) + # Find the jobs that aren't currently running or that exceeded the threshold. + completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) + return unless completed_jids.any? - # Find the jobs that aren't currently running or that exceeded the threshold. - completed_jids = Gitlab::SidekiqStatus.completed_jids(jids).to_set + completed_project_ids = jids_and_ids.values_at(*completed_jids) - if completed_jids.any? - completed_jids_count += completed_jids.count - group.each do |project| - project.mark_import_as_failed(error_message) if completed_jids.include?(project.import_jid) - end + # We select the projects again, because they may have transitioned from + # scheduled/started to finished/failed while we were looking up their Sidekiq status. + completed_projects = enqueued_projects_with_jid.where(id: completed_project_ids) - Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_jids.to_a.join(', ')}") - end - end + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_projects.map(&:import_jid).join(', ')}") - completed_jids_count + completed_projects.each do |project| + project.mark_import_as_failed(error_message) + end.count end - def started_projects - Project.with_import_status(:started) + def enqueued_projects + Project.with_import_status(:scheduled, :started) end - def started_projects_with_jid - started_projects.where.not(import_jid: nil) + def enqueued_projects_with_jid + enqueued_projects.where.not(import_jid: nil) end - def started_projects_without_jid - started_projects.where(import_jid: nil) + def enqueued_projects_without_jid + enqueued_projects.where(import_jid: nil) end def error_message diff --git a/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml b/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml new file mode 100644 index 00000000000..f958f3f1272 --- /dev/null +++ b/changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml @@ -0,0 +1,5 @@ +--- +title: Add verification for GitLab Pages custom domains +merge_request: +author: +type: security diff --git a/changelogs/unreleased/34130-null-pipes.yml b/changelogs/unreleased/34130-null-pipes.yml deleted file mode 100644 index a56e5cf8db2..00000000000 --- a/changelogs/unreleased/34130-null-pipes.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Prevent MR Widget error when no CI configured -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml b/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml deleted file mode 100644 index 27eee7d943b..00000000000 --- a/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Improve query performance of MembersFinder. -merge_request: 17190 -author: -type: performance diff --git a/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml b/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml deleted file mode 100644 index 507367c98c4..00000000000 --- a/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Enable Legacy Authorization by default on Cluster creations -merge_request: 17302 -author: -type: fixed diff --git a/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml b/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml new file mode 100644 index 00000000000..6cf0de5b3fa --- /dev/null +++ b/changelogs/unreleased/42044-osw-add-button-to-deploy-runner-to-kubernetes.yml @@ -0,0 +1,5 @@ +--- +title: Add a button to deploy a runner to a Kubernetes cluster in the settings page +merge_request: 17278 +author: +type: changed diff --git a/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml b/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml new file mode 100644 index 00000000000..d29f79aaaf8 --- /dev/null +++ b/changelogs/unreleased/42545-milestion-quick-actions-for-groups.yml @@ -0,0 +1,5 @@ +--- +title: Allows the usage of /milestone quick action for group milestones +merge_request: 17239 +author: Jacopo Beschi @jacopo-beschi +type: fixed diff --git a/changelogs/unreleased/42877-snippets-dashboard-slow.yml b/changelogs/unreleased/42877-snippets-dashboard-slow.yml deleted file mode 100644 index 839b44ad272..00000000000 --- a/changelogs/unreleased/42877-snippets-dashboard-slow.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Improve query performance for snippets dashboard. -merge_request: 17088 -author: -type: performance diff --git a/changelogs/unreleased/43373-fix-cache-index-appending.yml b/changelogs/unreleased/43373-fix-cache-index-appending.yml deleted file mode 100644 index fdb293ea04d..00000000000 --- a/changelogs/unreleased/43373-fix-cache-index-appending.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix issue with cache key being empty when variable used as the key -merge_request: 17260 -author: -type: fixed diff --git a/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml b/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml new file mode 100644 index 00000000000..c10b0e7a3cf --- /dev/null +++ b/changelogs/unreleased/43496-error-message-for-gke-clusters-persists-in-the-next-page.yml @@ -0,0 +1,5 @@ +--- +title: Do not persist Google Project verification flash errors after a page reload +merge_request: 17299 +author: +type: fixed diff --git a/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml b/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml new file mode 100644 index 00000000000..bda4ec84e5c --- /dev/null +++ b/changelogs/unreleased/43598-fix-duplicate-label-load-failure.yml @@ -0,0 +1,5 @@ +--- +title: Fix Group labels load failure when there are duplicate labels present +merge_request: 17353 +author: +type: fixed diff --git a/changelogs/unreleased/dm-stuck-import-jobs-verify.yml b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml new file mode 100644 index 00000000000..ed2c2d30f0d --- /dev/null +++ b/changelogs/unreleased/dm-stuck-import-jobs-verify.yml @@ -0,0 +1,5 @@ +--- +title: Verify project import status again before marking as failed +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/fix-500-for-invalid-upload-path.yml b/changelogs/unreleased/fix-500-for-invalid-upload-path.yml deleted file mode 100644 index a4ce00c64c4..00000000000 --- a/changelogs/unreleased/fix-500-for-invalid-upload-path.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix 500 error when loading an invalid upload URL -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/flipper-caching.yml b/changelogs/unreleased/flipper-caching.yml deleted file mode 100644 index 6db27fd579e..00000000000 --- a/changelogs/unreleased/flipper-caching.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Increase feature flag cache TTL to one hour -merge_request: -author: -type: performance diff --git a/changelogs/unreleased/grpc-unavailable-restart.yml b/changelogs/unreleased/grpc-unavailable-restart.yml new file mode 100644 index 00000000000..5ce08d66004 --- /dev/null +++ b/changelogs/unreleased/grpc-unavailable-restart.yml @@ -0,0 +1,5 @@ +--- +title: Restart Unicorn and Sidekiq when GRPC throws 14:Endpoint read failed +merge_request: 17293 +author: +type: fixed diff --git a/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml b/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml deleted file mode 100644 index 09112fba85e..00000000000 --- a/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Only check LFS integrity for first ref in a push to avoid timeout -merge_request: 17098 -author: -type: performance diff --git a/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml b/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml deleted file mode 100644 index 690536a533b..00000000000 --- a/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix single digit value clipping for stacked progress bar -merge_request: 17217 -author: -type: fixed diff --git a/changelogs/unreleased/minimal-fix-for-artifacts-service.yml b/changelogs/unreleased/minimal-fix-for-artifacts-service.yml new file mode 100644 index 00000000000..11f5bc17759 --- /dev/null +++ b/changelogs/unreleased/minimal-fix-for-artifacts-service.yml @@ -0,0 +1,5 @@ +--- +title: Prevent trace artifact migration to incur data loss +merge_request: 17313 +author: +type: fixed diff --git a/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml b/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml new file mode 100644 index 00000000000..a761d610da1 --- /dev/null +++ b/changelogs/unreleased/mk-fix-error-code-for-repo-does-not-exist.yml @@ -0,0 +1,5 @@ +--- +title: Return a 404 instead of 403 if the repository does not exist on disk +merge_request: 17341 +author: +type: fixed diff --git a/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml b/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml new file mode 100644 index 00000000000..d65318d7ba1 --- /dev/null +++ b/changelogs/unreleased/refactor-move-filtered-search-vue-component.yml @@ -0,0 +1,5 @@ +--- +title: Move RecentSearchesDropdownContent vue component +merge_request: 16951 +author: George Tsiolis +type: performance diff --git a/changelogs/unreleased/sh-guard-read-only-user-updates.yml b/changelogs/unreleased/sh-guard-read-only-user-updates.yml deleted file mode 100644 index b8dbd840ed9..00000000000 --- a/changelogs/unreleased/sh-guard-read-only-user-updates.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Don't attempt to update user tracked fields if database is in read-only -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/zj-branch-contains-git-message.yml b/changelogs/unreleased/zj-branch-contains-git-message.yml deleted file mode 100644 index ce034e7ec87..00000000000 --- a/changelogs/unreleased/zj-branch-contains-git-message.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Allow branch names to be named the same as the sha it points to -merge_request: -author: -type: fixed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index bbc2bcfb0cc..bd696a7f2c5 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -214,6 +214,10 @@ production: &base repository_archive_cache_worker: cron: "0 * * * *" + # Verify custom GitLab Pages domains + pages_domain_verification_cron_worker: + cron: "*/15 * * * *" + registry: # enabled: true # host: registry.example.com diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 17a8801f7bc..ea0dee7af53 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *' Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker' +Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *' +Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker' + # # GitLab Shell # diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 0f164e628f9..161fb185c9b 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -10,7 +10,7 @@ Sidekiq.configure_server do |config| config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] - chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] + chain.add Gitlab::SidekiqMiddleware::Shutdown chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0' chain.add Gitlab::SidekiqStatus::ServerMiddleware end diff --git a/config/routes/project.rb b/config/routes/project.rb index 1912808f9c0..8fe545b721e 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do end resource :pages, only: [:show, :destroy] do - resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } + resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do + member do + post :verify + end + end end resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do @@ -74,7 +78,9 @@ constraints(ProjectUrlConstrainer.new) do resource :mattermost, only: [:new, :create] namespace :prometheus do - get :active_metrics + resources :metrics, constraints: { id: %r{[^\/]+} }, only: [] do + get :active_common, on: :collection + end end resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 31a38f2b508..f037e3d1221 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -67,3 +67,4 @@ - [gcp_cluster, 1] - [project_migrate_hashed_storage, 1] - [storage_migrator, 1] + - [pages_domain_verification, 1] diff --git a/config/webpack.config.js b/config/webpack.config.js index 94ff39485fb..be827903a6a 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -52,20 +52,15 @@ var config = { entry: { balsamiq_viewer: './blob/balsamiq_viewer.js', blob: './blob_edit/blob_bundle.js', - boards: './boards/boards_bundle.js', common: './commons/index.js', common_vue: './vue_shared/vue_resource_interceptor.js', cycle_analytics: './cycle_analytics/cycle_analytics_bundle.js', commit_pipelines: './commit/pipelines/pipelines_bundle.js', - deploy_keys: './deploy_keys/index.js', diff_notes: './diff_notes/diff_notes_bundle.js', environments: './environments/environments_bundle.js', environments_folder: './environments/folder/environments_folder_bundle.js', filtered_search: './filtered_search/filtered_search_bundle.js', help: './help/help.js', - issue_show: './issue_show/index.js', - locale: './locale/index.js', - main: './main.js', merge_conflicts: './merge_conflicts/merge_conflicts_bundle.js', monitoring: './monitoring/monitoring_bundle.js', network: './network/network_bundle.js', @@ -78,18 +73,24 @@ var config = { protected_branches: './protected_branches', protected_tags: './protected_tags', registry_list: './registry/index.js', - ide: './ide/index.js', sidebar: './sidebar/sidebar_bundle.js', snippet: './snippet/snippet_bundle.js', sketch_viewer: './blob/sketch_viewer.js', stl_viewer: './blob/stl_viewer.js', terminal: './terminal/terminal_bundle.js', - u2f: ['vendor/u2f'], ui_development_kit: './ui_development_kit.js', - raven: './raven/index.js', vue_merge_request_widget: './vue_merge_request_widget/index.js', - test: './test.js', two_factor_auth: './two_factor_auth.js', + + + common: './commons/index.js', + common_vue: './vue_shared/vue_resource_interceptor.js', + locale: './locale/index.js', + main: './main.js', + ide: './ide/index.js', + raven: './raven/index.js', + test: './test.js', + u2f: ['vendor/u2f'], webpack_runtime: './webpack.js', }, @@ -252,7 +253,6 @@ var config = { 'environments_folder', 'filtered_search', 'groups', - 'issue_show', 'merge_conflicts', 'monitoring', 'notebook_viewer', diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb new file mode 100644 index 00000000000..d3f68cb7d45 --- /dev/null +++ b/db/migrate/20180215181245_users_name_lower_index.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UsersNameLowerIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + INDEX_NAME = 'index_on_users_name_lower' + + disable_ddl_transaction! + + def up + return unless Gitlab::Database.postgresql? + + # On GitLab.com this produces an index with a size of roughly 60 MB. + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))" + end + + def down + return unless Gitlab::Database.postgresql? + + if supports_drop_index_concurrently? + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + end + end +end diff --git a/db/migrate/20180216120000_add_pages_domain_verification.rb b/db/migrate/20180216120000_add_pages_domain_verification.rb new file mode 100644 index 00000000000..8b7cae92285 --- /dev/null +++ b/db/migrate/20180216120000_add_pages_domain_verification.rb @@ -0,0 +1,8 @@ +class AddPagesDomainVerification < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :verified_at, :datetime_with_timezone + add_column :pages_domains, :verification_code, :string + end +end diff --git a/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb new file mode 100644 index 00000000000..825dfb52dce --- /dev/null +++ b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb @@ -0,0 +1,15 @@ +class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, :verified_at + end + + def down + remove_concurrent_index :pages_domains, :verified_at + end +end diff --git a/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb new file mode 100644 index 00000000000..06d458028b3 --- /dev/null +++ b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb @@ -0,0 +1,7 @@ +class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false + end +end diff --git a/db/migrate/20180216120030_add_pages_domain_enabled_until.rb b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb new file mode 100644 index 00000000000..b40653044dd --- /dev/null +++ b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb @@ -0,0 +1,7 @@ +class AddPagesDomainEnabledUntil < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :enabled_until, :datetime_with_timezone + end +end diff --git a/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb new file mode 100644 index 00000000000..00f6e4979da --- /dev/null +++ b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb @@ -0,0 +1,17 @@ +class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, [:project_id, :enabled_until] + add_concurrent_index :pages_domains, [:verified_at, :enabled_until] + end + + def down + remove_concurrent_index :pages_domains, [:verified_at, :enabled_until] + remove_concurrent_index :pages_domains, [:project_id, :enabled_until] + end +end diff --git a/db/migrate/20180216120050_pages_domains_verification_grace_period.rb b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb new file mode 100644 index 00000000000..d7f8634b536 --- /dev/null +++ b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb @@ -0,0 +1,26 @@ +class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + now = Time.now + grace = now + 30.days + + PagesDomain.each_batch do |relation| + relation.update_all(verified_at: now, enabled_until: grace) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb new file mode 100644 index 00000000000..d423673d2a5 --- /dev/null +++ b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb @@ -0,0 +1,41 @@ +class FillPagesDomainVerificationCode < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + PagesDomain.where(verification_code: [nil, '']).each_batch do |relation| + connection.execute(set_codes_sql(relation)) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + + change_column_null(:pages_domains, :verification_code, false) + end + + def down + change_column_null(:pages_domains, :verification_code, true) + end + + private + + def set_codes_sql(relation) + ids = relation.pluck(:id) + whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" } + + <<~SQL + UPDATE pages_domains + SET verification_code = + CASE id + #{whens.join("\n")} + END + WHERE id IN(#{ids.join(',')}) + SQL + end +end diff --git a/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb new file mode 100644 index 00000000000..bf9bf4e660f --- /dev/null +++ b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb @@ -0,0 +1,16 @@ +class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + def up + PagesDomain.each_batch do |relation| + ids = relation.pluck(:id).map { |id| [id] } + PagesDomainVerificationWorker.bulk_perform_async(ids) + end + end + + def down + # no-op + end +end diff --git a/db/schema.rb b/db/schema.rb index 409d1ac7644..5bb461169f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180213131630) do +ActiveRecord::Schema.define(version: 20180216121030) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180213131630) do t.integer "gitaly_timeout_fast", default: 10, null: false t.boolean "authorized_keys_enabled", default: true, null: false t.string "auto_devops_domain" + t.boolean "pages_domain_verification_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -1313,10 +1314,16 @@ ActiveRecord::Schema.define(version: 20180213131630) do t.string "encrypted_key_iv" t.string "encrypted_key_salt" t.string "domain" + t.datetime_with_timezone "verified_at" + t.string "verification_code", null: false + t.datetime_with_timezone "enabled_until" end add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree + add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree + add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree + add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree create_table "personal_access_tokens", force: :cascade do |t| t.integer "user_id", null: false diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index e3b10119090..d9a61aea6ef 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -2,7 +2,9 @@ [Gitaly](https://gitlab.com/gitlab-org/gitaly) (introduced in GitLab 9.0) is a service that provides high-level RPC access to Git -repositories. Gitaly is a mandatory component in GitLab 9.4 and newer. +repositories. Gitaly was optional when it was first introduced in +GitLab, but since GitLab 9.4 it is a mandatory component of the +application. GitLab components that access Git repositories (gitlab-rails, gitlab-shell, gitlab-workhorse) act as clients to Gitaly. End users do @@ -184,14 +186,20 @@ Gitaly logs on your Gitaly server (`sudo gitlab-ctl tail gitaly` or coming in. One sure way to trigger a Gitaly request is to clone a repository from your GitLab server over HTTP. -## Disabling or enabling the Gitaly service +## Disabling or enabling the Gitaly service in a cluster environment If you are running Gitaly [as a remote service](#running-gitaly-on-its-own-server) you may want to disable the local Gitaly service that runs on your Gitlab server by default. -To disable the Gitaly service in your Omnibus installation, add the -following line to `/etc/gitlab/gitlab.rb`: +> 'Disabling Gitaly' only makes sense when you run GitLab in a custom +cluster configuration, where different services run on different +machines. Disabling Gitaly on all machines in the cluster is not a +valid configuration. + +If you are setting up a GitLab cluster where Gitaly does not need to +run on all machines, you can disable the Gitaly service in your +Omnibus installation, add the following line to `/etc/gitlab/gitlab.rb`: ```ruby gitaly['enable'] = false @@ -200,11 +208,13 @@ gitaly['enable'] = false When you run `gitlab-ctl reconfigure` the Gitaly service will be disabled. -To disable the Gitaly service in an installation from source, add the -following to `/etc/default/gitlab`: +To disable the Gitaly service in a GitLab cluster where you installed +GitLab from source, add the following to `/etc/default/gitlab` on the +machine where you want to disable Gitaly. ```shell gitaly_enabled=false ``` -When you run `service gitlab restart` Gitaly will be disabled. +When you run `service gitlab restart` Gitaly will be disabled on this +particular machine. diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index edb3e4c961e..00c631fdaae 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -226,6 +226,18 @@ world. Custom domains and TLS are supported. 1. [Reconfigure GitLab][reconfigure] +### Custom domain verification + +To prevent malicious users from hijacking domains that don't belong to them, +GitLab supports [custom domain verification](../../user/project/pages/getting_started_part_three.md#dns-txt-record). +When adding a custom domain, users will be required to prove they own it by +adding a GitLab-controlled verification code to the DNS records for that domain. + +If your userbase is private or otherwise trusted, you can disable the +verification requirement. Navigate to `Admin area ➔ Settings` and uncheck +**Require users to prove ownership of custom domains** in the Pages section. +This setting is enabled by default. + ## Change storage path Follow the steps below to change the default path where GitLab Pages' contents diff --git a/doc/api/projects.md b/doc/api/projects.md index 9e649efea9c..4f4ab906c1a 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1224,7 +1224,7 @@ POST /projects/:id/hooks | `note_events` | boolean | no | Trigger hook on note events | | `job_events` | boolean | no | Trigger hook on job events | | `pipeline_events` | boolean | no | Trigger hook on pipeline events | -| `wiki_events` | boolean | no | Trigger hook on wiki events | +| `wiki_page_events` | boolean | no | Trigger hook on wiki events | | `enable_ssl_verification` | boolean | no | Do SSL verification when triggering the hook | | `token` | string | no | Secret token to validate received payloads; this will not be returned in the response | diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index d3e9bf9e6a8..10e6321eb82 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -8,8 +8,8 @@ Milestones allow you to organize issues and merge requests into a cohesive group ## Project milestones and group milestones -- **Project milestones** can be assigned to issues or merge requests in that project only. -- **Group milestones** can be assigned to any issue or merge request of any project in that group. +- **Project milestones** can be assigned to issues or merge requests in that project only. +- **Group milestones** can be assigned to any issue or merge request of any project in that group. - In the [future](https://gitlab.com/gitlab-org/gitlab-ce/issues/36862), you will be able to assign group milestones to issues and merge reqeusts of projects in [subgroups](../../group/subgroups/index.md). ## Creating milestones @@ -108,4 +108,4 @@ The milestone sidebar on the milestone view shows the following: For project milestones only, the milestone sidebar shows the total issue weight of all issues that have the milestone assigned. -
\ No newline at end of file + diff --git a/doc/user/project/pages/getting_started_part_three.md b/doc/user/project/pages/getting_started_part_three.md index b6cf68a02a2..430fe3af1f8 100644 --- a/doc/user/project/pages/getting_started_part_three.md +++ b/doc/user/project/pages/getting_started_part_three.md @@ -62,7 +62,7 @@ for the most popular hosting services: - [Microsoft](https://msdn.microsoft.com/en-us/library/bb727018.aspx) If your hosting service is not listed above, you can just try to -search the web for "how to add dns record on <my hosting service>". +search the web for `how to add dns record on <my hosting service>`. ### DNS A record @@ -95,12 +95,32 @@ without any `/project-name`.  -### TL;DR +#### DNS TXT record + +Unless your GitLab administrator has [disabled custom domain verification](../../../administration/pages/index.md#custom-domain-verification), +you'll have to prove that you own the domain by creating a `TXT` record +containing a verification code. The code will be displayed after you +[add your custom domain to GitLab Pages settings](#add-your-custom-domain-to-gitlab-pages-settings). + +If using a [DNS A record](#dns-a-record), you can place the TXT record directly +under the domain. If using a [DNS CNAME record](#dns-cname-record), the two record types won't +co-exist, so you need to place the TXT record in a special subdomain of its own. + +#### TL;DR + +If the domain has multiple uses (e.g., you host email on it as well): | From | DNS Record | To | | ---- | ---------- | -- | | domain.com | A | 52.167.214.135 | -| subdomain.domain.com | CNAME | namespace.gitlab.io | +| domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff | + +If the domain is dedicated to GitLab Pages use and no other services run on it: + +| From | DNS Record | To | +| ---- | ---------- | -- | +| subdomain.domain.com | CNAME | gitlab.io | +| _gitlab-pages-verification-code.subdomain.domain.com | TXT | gitlab-pages-verification-code=00112233445566778899aabbccddeeff | > **Notes**: > @@ -121,6 +141,17 @@ your site will be accessible only via HTTP:  +Once you have added a new domain, you will need to **verify your ownership** +(unless the GitLab administrator has disabled this feature). A verification code +will be shown to you; add it as a [DNS TXT record](#dns-txt-record), then press +the "Verify ownership" button to activate your new domain: + + + +Once your domain has been verified, leave the verification record in place - +your domain will be periodically reverified, and may be disabled if the record +is removed. + You can add more than one alias (custom domains and subdomains) to the same project. An alias can be understood as having many doors leading to the same room. @@ -128,8 +159,8 @@ All the aliases you've set to your site will be listed on **Setting > Pages**. From that page, you can view, add, and remove them. Note that [DNS propagation may take some time (up to 24h)](http://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/domain-names-dns-changes), -although it's usually a matter of minutes to complete. Until it does, visit attempts -to your domain will respond with a 404. +although it's usually a matter of minutes to complete. Until it does, verification +will fail and attempts to visit your domain will respond with a 404. Read through the [general documentation on GitLab Pages](introduction.md#add-a-custom-domain-to-your-pages-website) to learn more about adding custom domains to GitLab Pages sites. diff --git a/doc/user/project/pages/img/verify_your_domain.png b/doc/user/project/pages/img/verify_your_domain.png Binary files differnew file mode 100644 index 00000000000..89c69cac9a5 --- /dev/null +++ b/doc/user/project/pages/img/verify_your_domain.png diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 45c737c6c29..167878ba600 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1154,6 +1154,10 @@ module API expose :domain expose :url expose :project_id + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, @@ -1165,6 +1169,10 @@ module API class PagesDomain < Grape::Entity expose :domain expose :url + expose :verified?, as: :verified + expose :verification_code, as: :verification_code + expose :enabled_until + expose :certificate, if: ->(pages_domain, _) { pages_domain.certificate? }, using: PagesDomainCertificate do |pages_domain| diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bbdb593d4e2..6400089a22f 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -199,7 +199,7 @@ module Gitlab def check_repository_existence! unless repository.exists? - raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + raise NotFoundError, ERROR_MESSAGES[:no_repo] end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c5d3e944f7d..9cd76630484 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -125,6 +125,8 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + rescue GRPC::Unavailable => ex + handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -135,6 +137,27 @@ module Gitlab duration) end + def self.handle_grpc_unavailable!(ex) + status = ex.to_status + raise ex unless status.details == 'Endpoint read failed' + + # There is a bug in grpc 1.8.x that causes a client process to get stuck + # always raising '14:Endpoint read failed'. The only thing that we can + # do to recover is to restart the process. + # + # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + + if Sidekiq.server? + raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) + else + # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. + Process.kill('QUIT', Process.pid) + end + + raise ex + end + private_class_method :handle_grpc_unavailable! + def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index f654508c391..f7a8eae0be4 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -15,16 +15,22 @@ module Gitlab # push to that array when done. Once the waiter has popped `count` items, it # knows all the jobs are done. class JobWaiter + KEY_PREFIX = "gitlab:job_waiter".freeze + def self.notify(key, jid) Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end + def self.key?(key) + key.is_a?(String) && key =~ /\A#{KEY_PREFIX}:\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ + end + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 729fef34b35..e91c6fb2e27 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -6,9 +6,14 @@ module Gitlab attr_accessor :name, :priority, :metrics validates :name, :priority, :metrics, presence: true - def self.all + def self.common_metrics AdditionalMetricsParser.load_groups_from_yaml end + + # EE only + def self.for_project(_) + common_metrics + end end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 294a6ae34ca..972ab75d1d5 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( + deployment.project, common_query_context( deployment.environment, timeframe_start: (deployment.created_at - 30.minutes).to_f, diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index 32fe8201a8d..9273e69e158 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -7,6 +7,7 @@ module Gitlab def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( + environment.project, common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) ) end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index 4c3edccc71a..5710ad47c1a 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -18,7 +18,7 @@ module Gitlab private def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) lookup = active_series_lookup(metrics_groups) groups = {} diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 5cddc96a643..0c280dc9a3c 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -2,10 +2,10 @@ module Gitlab module Prometheus module Queries module QueryAdditionalMetrics - def query_metrics(query_context) + def query_metrics(project, query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics.map do |group| + groups = matched_metrics(project).map do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -60,8 +60,8 @@ module Gitlab @available_metrics ||= client_label_values || [] end - def matched_metrics - result = Gitlab::Prometheus::MetricGroup.all.map do |group| + def matched_metrics(project) + result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group| group.metrics.select! do |metric| metric.required_metrics.all?(&available_metrics.method(:include?)) end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 10527972663..659021c9ac9 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -1,8 +1,9 @@ module Gitlab - PrometheusError = Class.new(StandardError) - # Helper methods to interact with Prometheus network services & resources class PrometheusClient + Error = Class.new(StandardError) + QueryError = Class.new(Gitlab::PrometheusClient::Error) + attr_reader :rest_client, :headers def initialize(rest_client) @@ -22,10 +23,10 @@ module Gitlab def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', - query: query, - start: start.to_f, - end: stop.to_f, - step: 1.minute.to_i) + query: query, + start: start.to_f, + end: stop.to_f, + step: 1.minute.to_i) end end @@ -43,22 +44,22 @@ module Gitlab path = ['api', 'v1', type].join('/') get(path, args) rescue JSON::ParserError - raise PrometheusError, 'Parsing response failed' + raise PrometheusClient::Error, 'Parsing response failed' rescue Errno::ECONNREFUSED - raise PrometheusError, 'Connection refused' + raise PrometheusClient::Error, 'Connection refused' end def get(path, args) response = rest_client[path].get(params: args) handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex handle_exception_response(ex.response) rescue RestClient::Exception - raise PrometheusError, "Network connection error" + raise PrometheusClient::Error, "Network connection error" end def handle_response(response) @@ -66,16 +67,18 @@ module Gitlab if response.code == 200 && json_data['status'] == 'success' json_data['data'] || {} else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end def handle_exception_response(response) - if response.code == 400 + if response.code == 200 && response['status'] == 'success' + response['data'] || {} + elsif response.code == 400 json_data = JSON.parse(response.body) - raise PrometheusError, json_data['error'] || 'Bad data received' + raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusError, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response.code} - #{response.body}" end end diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 3937d9c153a..96415271316 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -24,15 +24,14 @@ module Gitlab action_block.nil? end - def available?(opts) + def available?(context) return true unless condition_block - context = OpenStruct.new(opts) context.instance_exec(&condition_block) end - def explain(context, opts, arg) - return unless available?(opts) + def explain(context, arg) + return unless available?(context) if explanation.respond_to?(:call) execute_block(explanation, context, arg) @@ -41,15 +40,13 @@ module Gitlab end end - def execute(context, opts, arg) - return if noop? || !available?(opts) + def execute(context, arg) + return if noop? || !available?(context) execute_block(action_block, context, arg) end - def to_h(opts) - context = OpenStruct.new(opts) - + def to_h(context) desc = description if desc.respond_to?(:call) desc = context.instance_exec(&desc) rescue '' diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 536765305e1..d82dccd0db5 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -62,9 +62,8 @@ module Gitlab # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. - # It accepts a block that will be evaluated with the context given to - # `CommandDefintion#to_h`. - # + # It accepts a block that will be evaluated with the context + # of a QuickActions::InterpretService instance # Example: # # condition do diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index c0878a34fb1..075ff91700c 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content, opts = {}) + def extract_commands(content) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex(opts)) do + content.gsub!(commands_regex) do if $~[:cmd] commands << [$~[:cmd], $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex(opts) - names = command_names(opts).map(&:to_s) + def commands_regex + names = command_names.map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -133,7 +133,7 @@ module Gitlab [content, commands] end - def command_names(opts) + def command_names command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index b89ae2505c9..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - def call(worker, job, queue) - yield - - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - return unless MUTEX.try_lock - - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status.zero? - - output.to_i - end - - def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - sleep(time) - - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" - Process.kill(signal, pid) - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb new file mode 100644 index 00000000000..c2b8d6de66e --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -0,0 +1,133 @@ +require 'mutex_m' + +module Gitlab + module SidekiqMiddleware + class Shutdown + extend Mutex_m + + # Default the RSS limit to 0, meaning the MemoryKiller is disabled + MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i + # Wait 30 seconds for running jobs to finish during graceful shutdown + SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i + + # This exception can be used to request that the middleware start shutting down Sidekiq + WantShutdown = Class.new(StandardError) + + ShutdownWithoutRaise = Class.new(WantShutdown) + private_constant :ShutdownWithoutRaise + + # For testing only, to avoid race conditions (?) in Rspec mocks. + attr_reader :trace + + # We store the shutdown thread in a class variable to ensure that there + # can be only one shutdown thread in the process. + def self.create_shutdown_thread + mu_synchronize do + return unless @shutdown_thread.nil? + + @shutdown_thread = Thread.new { yield } + end + end + + # For testing only: so we can wait for the shutdown thread to finish. + def self.shutdown_thread + mu_synchronize { @shutdown_thread } + end + + # For testing only: so that we can reset the global state before each test. + def self.clear_shutdown_thread + mu_synchronize { @shutdown_thread = nil } + end + + def initialize + @trace = Queue.new if Rails.env.test? + end + + def call(worker, job, queue) + shutdown_exception = nil + + begin + yield + check_rss! + rescue WantShutdown => ex + shutdown_exception = ex + end + + return unless shutdown_exception + + self.class.create_shutdown_thread do + do_shutdown(worker, job, shutdown_exception) + end + + raise shutdown_exception unless shutdown_exception.is_a?(ShutdownWithoutRaise) + end + + private + + def do_shutdown(worker, job, shutdown_exception) + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} shutting down because of #{shutdown_exception} after job "\ + "#{worker.class} JID-#{job['jid']}" + Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + + # Wait `GRACE_TIME` to give the memory intensive job time to finish. + # Then, tell Sidekiq to stop fetching new jobs. + wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') + + # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. + # Then, tell Sidekiq to gracefully shut down by giving jobs a few more + # moments to finish, killing and requeuing them if they didn't, and + # then terminating itself. + wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') + + # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. + wait_and_signal(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') + end + + def check_rss! + return unless MAX_RSS > 0 + + current_rss = get_rss + return unless current_rss > MAX_RSS + + raise ShutdownWithoutRaise.new("current RSS #{current_rss} exceeds maximum RSS #{MAX_RSS}") + end + + def get_rss + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) + return 0 unless status.zero? + + output.to_i + end + + def wait_and_signal(time, signal, explanation) + Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + sleep(time) + + Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + kill(signal, pid) + end + + def pid + Process.pid + end + + def sleep(time) + if Rails.env.test? + @trace << [:sleep, time] + else + Kernel.sleep(time) + end + end + + def kill(signal, pid) + if Rails.env.test? + @trace << [:kill, signal, pid] + else + Process.kill(signal, pid) + end + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 5f0c98cb5a4..53744bad1f4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -25,7 +25,11 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def fuzzy_arel_match(column, query) + # column - The column name to search in. + # query - The text to search for. + # lower_exact_match - When set to `true` we'll fall back to using + # `LOWER(column) = query` instead of using `ILIKE`. + def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish return nil unless query.present? @@ -36,7 +40,13 @@ module Gitlab else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - arel_table[column].matches(sanitize_sql_like(query)) + if lower_exact_match + Arel::Nodes::NamedFunction + .new('LOWER', [arel_table[column]]) + .eq(query) + else + arel_table[column].matches(sanitize_sql_like(query)) + end end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 31cbd651edb..1c7a8a90f5c 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,4 +18,5 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up ReworkRedirectRoutesIndexes.new.up + UsersNameLowerIndex.new.up end @@ -130,6 +130,7 @@ module QA autoload :DeployKeys, 'qa/page/project/settings/deploy_keys' autoload :SecretVariables, 'qa/page/project/settings/secret_variables' autoload :Runners, 'qa/page/project/settings/runners' + autoload :MergeRequest, 'qa/page/project/settings/merge_request' end module Issue @@ -145,6 +146,7 @@ module QA module MergeRequest autoload :New, 'qa/page/merge_request/new' + autoload :Show, 'qa/page/merge_request/show' end module Admin diff --git a/qa/qa/factory/base.rb b/qa/qa/factory/base.rb index bd66b74a164..afaa96b4541 100644 --- a/qa/qa/factory/base.rb +++ b/qa/qa/factory/base.rb @@ -22,7 +22,7 @@ module QA factory.fabricate!(*args) - return Factory::Product.populate!(self) + return Factory::Product.populate!(factory) end end diff --git a/qa/qa/factory/product.rb b/qa/qa/factory/product.rb index d004e642f9b..996b7f14f61 100644 --- a/qa/qa/factory/product.rb +++ b/qa/qa/factory/product.rb @@ -17,8 +17,9 @@ module QA def self.populate!(factory) new.tap do |product| - factory.attributes.each_value do |attribute| - product.instance_exec(&attribute.block).tap do |value| + factory.class.attributes.each_value do |attribute| + product.instance_exec(factory, attribute.block) do |factory, block| + value = block.call(factory) product.define_singleton_method(attribute.name) { value } end end diff --git a/qa/qa/factory/repository/push.rb b/qa/qa/factory/repository/push.rb index 2f4de4173d4..6e8905cde78 100644 --- a/qa/qa/factory/repository/push.rb +++ b/qa/qa/factory/repository/push.rb @@ -2,7 +2,7 @@ module QA module Factory module Repository class Push < Factory::Base - attr_writer :file_name, :file_content, :commit_message, :branch_name + attr_writer :file_name, :file_content, :commit_message, :branch_name, :new_branch dependency Factory::Resource::Project, as: :project do |project| project.name = 'project-with-code' @@ -14,6 +14,7 @@ module QA @file_content = '# This is test project' @commit_message = "Add #{@file_name}" @branch_name = 'master' + @new_branch = true end def fabricate! @@ -29,6 +30,7 @@ module QA repository.clone repository.configure_identity('GitLab QA', 'root@gitlab.com') + repository.checkout(@branch_name) unless @new_branch repository.add_file(@file_name, @file_content) repository.commit(@commit_message) repository.push_changes(@branch_name) diff --git a/qa/qa/factory/resource/merge_request.rb b/qa/qa/factory/resource/merge_request.rb index ce04e904aaf..539fe6b8a70 100644 --- a/qa/qa/factory/resource/merge_request.rb +++ b/qa/qa/factory/resource/merge_request.rb @@ -9,11 +9,20 @@ module QA :source_branch, :target_branch + product :project do |factory| + factory.project + end + + product :source_branch do |factory| + factory.source_branch + end + dependency Factory::Resource::Project, as: :project do |project| project.name = 'project-with-merge-request' end dependency Factory::Repository::Push, as: :target do |push, factory| + factory.project.visit! push.project = factory.project push.branch_name = "master:#{factory.target_branch}" end diff --git a/qa/qa/git/repository.rb b/qa/qa/git/repository.rb index 4c4ef3ef477..b3150e8f3fa 100644 --- a/qa/qa/git/repository.rb +++ b/qa/qa/git/repository.rb @@ -36,6 +36,10 @@ module QA `git clone #{opts} #{@uri.to_s} ./ #{suppress_output}` end + def checkout(branch_name) + `git checkout "#{branch_name}"` + end + def shallow_clone clone('--depth 1') end diff --git a/qa/qa/page/merge_request/show.rb b/qa/qa/page/merge_request/show.rb new file mode 100644 index 00000000000..35875487da8 --- /dev/null +++ b/qa/qa/page/merge_request/show.rb @@ -0,0 +1,46 @@ +module QA + module Page + module MergeRequest + class Show < Page::Base + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_ready_to_merge.js' do + element :merge_button + end + + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_merged.vue' do + element :merged_status, 'The changes were merged into' + end + + view 'app/assets/javascripts/vue_merge_request_widget/components/states/mr_widget_rebase.vue' do + element :mr_rebase_button + element :fast_forward_nessage, "Fast-forward merge is not possible" + end + + def rebase! + wait(reload: false) do + click_element :mr_rebase_button + + has_text?("The source branch HEAD has recently changed.") + end + end + + def fast_forward_possible? + !has_text?("Fast-forward merge is not possible") + end + + def has_merge_button? + refresh + + has_selector?('.accept-merge-request') + end + + def merge! + wait(reload: false) do + click_element :merge_button + + has_text?("The changes were merged into") + end + end + end + end + end +end diff --git a/qa/qa/page/project/settings/merge_request.rb b/qa/qa/page/project/settings/merge_request.rb new file mode 100644 index 00000000000..b147c91b467 --- /dev/null +++ b/qa/qa/page/project/settings/merge_request.rb @@ -0,0 +1,27 @@ +module QA + module Page + module Project + module Settings + class MergeRequest < QA::Page::Base + include Common + + view 'app/views/projects/_merge_request_fast_forward_settings.html.haml' do + element :radio_button_merge_ff + end + + view 'app/views/projects/edit.html.haml' do + element :merge_request_settings, 'Merge request settings' + element :save_merge_request_changes + end + + def enable_ff_only + expand_section('Merge request settings') do + click_element :radio_button_merge_ff + click_element :save_merge_request_changes + end + end + end + end + end + end +end diff --git a/qa/qa/specs/features/merge_request/rebase_spec.rb b/qa/qa/specs/features/merge_request/rebase_spec.rb new file mode 100644 index 00000000000..2a44d42af6f --- /dev/null +++ b/qa/qa/specs/features/merge_request/rebase_spec.rb @@ -0,0 +1,39 @@ +module QA + feature 'merge request rebase', :core do + scenario 'rebases source branch of merge request' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + project = Factory::Resource::Project.fabricate! do |project| + project.name = "only-fast-forward" + end + + Page::Menu::Side.act { go_to_settings } + Page::Project::Settings::MergeRequest.act { enable_ff_only } + + merge_request = Factory::Resource::MergeRequest.fabricate! do |merge_request| + merge_request.project = project + merge_request.title = 'Needs rebasing' + end + + Factory::Repository::Push.fabricate! do |push| + push.project = project + push.file_name = "other.txt" + push.file_content = "New file added!" + end + + merge_request.visit! + + Page::MergeRequest::Show.perform do |merge_request| + expect(merge_request).to have_content('Needs rebasing') + expect(merge_request).not_to be_fast_forward_possible + expect(merge_request).not_to have_merge_button + + merge_request.rebase! + + expect(merge_request).to have_merge_button + expect(merge_request.fast_forward_possible?).to be_truthy + end + end + end +end diff --git a/qa/spec/factory/base_spec.rb b/qa/spec/factory/base_spec.rb index c5663049be8..04e04886699 100644 --- a/qa/spec/factory/base_spec.rb +++ b/qa/spec/factory/base_spec.rb @@ -7,6 +7,7 @@ describe QA::Factory::Base do before do allow(QA::Factory::Product).to receive(:new).and_return(product) + allow(QA::Factory::Product).to receive(:populate!).and_return(product) end it 'instantiates the factory and calls factory method' do @@ -76,6 +77,7 @@ describe QA::Factory::Base do allow(subject).to receive(:new).and_return(instance) allow(instance).to receive(:mydep).and_return(nil) allow(QA::Factory::Product).to receive(:new) + allow(QA::Factory::Product).to receive(:populate!) end it 'builds all dependencies first' do @@ -89,8 +91,16 @@ describe QA::Factory::Base do describe '.product' do subject do Class.new(described_class) do + def fabricate! + "any" + end + + # Defined only to be stubbed + def self.find_page + end + product :token do - page.do_something_on_page! + find_page.do_something_on_page! 'resulting value' end end @@ -105,16 +115,17 @@ describe QA::Factory::Base do let(:page) { spy('page') } before do - allow(subject).to receive(:new).and_return(factory) + allow(factory).to receive(:class).and_return(subject) allow(QA::Factory::Product).to receive(:new).and_return(product) allow(product).to receive(:page).and_return(page) + allow(subject).to receive(:find_page).and_return(page) end it 'populates product after fabrication' do subject.fabricate! - expect(page).to have_received(:do_something_on_page!) expect(product.token).to eq 'resulting value' + expect(page).to have_received(:do_something_on_page!) end end end diff --git a/qa/spec/factory/product_spec.rb b/qa/spec/factory/product_spec.rb index fdfb1ec90cc..f245aabbf43 100644 --- a/qa/spec/factory/product_spec.rb +++ b/qa/spec/factory/product_spec.rb @@ -1,9 +1,20 @@ describe QA::Factory::Product do - let(:factory) { spy('factory') } + let(:factory) do + QA::Factory::Base.new + end + + let(:attributes) do + { test: QA::Factory::Product::Attribute.new(:test, proc { 'returned' }) } + end + let(:product) { spy('product') } + before do + allow(QA::Factory::Base).to receive(:attributes).and_return(attributes) + end + describe '.populate!' do - it 'returns a fabrication product' do + it 'returns a fabrication product and define factory attributes as its methods' do expect(described_class).to receive(:new).and_return(product) result = described_class.populate!(factory) do |instance| @@ -11,6 +22,7 @@ describe QA::Factory::Product do end expect(result).to be product + expect(result.test).to eq('returned') end end diff --git a/spec/controllers/autocomplete_controller_spec.rb b/spec/controllers/autocomplete_controller_spec.rb index 73fff6eb5ca..b7257fac608 100644 --- a/spec/controllers/autocomplete_controller_spec.rb +++ b/spec/controllers/autocomplete_controller_spec.rb @@ -109,15 +109,17 @@ describe AutocompleteController do end context 'limited users per page' do - let(:per_page) { 2 } - before do + 25.times do + create(:user) + end + sign_in(user) - get(:users, per_page: per_page) + get(:users) end it { expect(json_response).to be_kind_of(Array) } - it { expect(json_response.size).to eq(per_page) } + it { expect(json_response.size).to eq(20) } end context 'unauthenticated user' do diff --git a/spec/controllers/projects/clusters/gcp_controller_spec.rb b/spec/controllers/projects/clusters/gcp_controller_spec.rb index 775f9db1c6e..e14ba29fa70 100644 --- a/spec/controllers/projects/clusters/gcp_controller_spec.rb +++ b/spec/controllers/projects/clusters/gcp_controller_spec.rb @@ -161,7 +161,7 @@ describe Projects::Clusters::GcpController do it 'renders the cluster form with an error' do go - expect(response).to set_flash[:alert] + expect(response).to set_flash.now[:alert] expect(response).to render_template('new') end end diff --git a/spec/controllers/projects/pages_domains_controller_spec.rb b/spec/controllers/projects/pages_domains_controller_spec.rb index e9e7d357d9c..2192fd5cae2 100644 --- a/spec/controllers/projects/pages_domains_controller_spec.rb +++ b/spec/controllers/projects/pages_domains_controller_spec.rb @@ -46,7 +46,46 @@ describe Projects::PagesDomainsController do post(:create, request_params.merge(pages_domain: pages_domain_params)) end.to change { PagesDomain.count }.by(1) - expect(response).to redirect_to(project_pages_path(project)) + created_domain = PagesDomain.reorder(:id).last + + expect(created_domain).to be_present + expect(response).to redirect_to(project_pages_domain_path(project, created_domain)) + end + end + + describe 'POST verify' do + let(:params) { request_params.merge(id: pages_domain.domain) } + + def stub_service + service = double(:service) + + expect(VerifyPagesDomainService).to receive(:new) { service } + + service + end + + it 'handles verification success' do + expect(stub_service).to receive(:execute).and_return(status: :success) + + post :verify, params + + expect(response).to redirect_to project_pages_domain_path(project, pages_domain) + expect(flash[:notice]).to eq('Successfully verified domain ownership') + end + + it 'handles verification failure' do + expect(stub_service).to receive(:execute).and_return(status: :failed) + + post :verify, params + + expect(response).to redirect_to project_pages_domain_path(project, pages_domain) + expect(flash[:alert]).to eq('Failed to verify domain ownership') + end + + it 'returns a 404 response for an unknown domain' do + post :verify, request_params.merge(id: 'unknown-domain') + + expect(response).to have_gitlab_http_status(404) end end diff --git a/spec/controllers/projects/prometheus_controller_spec.rb b/spec/controllers/projects/prometheus/metrics_controller_spec.rb index bbfe78d305a..f17f819feee 100644 --- a/spec/controllers/projects/prometheus_controller_spec.rb +++ b/spec/controllers/projects/prometheus/metrics_controller_spec.rb @@ -1,20 +1,20 @@ -require('spec_helper') +require 'spec_helper' -describe Projects::PrometheusController do +describe Projects::Prometheus::MetricsController do let(:user) { create(:user) } - let!(:project) { create(:project) } + let(:project) { create(:project) } let(:prometheus_service) { double('prometheus_service') } before do allow(controller).to receive(:project).and_return(project) - allow(project).to receive(:prometheus_service).and_return(prometheus_service) + allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return(prometheus_service) project.add_master(user) sign_in(user) end - describe 'GET #active_metrics' do + describe 'GET #active_common' do context 'when prometheus metrics are enabled' do context 'when data is not present' do before do @@ -22,7 +22,7 @@ describe Projects::PrometheusController do end it 'returns no content response' do - get :active_metrics, project_params(format: :json) + get :active_common, project_params(format: :json) expect(response).to have_gitlab_http_status(204) end @@ -36,7 +36,7 @@ describe Projects::PrometheusController do end it 'returns no content response' do - get :active_metrics, project_params(format: :json) + get :active_common, project_params(format: :json) expect(response).to have_gitlab_http_status(200) expect(json_response).to eq(sample_response.deep_stringify_keys) @@ -45,7 +45,7 @@ describe Projects::PrometheusController do context 'when requesting non json response' do it 'returns not found response' do - get :active_metrics, project_params + get :active_common, project_params expect(response).to have_gitlab_http_status(404) end diff --git a/spec/factories/pages_domains.rb b/spec/factories/pages_domains.rb index 61b04708da2..35b44e1c52e 100644 --- a/spec/factories/pages_domains.rb +++ b/spec/factories/pages_domains.rb @@ -1,6 +1,25 @@ FactoryBot.define do factory :pages_domain, class: 'PagesDomain' do - domain 'my.domain.com' + sequence(:domain) { |n| "my#{n}.domain.com" } + verified_at { Time.now } + enabled_until { 1.week.from_now } + + trait :disabled do + verified_at nil + enabled_until nil + end + + trait :unverified do + verified_at nil + end + + trait :reverify do + enabled_until { 1.hour.from_now } + end + + trait :expired do + enabled_until { 1.hour.ago } + end trait :with_certificate do certificate '-----BEGIN CERTIFICATE----- diff --git a/spec/features/admin/admin_runners_spec.rb b/spec/features/admin/admin_runners_spec.rb index a01c129defd..7eeed7da998 100644 --- a/spec/features/admin/admin_runners_spec.rb +++ b/spec/features/admin/admin_runners_spec.rb @@ -19,7 +19,7 @@ describe "Admin Runners" do end it 'has all necessary texts' do - expect(page).to have_text "How to setup" + expect(page).to have_text "Setup a shared Runner manually" expect(page).to have_text "Runners with last contact more than a minute ago: 1" end @@ -54,7 +54,7 @@ describe "Admin Runners" do end it 'has all necessary texts including no runner message' do - expect(page).to have_text "How to setup" + expect(page).to have_text "Setup a shared Runner manually" expect(page).to have_text "Runners with last contact more than a minute ago: 0" expect(page).to have_text 'No runners found' end diff --git a/spec/features/issues/filtered_search/recent_searches_spec.rb b/spec/features/issues/filtered_search/recent_searches_spec.rb index f355cec3ba9..41b9ada988a 100644 --- a/spec/features/issues/filtered_search/recent_searches_spec.rb +++ b/spec/features/issues/filtered_search/recent_searches_spec.rb @@ -39,8 +39,8 @@ describe 'Recent searches', :js do items = all('.filtered-search-history-dropdown-item', visible: false, count: 2) - expect(items[0].text).to eq('label:~qux garply') - expect(items[1].text).to eq('label:~foo bar') + expect(items[0].text).to eq('label: ~qux garply') + expect(items[1].text).to eq('label: ~foo bar') end it 'saved recent searches are restored last on the list' do diff --git a/spec/features/projects/pages_spec.rb b/spec/features/projects/pages_spec.rb index 3f1ef0b2a47..a96f2c186a4 100644 --- a/spec/features/projects/pages_spec.rb +++ b/spec/features/projects/pages_spec.rb @@ -60,7 +60,6 @@ feature 'Pages' do fill_in 'Domain', with: 'my.test.domain.com' click_button 'Create New Domain' - expect(page).to have_content('Domains (1)') expect(page).to have_content('my.test.domain.com') end end @@ -159,7 +158,6 @@ feature 'Pages' do fill_in 'Key (PEM)', with: certificate_key click_button 'Create New Domain' - expect(page).to have_content('Domains (1)') expect(page).to have_content('my.test.domain.com') end end diff --git a/spec/features/projects_spec.rb b/spec/features/projects_spec.rb index 645d12da09f..cfe979a8647 100644 --- a/spec/features/projects_spec.rb +++ b/spec/features/projects_spec.rb @@ -146,8 +146,8 @@ feature 'Project' do end describe 'removal', :js do - let(:user) { create(:user, username: 'test', name: 'test') } - let(:project) { create(:project, namespace: user.namespace, name: 'project1') } + let(:user) { create(:user) } + let(:project) { create(:project, namespace: user.namespace) } before do sign_in(user) @@ -156,8 +156,8 @@ feature 'Project' do end it 'removes a project' do - expect { remove_with_confirm('Remove project', project.path) }.to change {Project.count}.by(-1) - expect(page).to have_content "Project 'test / project1' is in the process of being deleted." + expect { remove_with_confirm('Remove project', project.path) }.to change { Project.count }.by(-1) + expect(page).to have_content "Project '#{project.full_name}' is in the process of being deleted." expect(Project.all.count).to be_zero expect(project.issues).to be_empty expect(project.merge_requests).to be_empty diff --git a/spec/features/runners_spec.rb b/spec/features/runners_spec.rb index aec9de6c7ca..df65c2d2f83 100644 --- a/spec/features/runners_spec.rb +++ b/spec/features/runners_spec.rb @@ -7,6 +7,20 @@ feature 'Runners' do sign_in(user) end + context 'when user opens runners page' do + given(:project) { create(:project) } + + background do + project.add_master(user) + end + + scenario 'user can see a button to install runners on kubernetes clusters' do + visit runners_path(project) + + expect(page).to have_link('Install Runner on Kubernetes', href: project_clusters_path(project)) + end + end + context 'when a project has enabled shared_runners' do given(:project) { create(:project) } diff --git a/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json b/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json index e8c17298b43..ed8ed9085c0 100644 --- a/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json +++ b/spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json @@ -4,6 +4,9 @@ "domain": { "type": "string" }, "url": { "type": "uri" }, "project_id": { "type": "integer" }, + "verified": { "type": "boolean" }, + "verification_code": { "type": ["string", "null"] }, + "enabled_until": { "type": ["date", "null"] }, "certificate_expiration": { "type": "object", "properties": { @@ -14,6 +17,6 @@ "additionalProperties": false } }, - "required": ["domain", "url", "project_id"], + "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until"], "additionalProperties": false } diff --git a/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json b/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json index 08db8d47050..b57d544f896 100644 --- a/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json +++ b/spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json @@ -3,6 +3,9 @@ "properties": { "domain": { "type": "string" }, "url": { "type": "uri" }, + "verified": { "type": "boolean" }, + "verification_code": { "type": ["string", "null"] }, + "enabled_until": { "type": ["date", "null"] }, "certificate": { "type": "object", "properties": { @@ -15,6 +18,6 @@ "additionalProperties": false } }, - "required": ["domain", "url"], + "required": ["domain", "url", "verified", "verification_code", "enabled_until"], "additionalProperties": false } diff --git a/spec/helpers/issuables_helper_spec.rb b/spec/helpers/issuables_helper_spec.rb index 7fa665aecdc..2fecd1a3d27 100644 --- a/spec/helpers/issuables_helper_spec.rb +++ b/spec/helpers/issuables_helper_spec.rb @@ -173,23 +173,23 @@ describe IssuablesHelper do @project = issue.project expected_data = { - 'endpoint' => "/#{@project.full_path}/issues/#{issue.iid}", - 'updateEndpoint' => "/#{@project.full_path}/issues/#{issue.iid}.json", - 'canUpdate' => true, - 'canDestroy' => true, - 'issuableRef' => "##{issue.iid}", - 'markdownPreviewPath' => "/#{@project.full_path}/preview_markdown", - 'markdownDocsPath' => '/help/user/markdown', - 'issuableTemplates' => [], - 'projectPath' => @project.path, - 'projectNamespace' => @project.namespace.path, - 'initialTitleHtml' => issue.title, - 'initialTitleText' => issue.title, - 'initialDescriptionHtml' => '<p dir="auto">issue text</p>', - 'initialDescriptionText' => 'issue text', - 'initialTaskStatus' => '0 of 0 tasks completed' + endpoint: "/#{@project.full_path}/issues/#{issue.iid}", + updateEndpoint: "/#{@project.full_path}/issues/#{issue.iid}.json", + canUpdate: true, + canDestroy: true, + issuableRef: "##{issue.iid}", + markdownPreviewPath: "/#{@project.full_path}/preview_markdown", + markdownDocsPath: '/help/user/markdown', + issuableTemplates: [], + projectPath: @project.path, + projectNamespace: @project.namespace.path, + initialTitleHtml: issue.title, + initialTitleText: issue.title, + initialDescriptionHtml: '<p dir="auto">issue text</p>', + initialDescriptionText: 'issue text', + initialTaskStatus: '0 of 0 tasks completed' } - expect(JSON.parse(helper.issuable_initial_data(issue))).to eq(expected_data) + expect(helper.issuable_initial_data(issue)).to eq(expected_data) end end diff --git a/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js index 4a516c517ef..59bd2650081 100644 --- a/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js +++ b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js @@ -1,7 +1,6 @@ import Vue from 'vue'; import eventHub from '~/filtered_search/event_hub'; -import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content'; - +import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content.vue'; import FilteredSearchTokenKeys from '~/filtered_search/filtered_search_token_keys'; const createComponent = (propsData) => { diff --git a/spec/javascripts/test_bundle.js b/spec/javascripts/test_bundle.js index 323b8a9572d..94fcc6c7f2b 100644 --- a/spec/javascripts/test_bundle.js +++ b/spec/javascripts/test_bundle.js @@ -113,7 +113,9 @@ if (process.env.BABEL_ENV === 'coverage') { // exempt these files from the coverage report const troubleMakers = [ './blob_edit/blob_bundle.js', - './boards/boards_bundle.js', + './boards/components/modal/empty_state.js', + './boards/components/modal/footer.js', + './boards/components/modal/header.js', './cycle_analytics/cycle_analytics_bundle.js', './cycle_analytics/components/stage_plan_component.js', './cycle_analytics/components/stage_staging_component.js', diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 19d3f55501e..6f07e423c1b 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -534,6 +534,19 @@ describe Gitlab::GitAccess do expect { pull_access_check }.to raise_unauthorized('Your account has been blocked.') end + context 'when the project repository does not exist' do + it 'returns not found' do + project.add_guest(user) + repo = project.repository + FileUtils.rm_rf(repo.path) + + # Sanity check for rm_rf + expect(repo.exists?).to eq(false) + + expect { pull_access_check }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.') + end + end + describe 'without access to project' do context 'pull code' do it { expect { pull_access_check }.to raise_not_found } diff --git a/spec/lib/gitlab/git_access_wiki_spec.rb b/spec/lib/gitlab/git_access_wiki_spec.rb index 215f1ecc9c5..730ede99fc9 100644 --- a/spec/lib/gitlab/git_access_wiki_spec.rb +++ b/spec/lib/gitlab/git_access_wiki_spec.rb @@ -57,7 +57,7 @@ describe Gitlab::GitAccessWiki do # Sanity check for rm_rf expect(wiki_repo.exists?).to eq(false) - expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'A repository for this project does not exist yet.') + expect { subject }.to raise_error(Gitlab::GitAccess::NotFoundError, 'A repository for this project does not exist yet.') end end end diff --git a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb b/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb index 2b488101496..c95719eff1d 100644 --- a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb @@ -24,7 +24,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do context 'with one group where two metrics is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) allow(client).to receive(:label_values).and_return(metric_names) end @@ -70,7 +70,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do context 'with one group where only one metric is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) allow(client).to receive(:label_values).and_return('metric_a') end @@ -99,7 +99,7 @@ describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do let(:second_metric_group) { simple_metric_group(name: 'nameb', metrics: simple_metrics(added_metric_name: 'metric_c')) } before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group, second_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group, second_metric_group]) allow(client).to receive(:label_values).and_return('metric_c') end diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 5d86007f71f..4c3b8deefb9 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -19,41 +19,41 @@ describe Gitlab::PrometheusClient do # - execute_query: A query call shared_examples 'failure response' do context 'when request returns 400 with an error message' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 400, body: { error: 'bar!' }) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'bar!') + .to raise_error(Gitlab::PrometheusClient::Error, 'bar!') expect(req_stub).to have_been_requested end end context 'when request returns 400 without an error message' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 400) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'Bad data received') + .to raise_error(Gitlab::PrometheusClient::Error, 'Bad data received') expect(req_stub).to have_been_requested end end context 'when request returns 500' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 500, body: { message: 'FAIL!' }) expect { execute_query } - .to raise_error(Gitlab::PrometheusError, '500 - {"message":"FAIL!"}') + .to raise_error(Gitlab::PrometheusClient::Error, '500 - {"message":"FAIL!"}') expect(req_stub).to have_been_requested end end context 'when request returns non json data' do - it 'raises a Gitlab::PrometheusError error' do + it 'raises a Gitlab::PrometheusClient::Error error' do req_stub = stub_prometheus_request(query_url, status: 200, body: 'not json') expect { execute_query } - .to raise_error(Gitlab::PrometheusError, 'Parsing response failed') + .to raise_error(Gitlab::PrometheusClient::Error, 'Parsing response failed') expect(req_stub).to have_been_requested end end @@ -65,27 +65,27 @@ describe Gitlab::PrometheusClient do subject { described_class.new(RestClient::Resource.new(prometheus_url)) } context 'exceptions are raised' do - it 'raises a Gitlab::PrometheusError error when a SocketError is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a SocketError is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, SocketError) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "Can't connect to #{prometheus_url}") + .to raise_error(Gitlab::PrometheusClient::Error, "Can't connect to #{prometheus_url}") expect(req_stub).to have_been_requested end - it 'raises a Gitlab::PrometheusError error when a SSLError is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a SSLError is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, OpenSSL::SSL::SSLError) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "#{prometheus_url} contains invalid SSL data") + .to raise_error(Gitlab::PrometheusClient::Error, "#{prometheus_url} contains invalid SSL data") expect(req_stub).to have_been_requested end - it 'raises a Gitlab::PrometheusError error when a RestClient::Exception is rescued' do + it 'raises a Gitlab::PrometheusClient::Error error when a RestClient::Exception is rescued' do req_stub = stub_prometheus_request_with_exception(prometheus_url, RestClient::Exception) expect { subject.send(:get, '/', {}) } - .to raise_error(Gitlab::PrometheusError, "Network connection error") + .to raise_error(Gitlab::PrometheusClient::Error, "Network connection error") expect(req_stub).to have_been_requested end end diff --git a/spec/lib/gitlab/quick_actions/command_definition_spec.rb b/spec/lib/gitlab/quick_actions/command_definition_spec.rb index f44a562dc63..b03c1e23ca3 100644 --- a/spec/lib/gitlab/quick_actions/command_definition_spec.rb +++ b/spec/lib/gitlab/quick_actions/command_definition_spec.rb @@ -40,7 +40,7 @@ describe Gitlab::QuickActions::CommandDefinition do end describe "#available?" do - let(:opts) { { go: false } } + let(:opts) { OpenStruct.new(go: false) } context "when the command has a condition block" do before do @@ -78,7 +78,7 @@ describe Gitlab::QuickActions::CommandDefinition do it "doesn't execute the command" do expect(context).not_to receive(:instance_exec) - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -95,7 +95,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it "doesn't execute the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -109,7 +109,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -117,7 +117,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "executes the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be true end @@ -131,7 +131,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -139,7 +139,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "doesn't execute the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be false end @@ -153,7 +153,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is provided an argument" do it "executes the command" do - subject.execute(context, {}, true) + subject.execute(context, true) expect(context.run).to be true end @@ -161,7 +161,7 @@ describe Gitlab::QuickActions::CommandDefinition do context "when the command is not provided an argument" do it "executes the command" do - subject.execute(context, {}, nil) + subject.execute(context, nil) expect(context.run).to be true end @@ -175,7 +175,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'executes the command passing the parsed param' do - subject.execute(context, {}, 'something ') + subject.execute(context, 'something ') expect(context.received_arg).to eq('something') end @@ -192,7 +192,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'returns nil' do - result = subject.explain({}, {}, nil) + result = subject.explain({}, nil) expect(result).to be_nil end @@ -204,7 +204,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'returns this static string' do - result = subject.explain({}, {}, nil) + result = subject.explain({}, nil) expect(result).to eq 'Explanation' end @@ -216,7 +216,7 @@ describe Gitlab::QuickActions::CommandDefinition do end it 'invokes the proc' do - result = subject.explain({}, {}, 'explanation') + result = subject.explain({}, 'explanation') expect(result).to eq 'Dynamic explanation' end diff --git a/spec/lib/gitlab/quick_actions/dsl_spec.rb b/spec/lib/gitlab/quick_actions/dsl_spec.rb index ff59dc48bcb..067a30fd7e2 100644 --- a/spec/lib/gitlab/quick_actions/dsl_spec.rb +++ b/spec/lib/gitlab/quick_actions/dsl_spec.rb @@ -76,7 +76,7 @@ describe Gitlab::QuickActions::Dsl do expect(dynamic_description_def.name).to eq(:dynamic_description) expect(dynamic_description_def.aliases).to eq([]) - expect(dynamic_description_def.to_h(noteable: 'issue')[:description]).to eq('A dynamic description for ISSUE') + expect(dynamic_description_def.to_h(OpenStruct.new(noteable: 'issue'))[:description]).to eq('A dynamic description for ISSUE') expect(dynamic_description_def.explanation).to eq('') expect(dynamic_description_def.params).to eq(['The first argument', 'The second argument']) expect(dynamic_description_def.condition_block).to be_nil diff --git a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb deleted file mode 100644 index 8fdbbacd04d..00000000000 --- a/spec/lib/gitlab/sidekiq_middleware/memory_killer_spec.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'spec_helper' - -describe Gitlab::SidekiqMiddleware::MemoryKiller do - subject { described_class.new } - let(:pid) { 999 } - - let(:worker) { double(:worker, class: 'TestWorker') } - let(:job) { { 'jid' => 123 } } - let(:queue) { 'test_queue' } - - def run - thread = subject.call(worker, job, queue) { nil } - thread&.join - end - - before do - allow(subject).to receive(:get_rss).and_return(10.kilobytes) - allow(subject).to receive(:pid).and_return(pid) - end - - context 'when MAX_RSS is set to 0' do - before do - stub_const("#{described_class}::MAX_RSS", 0) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end - - context 'when MAX_RSS is exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 5.kilobytes) - end - - it 'sends the STP, TERM and KILL signals at expected times' do - expect(subject).to receive(:sleep).with(15 * 60).ordered - expect(Process).to receive(:kill).with('SIGSTP', pid).ordered - - expect(subject).to receive(:sleep).with(30).ordered - expect(Process).to receive(:kill).with('SIGTERM', pid).ordered - - expect(subject).to receive(:sleep).with(10).ordered - expect(Process).to receive(:kill).with('SIGKILL', pid).ordered - - run - end - end - - context 'when MAX_RSS is not exceeded' do - before do - stub_const("#{described_class}::MAX_RSS", 15.kilobytes) - end - - it 'does nothing' do - expect(subject).not_to receive(:sleep) - - run - end - end -end diff --git a/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb b/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb new file mode 100644 index 00000000000..0001795c3f0 --- /dev/null +++ b/spec/lib/gitlab/sidekiq_middleware/shutdown_spec.rb @@ -0,0 +1,88 @@ +require 'spec_helper' + +describe Gitlab::SidekiqMiddleware::Shutdown do + subject { described_class.new } + + let(:pid) { Process.pid } + let(:worker) { double(:worker, class: 'TestWorker') } + let(:job) { { 'jid' => 123 } } + let(:queue) { 'test_queue' } + let(:block) { proc { nil } } + + def run + subject.call(worker, job, queue) { block.call } + described_class.shutdown_thread&.join + end + + def pop_trace + subject.trace.pop(true) + end + + before do + allow(subject).to receive(:get_rss).and_return(10.kilobytes) + described_class.clear_shutdown_thread + end + + context 'when MAX_RSS is set to 0' do + before do + stub_const("#{described_class}::MAX_RSS", 0) + end + + it 'does nothing' do + expect(subject).not_to receive(:sleep) + + run + end + end + + def expect_shutdown_sequence + expect(pop_trace).to eq([:sleep, 15 * 60]) + expect(pop_trace).to eq([:kill, 'SIGTSTP', pid]) + + expect(pop_trace).to eq([:sleep, 30]) + expect(pop_trace).to eq([:kill, 'SIGTERM', pid]) + + expect(pop_trace).to eq([:sleep, 10]) + expect(pop_trace).to eq([:kill, 'SIGKILL', pid]) + end + + context 'when MAX_RSS is exceeded' do + before do + stub_const("#{described_class}::MAX_RSS", 5.kilobytes) + end + + it 'sends the TSTP, TERM and KILL signals at expected times' do + run + + expect_shutdown_sequence + end + end + + context 'when MAX_RSS is not exceeded' do + before do + stub_const("#{described_class}::MAX_RSS", 15.kilobytes) + end + + it 'does nothing' do + expect(subject).not_to receive(:sleep) + + run + end + end + + context 'when WantShutdown is raised' do + let(:block) { proc { raise described_class::WantShutdown } } + + it 'starts the shutdown sequence and re-raises the exception' do + expect { run }.to raise_exception(described_class::WantShutdown) + + # We can't expect 'run' to have joined on the shutdown thread, because + # it hit an exception. + shutdown_thread = described_class.shutdown_thread + expect(shutdown_thread).not_to be_nil + shutdown_thread.join + + expect_shutdown_sequence + end + end +end diff --git a/spec/lib/gitlab/sql/pattern_spec.rb b/spec/lib/gitlab/sql/pattern_spec.rb index ef51e3cc8df..5b5052de372 100644 --- a/spec/lib/gitlab/sql/pattern_spec.rb +++ b/spec/lib/gitlab/sql/pattern_spec.rb @@ -154,6 +154,12 @@ describe Gitlab::SQL::Pattern do it 'returns a single equality condition' do expect(fuzzy_arel_match.to_sql).to match(/title.*I?LIKE 'fo'/) end + + it 'uses LOWER instead of ILIKE when LOWER is enabled' do + rel = Issue.fuzzy_arel_match(:title, query, lower_exact_match: true) + + expect(rel.to_sql).to match(/LOWER\(.*title.*\).*=.*'fo'/) + end end context 'with two words both equal to 3 chars' do diff --git a/spec/mailers/emails/pages_domains_spec.rb b/spec/mailers/emails/pages_domains_spec.rb new file mode 100644 index 00000000000..fe428ea657d --- /dev/null +++ b/spec/mailers/emails/pages_domains_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' +require 'email_spec' + +describe Emails::PagesDomains do + include EmailSpec::Matchers + include_context 'gitlab email notification' + + set(:project) { create(:project) } + set(:domain) { create(:pages_domain, project: project) } + set(:user) { project.owner } + + shared_examples 'a pages domain email' do + it_behaves_like 'an email sent from GitLab' + it_behaves_like 'it should not have Gmail Actions links' + it_behaves_like 'a user cannot unsubscribe through footer link' + + it 'has the expected content' do + aggregate_failures do + is_expected.to have_subject(email_subject) + is_expected.to have_body_text(project.human_name) + is_expected.to have_body_text(domain.domain) + is_expected.to have_body_text domain.url + is_expected.to have_body_text project_pages_domain_url(project, domain) + is_expected.to have_body_text help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + end + end + end + + describe '#pages_domain_enabled_email' do + let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been enabled" } + + subject { Notify.pages_domain_enabled_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'has been enabled' } + end + + describe '#pages_domain_disabled_email' do + let(:email_subject) { "#{project.path} | GitLab Pages domain '#{domain.domain}' has been disabled" } + + subject { Notify.pages_domain_disabled_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'has been disabled' } + end + + describe '#pages_domain_verification_succeeded_email' do + let(:email_subject) { "#{project.path} | Verification succeeded for GitLab Pages domain '#{domain.domain}'" } + + subject { Notify.pages_domain_verification_succeeded_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it { is_expected.to have_body_text 'successfully verified' } + end + + describe '#pages_domain_verification_failed_email' do + let(:email_subject) { "#{project.path} | ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'" } + + subject { Notify.pages_domain_verification_failed_email(domain, user) } + + it_behaves_like 'a pages domain email' + + it 'says verification has failed and when the domain is enabled until' do + is_expected.to have_body_text 'Verification has failed' + is_expected.to have_body_text domain.enabled_until.strftime('%F %T') + end + end +end diff --git a/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb new file mode 100644 index 00000000000..afcaefa0591 --- /dev/null +++ b/spec/migrations/enqueue_verify_pages_domain_workers_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180216121030_enqueue_verify_pages_domain_workers') + +describe EnqueueVerifyPagesDomainWorkers, :sidekiq, :migration do + around do |example| + Sidekiq::Testing.fake! do + example.run + end + end + + describe '#up' do + it 'enqueues a verification worker for every domain' do + domains = 1.upto(3).map { |i| PagesDomain.create!(domain: "my#{i}.domain.com") } + + expect { migrate! }.to change(PagesDomainVerificationWorker.jobs, :size).by(3) + + enqueued_ids = PagesDomainVerificationWorker.jobs.map { |job| job['args'] } + expected_ids = domains.map { |domain| [domain.id] } + + expect(enqueued_ids).to match_array(expected_ids) + end + end +end diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index 9d12f96c642..95713d8b85b 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -1,6 +1,10 @@ require 'spec_helper' describe PagesDomain do + using RSpec::Parameterized::TableSyntax + + subject(:pages_domain) { described_class.new } + describe 'associations' do it { is_expected.to belong_to(:project) } end @@ -64,19 +68,51 @@ describe PagesDomain do end end + describe 'validations' do + it { is_expected.to validate_presence_of(:verification_code) } + end + + describe '#verification_code' do + subject { pages_domain.verification_code } + + it 'is set automatically with 128 bits of SecureRandom data' do + expect(SecureRandom).to receive(:hex).with(16) { 'verification code' } + + is_expected.to eq('verification code') + end + end + + describe '#keyed_verification_code' do + subject { pages_domain.keyed_verification_code } + + it { is_expected.to eq("gitlab-pages-verification-code=#{pages_domain.verification_code}") } + end + + describe '#verification_domain' do + subject { pages_domain.verification_domain } + + it { is_expected.to be_nil } + + it 'is a well-known subdomain if the domain is present' do + pages_domain.domain = 'example.com' + + is_expected.to eq('_gitlab-pages-verification-code.example.com') + end + end + describe '#url' do subject { domain.url } context 'without the certificate' do let(:domain) { build(:pages_domain, certificate: '') } - it { is_expected.to eq('http://my.domain.com') } + it { is_expected.to eq("http://#{domain.domain}") } end context 'with a certificate' do let(:domain) { build(:pages_domain, :with_certificate) } - it { is_expected.to eq('https://my.domain.com') } + it { is_expected.to eq("https://#{domain.domain}") } end end @@ -154,4 +190,108 @@ describe PagesDomain do # We test only existence of output, since the output is long it { is_expected.not_to be_empty } end + + describe '#update_daemon' do + it 'runs when the domain is created' do + domain = build(:pages_domain) + + expect(domain).to receive(:update_daemon) + + domain.save! + end + + it 'runs when the domain is destroyed' do + domain = create(:pages_domain) + + expect(domain).to receive(:update_daemon) + + domain.destroy! + end + + it 'delegates to Projects::UpdatePagesConfigurationService' do + service = instance_double('Projects::UpdatePagesConfigurationService') + expect(Projects::UpdatePagesConfigurationService).to receive(:new) { service } + expect(service).to receive(:execute) + + create(:pages_domain) + end + + context 'configuration updates when attributes change' do + set(:project1) { create(:project) } + set(:project2) { create(:project) } + set(:domain) { create(:pages_domain) } + + where(:attribute, :old_value, :new_value, :update_expected) do + now = Time.now + future = now + 1.day + + :project | nil | :project1 | true + :project | :project1 | :project1 | false + :project | :project1 | :project2 | true + :project | :project1 | nil | true + + # domain can't be set to nil + :domain | 'a.com' | 'a.com' | false + :domain | 'a.com' | 'b.com' | true + + # verification_code can't be set to nil + :verification_code | 'foo' | 'foo' | false + :verification_code | 'foo' | 'bar' | false + + :verified_at | nil | now | false + :verified_at | now | now | false + :verified_at | now | future | false + :verified_at | now | nil | false + + :enabled_until | nil | now | true + :enabled_until | now | now | false + :enabled_until | now | future | false + :enabled_until | now | nil | true + end + + with_them do + it 'runs if a relevant attribute has changed' do + a = old_value.is_a?(Symbol) ? send(old_value) : old_value + b = new_value.is_a?(Symbol) ? send(new_value) : new_value + + domain.update!(attribute => a) + + if update_expected + expect(domain).to receive(:update_daemon) + else + expect(domain).not_to receive(:update_daemon) + end + + domain.update!(attribute => b) + end + end + + context 'TLS configuration' do + set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) } + + let(:cert1) { domain_with_tls.certificate } + let(:cert2) { cert1 + ' ' } + let(:key1) { domain_with_tls.key } + let(:key2) { key1 + ' ' } + + it 'updates when added' do + expect(domain).to receive(:update_daemon) + + domain.update!(key: key1, certificate: cert1) + end + + it 'updates when changed' do + expect(domain_with_tls).to receive(:update_daemon) + + domain_with_tls.update!(key: key2, certificate: cert2) + end + + it 'updates when removed' do + expect(domain_with_tls).to receive(:update_daemon) + + domain_with_tls.update!(key: nil, certificate: nil) + end + end + end + end end diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index ed17e019d42..6693e5783a5 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -223,8 +223,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do context 'with cluster for all environments without prometheus installed' do context 'without environment supplied' do - it 'raises PrometheusError because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/) + it 'raises PrometheusClient::Error because cluster was not found' do + expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) end end @@ -242,8 +242,8 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do context 'with prod environment supplied' do let!(:environment) { create(:environment, project: project, name: 'prod') } - it 'raises PrometheusError because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/) + it 'raises PrometheusClient::Error because cluster was not found' do + expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) end end end diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index c6fdda203ad..1b0a5eac9b0 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -597,7 +597,7 @@ describe 'Git HTTP requests' do context "when a gitlab ci token is provided" do let(:project) { create(:project, :repository) } let(:build) { create(:ci_build, :running) } - let(:other_project) { create(:project) } + let(:other_project) { create(:project, :repository) } before do build.update!(project: project) # can't associate it on factory create @@ -648,10 +648,10 @@ describe 'Git HTTP requests' do context 'when the repo does not exist' do let(:project) { create(:project) } - it 'rejects pulls with 403 Forbidden' do + it 'rejects pulls with 404 Not Found' do clone_get path, env - expect(response).to have_gitlab_http_status(:forbidden) + expect(response).to have_gitlab_http_status(:not_found) expect(response.body).to eq(git_access_error(:no_repo)) end end diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index de829011e58..6bed8e812c0 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -68,10 +68,10 @@ describe 'OpenID Connect requests' do let!(:public_email) { build :email, email: 'public@example.com' } let!(:private_email) { build :email, email: 'private@example.com' } - let!(:group1) { create :group, path: 'group1' } - let!(:group2) { create :group, path: 'group2' } - let!(:group3) { create :group, path: 'group3', parent: group2 } - let!(:group4) { create :group, path: 'group4', parent: group3 } + let!(:group1) { create :group } + let!(:group2) { create :group } + let!(:group3) { create :group, parent: group2 } + let!(:group4) { create :group, parent: group3 } before do group1.add_user(user, GroupMember::OWNER) @@ -93,8 +93,8 @@ describe 'OpenID Connect requests' do 'groups' => anything })) - expected_groups = %w[group1 group2/group3] - expected_groups << 'group2/group3/group4' if Group.supports_nested_groups? + expected_groups = [group1.full_path, group3.full_path] + expected_groups << group4.full_path if Group.supports_nested_groups? expect(json_response['groups']).to match_array(expected_groups) end end diff --git a/spec/services/ci/create_trace_artifact_service_spec.rb b/spec/services/ci/create_trace_artifact_service_spec.rb index 847a88920fe..8c5e8e438c7 100644 --- a/spec/services/ci/create_trace_artifact_service_spec.rb +++ b/spec/services/ci/create_trace_artifact_service_spec.rb @@ -4,40 +4,60 @@ describe Ci::CreateTraceArtifactService do describe '#execute' do subject { described_class.new(nil, nil).execute(job) } - let(:job) { create(:ci_build) } - context 'when the job does not have trace artifact' do context 'when the job has a trace file' do - before do - allow_any_instance_of(Gitlab::Ci::Trace) - .to receive(:default_path) { expand_fixture_path('trace/sample_trace') } + let!(:job) { create(:ci_build, :trace_live) } + let!(:legacy_path) { job.trace.read { |stream| return stream.path } } + let!(:legacy_checksum) { Digest::SHA256.file(legacy_path).hexdigest } + let(:new_path) { job.job_artifacts_trace.file.path } + let(:new_checksum) { Digest::SHA256.file(new_path).hexdigest } - allow_any_instance_of(JobArtifactUploader).to receive(:move_to_cache) { false } - allow_any_instance_of(JobArtifactUploader).to receive(:move_to_store) { false } - end + it { expect(File.exist?(legacy_path)).to be_truthy } it 'creates trace artifact' do expect { subject }.to change { Ci::JobArtifact.count }.by(1) - expect(job.job_artifacts_trace.read_attribute(:file)).to eq('sample_trace') + expect(File.exist?(legacy_path)).to be_falsy + expect(File.exist?(new_path)).to be_truthy + expect(new_checksum).to eq(legacy_checksum) + expect(job.job_artifacts_trace.file.exists?).to be_truthy + expect(job.job_artifacts_trace.file.filename).to eq('job.log') end - context 'when the job has already had trace artifact' do + context 'when failed to create trace artifact record' do before do - create(:ci_job_artifact, :trace, job: job) + # When ActiveRecord error happens + allow_any_instance_of(Ci::JobArtifact).to receive(:save).and_return(false) + allow_any_instance_of(Ci::JobArtifact).to receive_message_chain(:errors, :full_messages) + .and_return("Error") + + subject rescue nil + + job.reload end - it 'does not create trace artifact' do - expect { subject }.not_to change { Ci::JobArtifact.count } + it 'keeps legacy trace and removes trace artifact' do + expect(File.exist?(legacy_path)).to be_truthy + expect(job.job_artifacts_trace).to be_nil end end end context 'when the job does not have a trace file' do + let!(:job) { create(:ci_build) } + it 'does not create trace artifact' do expect { subject }.not_to change { Ci::JobArtifact.count } end end end + + context 'when the job has already had trace artifact' do + let!(:job) { create(:ci_build, :trace_artifact) } + + it 'does not create trace artifact' do + expect { subject }.not_to change { Ci::JobArtifact.count } + end + end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index 836ffb7cea0..62fdf870090 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -1678,6 +1678,78 @@ describe NotificationService, :mailer do end end + describe 'Pages domains' do + set(:project) { create(:project) } + set(:domain) { create(:pages_domain, project: project) } + set(:u_blocked) { create(:user, :blocked) } + set(:u_silence) { create_user_with_notification(:disabled, 'silent', project) } + set(:u_owner) { project.owner } + set(:u_master1) { create(:user) } + set(:u_master2) { create(:user) } + set(:u_developer) { create(:user) } + + before do + project.add_master(u_blocked) + project.add_master(u_silence) + project.add_master(u_master1) + project.add_master(u_master2) + project.add_developer(u_developer) + + reset_delivered_emails! + end + + %i[ + pages_domain_enabled + pages_domain_disabled + pages_domain_verification_succeeded + pages_domain_verification_failed + ].each do |sym| + describe "##{sym}" do + subject(:notify!) { notification.send(sym, domain) } + + it 'emails current watching masters' do + expect(Notify).to receive(:"#{sym}_email").at_least(:once).and_call_original + + notify! + + should_only_email(u_master1, u_master2, u_owner) + end + + it 'emails nobody if the project is missing' do + domain.project = nil + + notify! + + should_not_email_anyone + end + end + end + + describe '#pages_domain_verification_failed' do + it 'emails current watching masters' do + notification.pages_domain_verification_failed(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + + describe '#pages_domain_enabled' do + it 'emails current watching masters' do + notification.pages_domain_enabled(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + + describe '#pages_domain_disabled' do + it 'emails current watching masters' do + notification.pages_domain_disabled(domain) + + should_only_email(u_master1, u_master2, u_owner) + end + end + end + def build_team(project) @u_watcher = create_global_setting_for(create(:user), :watch) @u_participating = create_global_setting_for(create(:user), :participating) diff --git a/spec/services/quick_actions/interpret_service_spec.rb b/spec/services/quick_actions/interpret_service_spec.rb index ae160d104f1..f793f55e51b 100644 --- a/spec/services/quick_actions/interpret_service_spec.rb +++ b/spec/services/quick_actions/interpret_service_spec.rb @@ -522,6 +522,22 @@ describe QuickActions::InterpretService do let(:issuable) { merge_request } end + context 'only group milestones available' do + let(:group) { create(:group) } + let(:project) { create(:project, :public, namespace: group) } + let(:milestone) { create(:milestone, group: group, title: '10.0') } + + it_behaves_like 'milestone command' do + let(:content) { "/milestone %#{milestone.title}" } + let(:issuable) { issue } + end + + it_behaves_like 'milestone command' do + let(:content) { "/milestone %#{milestone.title}" } + let(:issuable) { merge_request } + end + end + it_behaves_like 'remove_milestone command' do let(:content) { '/remove_milestone' } let(:issuable) { issue } diff --git a/spec/services/verify_pages_domain_service_spec.rb b/spec/services/verify_pages_domain_service_spec.rb new file mode 100644 index 00000000000..576db1dde2d --- /dev/null +++ b/spec/services/verify_pages_domain_service_spec.rb @@ -0,0 +1,270 @@ +require 'spec_helper' + +describe VerifyPagesDomainService do + using RSpec::Parameterized::TableSyntax + include EmailHelpers + + let(:error_status) { { status: :error, message: "Couldn't verify #{domain.domain}" } } + + subject(:service) { described_class.new(domain) } + + describe '#execute' do + context 'verification code recognition (verified domain)' do + where(:domain_sym, :code_sym) do + :domain | :verification_code + :domain | :keyed_verification_code + + :verification_domain | :verification_code + :verification_domain | :keyed_verification_code + end + + with_them do + set(:domain) { create(:pages_domain) } + + let(:domain_name) { domain.send(domain_sym) } + let(:verification_code) { domain.send(code_sym) } + + it 'verifies and enables the domain' do + stub_resolver(domain_name => ['something else', verification_code]) + + expect(service.execute).to eq(status: :success) + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'verifies and enables when the code is contained partway through a TXT record' do + stub_resolver(domain_name => "something #{verification_code} else") + + expect(service.execute).to eq(status: :success) + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'does not verify when the code is not present' do + stub_resolver(domain_name => 'something else') + + expect(service.execute).to eq(error_status) + + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + end + + context 'verified domain' do + set(:domain) { create(:pages_domain) } + + it 'unverifies (but does not disable) when the right code is not present' do + stub_resolver(domain.domain => 'something else') + + expect(service.execute).to eq(error_status) + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + + it 'unverifies (but does not disable) when no records are present' do + stub_resolver + + expect(service.execute).to eq(error_status) + expect(domain).not_to be_verified + expect(domain).to be_enabled + end + end + + context 'expired domain' do + set(:domain) { create(:pages_domain, :expired) } + + it 'verifies and enables when the right code is present' do + stub_resolver(domain.domain => domain.keyed_verification_code) + + expect(service.execute).to eq(status: :success) + + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'disables when the right code is not present' do + error_status[:message] += '. It is now disabled.' + + stub_resolver + + expect(service.execute).to eq(error_status) + + expect(domain).not_to be_verified + expect(domain).not_to be_enabled + end + end + end + + context 'timeout behaviour' do + let(:domain) { create(:pages_domain) } + + it 'sets a timeout on the DNS query' do + expect(stub_resolver).to receive(:timeouts=).with(described_class::RESOLVER_TIMEOUT_SECONDS) + + service.execute + end + end + + context 'email notifications' do + let(:notification_service) { instance_double('NotificationService') } + + where(:factory, :verification_succeeds, :expected_notification) do + nil | true | nil + nil | false | :verification_failed + :reverify | true | nil + :reverify | false | :verification_failed + :unverified | true | :verification_succeeded + :unverified | false | nil + :expired | true | nil + :expired | false | :disabled + :disabled | true | :enabled + :disabled | false | nil + end + + with_them do + let(:domain) { create(:pages_domain, *[factory].compact) } + + before do + allow(service).to receive(:notification_service) { notification_service } + + if verification_succeeds + stub_resolver(domain.domain => domain.verification_code) + else + stub_resolver + end + end + + it 'sends a notification if appropriate' do + if expected_notification + expect(notification_service).to receive(:"pages_domain_#{expected_notification}").with(domain) + end + + service.execute + end + end + + context 'pages verification disabled' do + let(:domain) { create(:pages_domain, :disabled) } + + before do + stub_application_setting(pages_domain_verification_enabled: false) + allow(service).to receive(:notification_service) { notification_service } + end + + it 'skips email notifications' do + expect(notification_service).not_to receive(:pages_domain_enabled) + + service.execute + end + end + end + + context 'pages configuration updates' do + context 'enabling a disabled domain' do + let(:domain) { create(:pages_domain, :disabled) } + + it 'schedules an update' do + stub_resolver(domain.domain => domain.verification_code) + + expect(domain).to receive(:update_daemon) + + service.execute + end + end + + context 'verifying an enabled domain' do + let(:domain) { create(:pages_domain) } + + it 'schedules an update' do + stub_resolver(domain.domain => domain.verification_code) + + expect(domain).not_to receive(:update_daemon) + + service.execute + end + end + + context 'disabling an expired domain' do + let(:domain) { create(:pages_domain, :expired) } + + it 'schedules an update' do + stub_resolver + + expect(domain).to receive(:update_daemon) + + service.execute + end + end + + context 'failing to verify a disabled domain' do + let(:domain) { create(:pages_domain, :disabled) } + + it 'does not schedule an update' do + stub_resolver + + expect(domain).not_to receive(:update_daemon) + + service.execute + end + end + end + + context 'no verification code' do + let(:domain) { create(:pages_domain) } + + it 'returns an error' do + domain.verification_code = '' + + disallow_resolver! + + expect(service.execute).to eq(status: :error, message: "No verification code set for #{domain.domain}") + end + end + + context 'pages domain verification is disabled' do + let(:domain) { create(:pages_domain, :disabled) } + + before do + stub_application_setting(pages_domain_verification_enabled: false) + end + + it 'extends domain validity by unconditionally reverifying' do + disallow_resolver! + + service.execute + + expect(domain).to be_verified + expect(domain).to be_enabled + end + + it 'does not shorten any grace period' do + grace = Time.now + 1.year + domain.update!(enabled_until: grace) + disallow_resolver! + + service.execute + + expect(domain.enabled_until).to be_like_time(grace) + end + end + end + + def disallow_resolver! + expect(Resolv::DNS).not_to receive(:open) + end + + def stub_resolver(stubbed_lookups = {}) + resolver = instance_double('Resolv::DNS') + allow(resolver).to receive(:timeouts=) + + expect(Resolv::DNS).to receive(:open).and_yield(resolver) + + allow(resolver).to receive(:getresources) { [] } + stubbed_lookups.each do |domain, records| + records = Array(records).map { |txt| Resolv::DNS::Resource::IN::TXT.new(txt) } + allow(resolver).to receive(:getresources).with(domain, Resolv::DNS::Resource::IN::TXT) { records } + end + + resolver + end +end diff --git a/spec/support/capybara.rb b/spec/support/capybara.rb index 5189c57b7db..8603b7f3e2c 100644 --- a/spec/support/capybara.rb +++ b/spec/support/capybara.rb @@ -78,8 +78,10 @@ RSpec.configure do |config| end config.after(:example, :js) do |example| - # prevent localstorage from introducing side effects based on test order - execute_script("localStorage.clear();") + # prevent localStorage from introducing side effects based on test order + unless ['', 'about:blank', 'data:,'].include? Capybara.current_session.driver.browser.current_url + execute_script("localStorage.clear();") + end # capybara/rspec already calls Capybara.reset_sessions! in an `after` hook, # but `block_and_wait_for_requests_complete` is called before it so by diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 1809ae1d141..5edc5de2a09 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -28,11 +28,11 @@ RSpec.configure do |config| end config.before(:each, :js) do - DatabaseCleaner.strategy = :deletion + DatabaseCleaner.strategy = :deletion, { cache_tables: false } end config.before(:each, :delete) do - DatabaseCleaner.strategy = :deletion + DatabaseCleaner.strategy = :deletion, { cache_tables: false } end config.before(:each, :migration) do diff --git a/spec/support/prometheus/additional_metrics_shared_examples.rb b/spec/support/prometheus/additional_metrics_shared_examples.rb index dbbd4ad4d40..c7c3346d39e 100644 --- a/spec/support/prometheus/additional_metrics_shared_examples.rb +++ b/spec/support/prometheus/additional_metrics_shared_examples.rb @@ -12,11 +12,12 @@ RSpec.shared_examples 'additional metrics query' do let(:client) { double('prometheus_client') } let(:query_result) { described_class.new(client).query(*query_params) } - let(:environment) { create(:environment, slug: 'environment-slug') } + let(:project) { create(:project) } + let(:environment) { create(:environment, slug: 'environment-slug', project: project) } before do allow(client).to receive(:label_values).and_return(metric_names) - allow(metric_group_class).to receive(:all).and_return([simple_metric_group(metrics: [simple_metric])]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group(metrics: [simple_metric])]) end context 'metrics query context' do @@ -24,13 +25,14 @@ RSpec.shared_examples 'additional metrics query' do shared_examples 'query context containing environment slug and filter' do it 'contains ci_environment_slug' do - expect(subject).to receive(:query_metrics).with(hash_including(ci_environment_slug: environment.slug)) + expect(subject).to receive(:query_metrics).with(project, hash_including(ci_environment_slug: environment.slug)) subject.query(*query_params) end it 'contains environment filter' do expect(subject).to receive(:query_metrics).with( + project, hash_including( environment_filter: "container_name!=\"POD\",environment=\"#{environment.slug}\"" ) @@ -48,7 +50,7 @@ RSpec.shared_examples 'additional metrics query' do it_behaves_like 'query context containing environment slug and filter' it 'query context contains kube_namespace' do - expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: kube_namespace)) + expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: kube_namespace)) subject.query(*query_params) end @@ -72,7 +74,7 @@ RSpec.shared_examples 'additional metrics query' do it_behaves_like 'query context containing environment slug and filter' it 'query context contains empty kube_namespace' do - expect(subject).to receive(:query_metrics).with(hash_including(kube_namespace: '')) + expect(subject).to receive(:query_metrics).with(project, hash_including(kube_namespace: '')) subject.query(*query_params) end @@ -81,7 +83,7 @@ RSpec.shared_examples 'additional metrics query' do context 'with one group where two metrics is found' do before do - allow(metric_group_class).to receive(:all).and_return([simple_metric_group]) + allow(metric_group_class).to receive(:common_metrics).and_return([simple_metric_group]) end context 'some queries return results' do @@ -117,7 +119,7 @@ RSpec.shared_examples 'additional metrics query' do let(:metrics) { [simple_metric(queries: [simple_query])] } before do - allow(metric_group_class).to receive(:all).and_return( + allow(metric_group_class).to receive(:common_metrics).and_return( [ simple_metric_group(name: 'group_a', metrics: [simple_metric(queries: [simple_query])]), simple_metric_group(name: 'group_b', metrics: [simple_metric(title: 'title_b', queries: [simple_query('b')])]) diff --git a/spec/workers/authorized_projects_worker_spec.rb b/spec/workers/authorized_projects_worker_spec.rb index 0d6eb536c33..d095138f6b7 100644 --- a/spec/workers/authorized_projects_worker_spec.rb +++ b/spec/workers/authorized_projects_worker_spec.rb @@ -1,79 +1,6 @@ require 'spec_helper' describe AuthorizedProjectsWorker do - let(:project) { create(:project) } - - def build_args_list(*ids, multiply: 1) - args_list = ids.map { |id| [id] } - args_list * multiply - end - - describe '.bulk_perform_and_wait' do - it 'schedules the ids and waits for the jobs to complete' do - args_list = build_args_list(project.owner.id) - - project.owner.project_authorizations.delete_all - described_class.bulk_perform_and_wait(args_list) - - expect(project.owner.project_authorizations.count).to eq(1) - end - - it 'inlines workloads <= 3 jobs' do - args_list = build_args_list(project.owner.id, multiply: 3) - expect(described_class).to receive(:bulk_perform_inline).with(args_list) - - described_class.bulk_perform_and_wait(args_list) - end - - it 'runs > 3 jobs using sidekiq' do - project.owner.project_authorizations.delete_all - - expect(described_class).to receive(:bulk_perform_async).and_call_original - - args_list = build_args_list(project.owner.id, multiply: 4) - described_class.bulk_perform_and_wait(args_list) - - expect(project.owner.project_authorizations.count).to eq(1) - end - end - - describe '.bulk_perform_inline' do - it 'refreshes the authorizations inline' do - project.owner.project_authorizations.delete_all - - expect_any_instance_of(described_class).to receive(:perform).and_call_original - - described_class.bulk_perform_inline(build_args_list(project.owner.id)) - - expect(project.owner.project_authorizations.count).to eq(1) - end - - it 'enqueues jobs if an error is raised' do - invalid_id = -1 - args_list = build_args_list(project.owner.id, invalid_id) - - allow_any_instance_of(described_class).to receive(:perform).with(project.owner.id) - allow_any_instance_of(described_class).to receive(:perform).with(invalid_id).and_raise(ArgumentError) - expect(described_class).to receive(:bulk_perform_async).with(build_args_list(invalid_id)) - - described_class.bulk_perform_inline(args_list) - end - end - - describe '.bulk_perform_async' do - it "uses it's respective sidekiq queue" do - args_list = build_args_list(project.owner.id) - push_bulk_args = { - 'class' => described_class, - 'args' => args_list - } - - expect(Sidekiq::Client).to receive(:push_bulk).with(push_bulk_args).once - - described_class.bulk_perform_async(args_list) - end - end - describe '#perform' do let(:user) { create(:user) } @@ -85,12 +12,6 @@ describe AuthorizedProjectsWorker do job.perform(user.id) end - it 'notifies the JobWaiter when done if the key is provided' do - expect(Gitlab::JobWaiter).to receive(:notify).with('notify-key', job.jid) - - job.perform(user.id, 'notify-key') - end - context "when the user is not found" do it "does nothing" do expect_any_instance_of(User).not_to receive(:refresh_authorized_projects) diff --git a/spec/workers/concerns/waitable_worker_spec.rb b/spec/workers/concerns/waitable_worker_spec.rb new file mode 100644 index 00000000000..4af0de86ac9 --- /dev/null +++ b/spec/workers/concerns/waitable_worker_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' + +describe WaitableWorker do + let(:worker) do + Class.new do + def self.name + 'Gitlab::Foo::Bar::DummyWorker' + end + + class << self + cattr_accessor(:counter) { 0 } + end + + include ApplicationWorker + prepend WaitableWorker + + def perform(i = 0) + self.class.counter += i + end + end + end + + subject(:job) { worker.new } + + describe '.bulk_perform_and_wait' do + it 'schedules the jobs and waits for them to complete' do + worker.bulk_perform_and_wait([[1], [2]]) + + expect(worker.counter).to eq(3) + end + + it 'inlines workloads <= 3 jobs' do + args_list = [[1], [2], [3]] + expect(worker).to receive(:bulk_perform_inline).with(args_list).and_call_original + + worker.bulk_perform_and_wait(args_list) + + expect(worker.counter).to eq(6) + end + + it 'runs > 3 jobs using sidekiq' do + expect(worker).to receive(:bulk_perform_async) + + worker.bulk_perform_and_wait([[1], [2], [3], [4]]) + end + end + + describe '.bulk_perform_inline' do + it 'runs the jobs inline' do + expect(worker).not_to receive(:bulk_perform_async) + + worker.bulk_perform_inline([[1], [2]]) + + expect(worker.counter).to eq(3) + end + + it 'enqueues jobs if an error is raised' do + expect(worker).to receive(:bulk_perform_async).with([['foo']]) + + worker.bulk_perform_inline([[1], ['foo']]) + end + end + + describe '#perform' do + shared_examples 'perform' do + it 'notifies the JobWaiter when done if the key is provided' do + key = Gitlab::JobWaiter.new.key + expect(Gitlab::JobWaiter).to receive(:notify).with(key, job.jid) + + job.perform(*args, key) + end + + it 'does not notify the JobWaiter when done if no key is provided' do + expect(Gitlab::JobWaiter).not_to receive(:notify) + + job.perform(*args) + end + end + + context 'when the worker takes arguments' do + let(:args) { [1] } + + it_behaves_like 'perform' + end + + context 'when the worker takes no arguments' do + let(:args) { [] } + + it_behaves_like 'perform' + end + end +end diff --git a/spec/workers/pages_domain_verification_cron_worker_spec.rb b/spec/workers/pages_domain_verification_cron_worker_spec.rb new file mode 100644 index 00000000000..8f780428c82 --- /dev/null +++ b/spec/workers/pages_domain_verification_cron_worker_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe PagesDomainVerificationCronWorker do + subject(:worker) { described_class.new } + + describe '#perform' do + it 'enqueues a PagesDomainVerificationWorker for domains needing verification' do + verified = create(:pages_domain) + reverify = create(:pages_domain, :reverify) + disabled = create(:pages_domain, :disabled) + + [reverify, disabled].each do |domain| + expect(PagesDomainVerificationWorker).to receive(:perform_async).with(domain.id) + end + + expect(PagesDomainVerificationWorker).not_to receive(:perform_async).with(verified.id) + + worker.perform + end + end +end diff --git a/spec/workers/pages_domain_verification_worker_spec.rb b/spec/workers/pages_domain_verification_worker_spec.rb new file mode 100644 index 00000000000..372fc95ab4a --- /dev/null +++ b/spec/workers/pages_domain_verification_worker_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe PagesDomainVerificationWorker do + subject(:worker) { described_class.new } + + let(:domain) { create(:pages_domain) } + + describe '#perform' do + it 'does nothing for a non-existent domain' do + domain.destroy + + expect(VerifyPagesDomainService).not_to receive(:new) + + expect { worker.perform(domain.id) }.not_to raise_error + end + + it 'delegates to VerifyPagesDomainService' do + service = double(:service) + expected_domain = satisfy { |obj| obj == domain } + + expect(VerifyPagesDomainService).to receive(:new).with(expected_domain) { service } + expect(service).to receive(:execute) + + worker.perform(domain.id) + end + end +end diff --git a/spec/workers/stuck_import_jobs_worker_spec.rb b/spec/workers/stuck_import_jobs_worker_spec.rb index a82eb54ffe4..069514552b1 100644 --- a/spec/workers/stuck_import_jobs_worker_spec.rb +++ b/spec/workers/stuck_import_jobs_worker_spec.rb @@ -2,35 +2,59 @@ require 'spec_helper' describe StuckImportJobsWorker do let(:worker) { described_class.new } - let(:exclusive_lease_uuid) { SecureRandom.uuid } - before do - allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid) - end + shared_examples 'project import job detection' do + context 'when the job has completed' do + context 'when the import status was already updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids) do + project.import_start + project.import_finish - describe 'with started import_status' do - let(:project) { create(:project, :import_started, import_jid: '123') } + [project.import_jid] + end + end + + it 'does not mark the project as failed' do + worker.perform + + expect(project.reload.import_status).to eq('finished') + end + end + + context 'when the import status was not updated' do + before do + allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([project.import_jid]) + end - describe 'long running import' do - it 'marks the project as failed' do - allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return(['123']) + it 'marks the project as failed' do + worker.perform - expect { worker.perform }.to change { project.reload.import_status }.to('failed') + expect(project.reload.import_status).to eq('failed') + end end end - describe 'running import' do - it 'does not mark the project as failed' do + context 'when the job is still in Sidekiq' do + before do allow(Gitlab::SidekiqStatus).to receive(:completed_jids).and_return([]) + end + it 'does not mark the project as failed' do expect { worker.perform }.not_to change { project.reload.import_status } end + end + end - describe 'import without import_jid' do - it 'marks the project as failed' do - expect { worker.perform }.to change { project.reload.import_status }.to('failed') - end - end + describe 'with scheduled import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_scheduled, import_jid: '123') } + end + end + + describe 'with started import_status' do + it_behaves_like 'project import job detection' do + let(:project) { create(:project, :import_started, import_jid: '123') } end end end |
