diff options
64 files changed, 902 insertions, 128 deletions
diff --git a/README.md b/README.md index b6e1cc9a432..96d8f709274 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ GitLab Community Edition (CE) is available freely under the MIT Expat license. All third party components incorporated into the GitLab Software are licensed under the original license provided by the owner of the applicable component. -All Documentation content that resides under the doc/ directory of this repository is licensed under Creative Commons: CC BY-SA 4.0. +All Documentation content that resides under the `doc/` directory of this repository is licensed under Creative Commons: CC BY-SA 4.0. ## Install a development environment diff --git a/app/assets/stylesheets/framework/awards.scss b/app/assets/stylesheets/framework/awards.scss index 8d11b92cf88..a265e4206f1 100644 --- a/app/assets/stylesheets/framework/awards.scss +++ b/app/assets/stylesheets/framework/awards.scss @@ -141,8 +141,8 @@ &:hover, &:active, &.is-active { - background-color: $row-hover; - border-color: $row-hover-border; + background-color: $blue-50; + border-color: $blue-200; box-shadow: none; outline: 0; diff --git a/app/assets/stylesheets/framework/common.scss b/app/assets/stylesheets/framework/common.scss index 48a87ea8616..79ca6e61e9a 100644 --- a/app/assets/stylesheets/framework/common.scss +++ b/app/assets/stylesheets/framework/common.scss @@ -71,7 +71,7 @@ pre { } &.card.card-body-pre { - border: 1px solid $well-pre-bg; + border: 1px solid $gray-darker; background: $gray-light; border-radius: 0; color: $well-pre-color; @@ -233,7 +233,7 @@ li.note { .error-message { padding: 10px; - background: $error-bg; + background: $red-400; margin: 0; color: $white-light; @@ -244,11 +244,11 @@ li.note { } .warning_message { - border-left: 4px solid $warning-message-border; - color: $warning-message-color; + border-left: 4px solid $orange-200; + color: $orange-700; padding: 10px; margin-bottom: 10px; - background: $warning-message-bg; + background: $orange-100; padding-left: 20px; &.centered { @@ -348,15 +348,6 @@ img.emoji { } } -.profiler-results { - top: 73px !important; - - .profiler-button, - .profiler-controls { - border-color: $profiler-border !important; - } -} - .dropzone .dz-preview .dz-progress { border-color: $border-color !important; diff --git a/app/assets/stylesheets/framework/files.scss b/app/assets/stylesheets/framework/files.scss index 54882633fea..3bdf5bfc93a 100644 --- a/app/assets/stylesheets/framework/files.scss +++ b/app/assets/stylesheets/framework/files.scss @@ -112,7 +112,7 @@ &.image_file, &.video { - background: $file-image-bg; + background: $gray-darker; text-align: center; padding: 30px; @@ -131,7 +131,7 @@ } &.blob-no-preview { - background: $blob-bg; + background: $gray-darker; text-shadow: 0 1px 2px $white-light; padding: 100px 0; } @@ -146,7 +146,7 @@ } tr { - border-bottom: 1px solid $blame-border; + border-bottom: 1px solid $gray-darker; &:last-child { border-bottom: 0; @@ -211,7 +211,7 @@ } &.logs { - background: $logs-bg; + background: $gray-darker; max-height: 700px; overflow-y: auto; @@ -233,7 +233,7 @@ } &:hover { - background: $row-hover; + background: $blue-50; } } } diff --git a/app/assets/stylesheets/framework/issue_box.scss b/app/assets/stylesheets/framework/issue_box.scss index 86de88729ee..da5f80d9d37 100644 --- a/app/assets/stylesheets/framework/issue_box.scss +++ b/app/assets/stylesheets/framework/issue_box.scss @@ -39,7 +39,7 @@ } &.status-box-expired { - background-color: $issue-status-expired; + background-color: $orange-500; } &.status-box-upcoming { diff --git a/app/assets/stylesheets/framework/lists.scss b/app/assets/stylesheets/framework/lists.scss index 4b67eab05b3..fdc0454d837 100644 --- a/app/assets/stylesheets/framework/lists.scss +++ b/app/assets/stylesheets/framework/lists.scss @@ -21,11 +21,11 @@ } &.disabled { - color: $list-text-disabled-color; + color: $gl-text-color-tertiary; } &:not(.ui-sort-disabled):hover { - background: $row-hover; + background: $blue-50; } &.unstyled { @@ -35,12 +35,12 @@ } &.warning-row { - background-color: $list-warning-row-bg; - border-color: $list-warning-row-border; - color: $list-warning-row-color; + background-color: $orange-100; + border-color: $orange-200; + color: $orange-700; &:hover { - background: $list-warning-row-bg; + background: $orange-100; } } @@ -73,7 +73,7 @@ } .card.card-body-title { - font-size: $list-font-size; + font-size: $gl-font-size; line-height: 18px; } } @@ -109,8 +109,8 @@ ul.content-list { li { border-color: $white-normal; - font-size: $list-font-size; - color: $list-text-color; + font-size: $gl-font-size; + color: $gl-text-color; &.no-description { .title { diff --git a/app/assets/stylesheets/framework/mixins.scss b/app/assets/stylesheets/framework/mixins.scss index 98bf26a5222..7edb89ce6f3 100644 --- a/app/assets/stylesheets/framework/mixins.scss +++ b/app/assets/stylesheets/framework/mixins.scss @@ -50,7 +50,7 @@ @include clearfix; padding: 10px 0; - border-bottom: 1px solid $list-border-light; + border-bottom: 1px solid $gray-darker; display: block; margin: 0; diff --git a/app/assets/stylesheets/framework/typography.scss b/app/assets/stylesheets/framework/typography.scss index eccc814b747..5c6110737a4 100644 --- a/app/assets/stylesheets/framework/typography.scss +++ b/app/assets/stylesheets/framework/typography.scss @@ -7,7 +7,7 @@ } a { - color: $md-link-color; + color: $blue-600; } img:not(.emoji) { diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 866cb88ba5b..96508a71bd8 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -203,21 +203,12 @@ $gl-header-color: #4c4e54; /* * Lists */ -$list-font-size: $gl-font-size; -$list-title-color: $gl-text-color; -$list-text-color: $gl-text-color; -$list-text-disabled-color: $gl-text-color-tertiary; -$list-border-light: #eee; $list-border: rgba(0, 0, 0, 0.05); $list-text-height: 42px; -$list-warning-row-bg: $orange-100; -$list-warning-row-border: $orange-200; -$list-warning-row-color: $orange-700; /* * Markdown */ -$md-link-color: $blue-600; $md-area-border: #ddd; /* @@ -249,8 +240,6 @@ $gl-bar-padding: 3px; /* * Misc */ -$row-hover: $blue-50; -$row-hover-border: $blue-200; $progress-color: #c0392b; $header-height: 40px; $ide-statusbar-height: 25px; @@ -258,19 +247,13 @@ $fixed-layout-width: 1280px; $limited-layout-width: 990px; $container-text-max-width: 540px; $gl-avatar-size: 40px; -$error-exclamation-point: $red-500; $border-radius-default: 4px; $border-radius-small: 2px; $settings-icon-size: 18px; -$provider-btn-not-active-color: $blue-500; -$link-underline-blue: $blue-500; -$active-item-blue: $blue-500; $layout-link-gray: #7e7c7c; $btn-side-margin: 10px; $btn-sm-side-margin: 7px; $btn-margin-5: 5px; -$issue-status-expired: $orange-500; -$issuable-sidebar-color: $gl-text-color-secondary; $sidebar-block-hover-color: #ebebeb; $group-path-color: #999; $namespace-kind-color: #aaa; @@ -292,7 +275,6 @@ $breadcrumb-min-height: 48px; * Common component specific colors */ $hint-color: #999; -$well-pre-bg: #eee; $well-pre-color: #555; $loading-color: #555; $update-author-color: #999; @@ -301,10 +283,6 @@ $user-mention-bg-hover: rgba($blue-500, 0.15); $time-color: #999; $project-member-show-color: #aaa; $gl-promo-color: #aaa; -$error-bg: $red-400; -$warning-message-bg: $orange-100; -$warning-message-border: $orange-200; -$warning-message-color: $orange-700; $control-group-descr-color: #666; $table-permission-x-bg: #d9edf7; $username-color: #666; @@ -374,7 +352,7 @@ $dropdown-min-height: 40px; $dropdown-max-height: 312px; $dropdown-vertical-offset: 4px; $dropdown-link-color: #555; -$dropdown-link-hover-bg: $row-hover; +$dropdown-link-hover-bg: $blue-50; $dropdown-empty-row-bg: rgba(#000, 0.04); $dropdown-border-color: $border-color; $dropdown-shadow-color: rgba(#000, 0.1); @@ -592,11 +570,7 @@ $events-body-border: #ddd; /* * Files */ -$file-image-bg: #eee; -$blob-bg: #eee; -$blame-border: #eee; $blame-line-numbers-border: #ddd; -$logs-bg: #eee; $logs-li-color: #888; $logs-p-color: #333; @@ -667,7 +641,6 @@ $login-devise-error-color: $red-700; * Nav */ $nav-link-gray: #959494; -$nav-badge-bg: #eee; $nav-toggle-gray: #666; /* diff --git a/app/assets/stylesheets/pages/boards.scss b/app/assets/stylesheets/pages/boards.scss index 91f470ca709..69d7de886b4 100644 --- a/app/assets/stylesheets/pages/boards.scss +++ b/app/assets/stylesheets/pages/boards.scss @@ -288,7 +288,7 @@ &.is-active, &.is-active .board-card-assignee:hover a { - background-color: $row-hover; + background-color: $blue-50; } .badge { diff --git a/app/assets/stylesheets/pages/cycle_analytics.scss b/app/assets/stylesheets/pages/cycle_analytics.scss index bba9f38d3dd..bc4c90711d7 100644 --- a/app/assets/stylesheets/pages/cycle_analytics.scss +++ b/app/assets/stylesheets/pages/cycle_analytics.scss @@ -165,7 +165,7 @@ border-right-color: transparent; border-top-color: $border-color; border-bottom-color: $border-color; - box-shadow: inset 2px 0 0 0 $active-item-blue; + box-shadow: inset 2px 0 0 0 $blue-500; .stage-name { font-weight: $gl-font-weight-bold; diff --git a/app/assets/stylesheets/pages/diff.scss b/app/assets/stylesheets/pages/diff.scss index 47778110bae..a999a70693e 100644 --- a/app/assets/stylesheets/pages/diff.scss +++ b/app/assets/stylesheets/pages/diff.scss @@ -186,7 +186,7 @@ } .image { - background: $file-image-bg; + background: $gray-darker; text-align: center; padding: 30px; diff --git a/app/assets/stylesheets/pages/events.scss b/app/assets/stylesheets/pages/events.scss index f79586b68b9..da0c9b44498 100644 --- a/app/assets/stylesheets/pages/events.scss +++ b/app/assets/stylesheets/pages/events.scss @@ -6,7 +6,7 @@ font-size: $gl-font-size; padding: $gl-padding-top 0 $gl-padding-top 40px; border-bottom: 1px solid $white-normal; - color: $list-text-color; + color: $gl-text-color; position: relative; &.event-inline { @@ -58,7 +58,7 @@ .event-title { @include str-truncated(calc(100% - 174px)); font-weight: $gl-font-weight-bold; - color: $list-text-color; + color: $gl-text-color; } .event-body { diff --git a/app/assets/stylesheets/pages/groups.scss b/app/assets/stylesheets/pages/groups.scss index fa8a0f26b5d..60b4d39bb1a 100644 --- a/app/assets/stylesheets/pages/groups.scss +++ b/app/assets/stylesheets/pages/groups.scss @@ -385,8 +385,8 @@ padding: $gl-padding-top; &:hover { - border-color: $row-hover-border; - background-color: $row-hover; + border-color: $blue-200; + background-color: $blue-50; cursor: pointer; } diff --git a/app/assets/stylesheets/pages/issuable.scss b/app/assets/stylesheets/pages/issuable.scss index d16a63d009a..6f0f82964c8 100644 --- a/app/assets/stylesheets/pages/issuable.scss +++ b/app/assets/stylesheets/pages/issuable.scss @@ -423,10 +423,10 @@ width: 100%; text-align: center; margin-bottom: 10px; - color: $issuable-sidebar-color; + color: $gl-text-color-secondary; svg { - fill: $issuable-sidebar-color; + fill: $gl-text-color-secondary; } &:hover:not(.disabled), @@ -457,14 +457,14 @@ } .avatar-counter:hover { - color: $issuable-sidebar-color; - border-color: $issuable-sidebar-color; + color: $gl-text-color-secondary; + border-color: $gl-text-color-secondary; } .btn-clipboard { border: 0; background: transparent; - color: $issuable-sidebar-color; + color: $gl-text-color-secondary; &:hover { color: $gl-text-color; @@ -821,7 +821,7 @@ svg { width: 16px; height: 16px; - fill: $issuable-sidebar-color; + fill: $gl-text-color-secondary; } &:hover svg { diff --git a/app/assets/stylesheets/pages/profile.scss b/app/assets/stylesheets/pages/profile.scss index b45e305897c..17f34319050 100644 --- a/app/assets/stylesheets/pages/profile.scss +++ b/app/assets/stylesheets/pages/profile.scss @@ -20,7 +20,7 @@ .account-btn-link, .profile-settings-sidebar a, .settings-sidebar a { - color: $md-link-color; + color: $blue-600; } .private-tokens-reset div.reset-action:not(:first-child) { @@ -137,7 +137,7 @@ .profile-settings-content { a { - color: $md-link-color; + color: $blue-600; } } @@ -170,7 +170,7 @@ background-color: $gray-light; &.not-active { - color: $provider-btn-not-active-color; + color: $blue-500; } } diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index fffb440027c..c11916454c8 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -449,8 +449,8 @@ &:hover:not(.disabled), &.forked { - background-color: $row-hover; - border-color: $row-hover-border; + background-color: $blue-50; + border-color: $blue-200; } .avatar-container, @@ -894,7 +894,7 @@ pre.light-well { .cannot-be-merged, .cannot-be-merged:hover { - color: $error-exclamation-point; + color: $red-500; margin-top: 2px; } diff --git a/app/assets/stylesheets/pages/settings.scss b/app/assets/stylesheets/pages/settings.scss index fb03970f64f..5aa4cdec9c3 100644 --- a/app/assets/stylesheets/pages/settings.scss +++ b/app/assets/stylesheets/pages/settings.scss @@ -113,9 +113,9 @@ .settings-message { padding: 5px; line-height: 1.3; - color: $warning-message-color; - background-color: $warning-message-bg; - border: 1px solid $warning-message-border; + color: $orange-700; + background-color: $orange-100; + border: 1px solid $orange-200; border-radius: $border-radius-base; } @@ -303,7 +303,7 @@ } .mirror-error-badge { - background-color: $error-bg; + background-color: $red-400; border-radius: $border-radius-default; color: $white-light; } diff --git a/app/assets/stylesheets/pages/todos.scss b/app/assets/stylesheets/pages/todos.scss index 010a2c05a1c..5d3b7b21ce4 100644 --- a/app/assets/stylesheets/pages/todos.scss +++ b/app/assets/stylesheets/pages/todos.scss @@ -12,8 +12,8 @@ flex-direction: row; &:hover { - background-color: $row-hover; - border-color: $row-hover-border; + background-color: $blue-50; + border-color: $blue-200; cursor: pointer; } @@ -22,7 +22,7 @@ border-bottom: 1px solid transparent; &:hover { - border-color: $row-hover-border; + border-color: $blue-200; } } diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index 1cc26d40ba9..dc5ca78ff58 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -111,9 +111,9 @@ &:hover:not(.tree-truncated-warning) { td { - background-color: $row-hover; - border-top: 1px solid $row-hover-border; - border-bottom: 1px solid $row-hover-border; + background-color: $blue-50; + border-top: 1px solid $blue-200; + border-bottom: 1px solid $blue-200; cursor: pointer; } } @@ -229,7 +229,7 @@ .upload-link { font-weight: $gl-font-weight-normal; - color: $md-link-color; + color: $blue-600; } .repo-charts { diff --git a/app/controllers/projects/pages_controller.rb b/app/controllers/projects/pages_controller.rb index cae6e2c40b8..ff49911d892 100644 --- a/app/controllers/projects/pages_controller.rb +++ b/app/controllers/projects/pages_controller.rb @@ -11,7 +11,7 @@ class Projects::PagesController < Projects::ApplicationController def destroy project.remove_pages - project.pages_domains.destroy_all + project.pages_domains.destroy_all # rubocop: disable DestroyAll respond_to do |format| format.html do diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index dd07f389fa5..49981db0d80 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -101,7 +101,7 @@ module Awardable end def remove_award_emoji(name, current_user) - award_emoji.where(name: name, user: current_user).destroy_all + award_emoji.where(name: name, user: current_user).destroy_all # rubocop: disable DestroyAll end def toggle_award_emoji(emoji_name, current_user) diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 65ed46ea202..c342d01243e 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -34,7 +34,7 @@ module FastDestroyAll included do before_destroy do - raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`' + raise ForbiddenActionError, '`destroy` and `destroy_all` are forbidden. Please use `fast_destroy_all`' end end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 2a1a4ef48b7..97bf5d611c2 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -29,11 +29,13 @@ class LfsObject < ActiveRecord::Base [nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store) end + # rubocop: disable DestroyAll def self.destroy_unreferenced joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") .where(lfs_objects_projects: { id: nil }) .destroy_all end + # rubocop: enable DestroyAll def self.calculate_oid(path) Digest::SHA256.file(path).hexdigest diff --git a/app/models/user.rb b/app/models/user.rb index fb19de4b980..13b04270a4a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -516,7 +516,7 @@ class User < ActiveRecord::Base otp_grace_period_started_at: nil, otp_backup_codes: nil ) - self.u2f_registrations.destroy_all + self.u2f_registrations.destroy_all # rubocop: disable DestroyAll end end diff --git a/app/services/labels/promote_service.rb b/app/services/labels/promote_service.rb index c0463052821..623a5f0950e 100644 --- a/app/services/labels/promote_service.rb +++ b/app/services/labels/promote_service.rb @@ -65,7 +65,7 @@ module Labels end def update_project_labels(label_ids) - Label.where(id: label_ids).destroy_all + Label.where(id: label_ids).destroy_all # rubocop: disable DestroyAll end def clone_label_to_group_label(label) diff --git a/app/services/milestones/promote_service.rb b/app/services/milestones/promote_service.rb index 37aa6d3a9bc..660b4faaec0 100644 --- a/app/services/milestones/promote_service.rb +++ b/app/services/milestones/promote_service.rb @@ -73,7 +73,7 @@ module Milestones end def destroy_old_milestones(milestone) - Milestone.where(id: milestone_ids_for_merge(milestone)).destroy_all + Milestone.where(id: milestone_ids_for_merge(milestone)).destroy_all # rubocop: disable DestroyAll end def group_project_ids diff --git a/app/services/projects/move_deploy_keys_projects_service.rb b/app/services/projects/move_deploy_keys_projects_service.rb index 40a22837eaf..9f3f44f30ea 100644 --- a/app/services/projects/move_deploy_keys_projects_service.rb +++ b/app/services/projects/move_deploy_keys_projects_service.rb @@ -27,7 +27,7 @@ module Projects end def remove_remaining_deploy_keys_projects - source_project.deploy_keys_projects.destroy_all + source_project.deploy_keys_projects.destroy_all # rubocop: disable DestroyAll end end end diff --git a/app/services/projects/move_lfs_objects_projects_service.rb b/app/services/projects/move_lfs_objects_projects_service.rb index a5099519594..f78546a1e9c 100644 --- a/app/services/projects/move_lfs_objects_projects_service.rb +++ b/app/services/projects/move_lfs_objects_projects_service.rb @@ -21,7 +21,7 @@ module Projects end def remove_remaining_lfs_objects_project - source_project.lfs_objects_projects.destroy_all + source_project.lfs_objects_projects.destroy_all # rubocop: disable DestroyAll end def non_existent_lfs_objects_projects diff --git a/app/services/projects/move_notification_settings_service.rb b/app/services/projects/move_notification_settings_service.rb index 746605d56f1..109a00dd6d9 100644 --- a/app/services/projects/move_notification_settings_service.rb +++ b/app/services/projects/move_notification_settings_service.rb @@ -22,7 +22,7 @@ module Projects # Remove remaining notification settings from source_project def remove_remaining_notification_settings - source_project.notification_settings.destroy_all + source_project.notification_settings.destroy_all # rubocop: disable DestroyAll end # Get users of current notification_settings diff --git a/app/services/projects/move_project_group_links_service.rb b/app/services/projects/move_project_group_links_service.rb index d9038030f7e..1efafdce36d 100644 --- a/app/services/projects/move_project_group_links_service.rb +++ b/app/services/projects/move_project_group_links_service.rb @@ -26,7 +26,7 @@ module Projects # Remove remaining project group links from source_project def remove_remaining_project_group_links - source_project.reload.project_group_links.destroy_all + source_project.reload.project_group_links.destroy_all # rubocop: disable DestroyAll end def group_links_in_target_project diff --git a/app/services/projects/move_project_members_service.rb b/app/services/projects/move_project_members_service.rb index bb0c0d10242..ec983582d94 100644 --- a/app/services/projects/move_project_members_service.rb +++ b/app/services/projects/move_project_members_service.rb @@ -25,7 +25,7 @@ module Projects def remove_remaining_members # Remove remaining members and authorizations from source_project - source_project.project_members.destroy_all + source_project.project_members.destroy_all # rubocop: disable DestroyAll end def project_members_in_target_project diff --git a/app/services/protected_branches/legacy_api_update_service.rb b/app/services/protected_branches/legacy_api_update_service.rb index 1f6bbe72f85..da8bf2ce02a 100644 --- a/app/services/protected_branches/legacy_api_update_service.rb +++ b/app/services/protected_branches/legacy_api_update_service.rb @@ -38,11 +38,11 @@ module ProtectedBranches def delete_redundant_access_levels unless @developers_can_merge.nil? - @protected_branch.merge_access_levels.destroy_all + @protected_branch.merge_access_levels.destroy_all # rubocop: disable DestroyAll end unless @developers_can_push.nil? - @protected_branch.push_access_levels.destroy_all + @protected_branch.push_access_levels.destroy_all # rubocop: disable DestroyAll end end end diff --git a/app/workers/remove_expired_group_links_worker.rb b/app/workers/remove_expired_group_links_worker.rb index 6b8b972a440..25128caf72f 100644 --- a/app/workers/remove_expired_group_links_worker.rb +++ b/app/workers/remove_expired_group_links_worker.rb @@ -5,6 +5,6 @@ class RemoveExpiredGroupLinksWorker include CronjobQueue def perform - ProjectGroupLink.expired.destroy_all + ProjectGroupLink.expired.destroy_all # rubocop: disable DestroyAll end end diff --git a/app/workers/remove_old_web_hook_logs_worker.rb b/app/workers/remove_old_web_hook_logs_worker.rb index 17140ac4450..0f486f8991d 100644 --- a/app/workers/remove_old_web_hook_logs_worker.rb +++ b/app/workers/remove_old_web_hook_logs_worker.rb @@ -6,7 +6,9 @@ class RemoveOldWebHookLogsWorker WEB_HOOK_LOG_LIFETIME = 2.days + # rubocop: disable DestroyAll def perform WebHookLog.destroy_all(['created_at < ?', Time.now - WEB_HOOK_LOG_LIFETIME]) end + # rubocop: enable DestroyAll end diff --git a/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb b/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb index 668c22bb51c..8ebf1a5234d 100644 --- a/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb +++ b/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb @@ -16,6 +16,6 @@ class RemoveAwardEmojisWithNoUser < ActiveRecord::Migration # disable_ddl_transaction! def up - AwardEmoji.joins('LEFT JOIN users ON users.id = user_id').where('users.id IS NULL').destroy_all + AwardEmoji.joins('LEFT JOIN users ON users.id = user_id').where('users.id IS NULL').destroy_all # rubocop: disable DestroyAll end end diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 88221db78f1..bd758c49eba 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -42,7 +42,7 @@ Registry, etc. ## Hashed Storage > **Warning:** Hashed storage is in **Beta**. For the latest updates, check the -> associated [issue](https://gitlab.com/gitlab-com/infrastructure/issues/2821) +> associated [issue](https://gitlab.com/gitlab-com/infrastructure/issues/3542) > and please report any problems you encounter. Hashed Storage is the new storage behavior we are rolling out with 10.0. Instead diff --git a/doc/ci/README.md b/doc/ci/README.md index 7666219acb0..d782d64e971 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -76,6 +76,8 @@ learn how to leverage its potential even more. - [Trigger pipelines on a schedule](../user/project/pipelines/schedules.md) - [Kubernetes clusters](../user/project/clusters/index.md) - Integrate one or more Kubernetes clusters to your project +- [Interactive web terminal](interactive_web_terminal/index.md) - Open an interactive + web terminal to debug the running jobs ## GitLab CI/CD for Docker diff --git a/doc/ci/interactive_web_terminal/img/finished_job_with_terminal_open.png b/doc/ci/interactive_web_terminal/img/finished_job_with_terminal_open.png Binary files differnew file mode 100644 index 00000000000..199268a1486 --- /dev/null +++ b/doc/ci/interactive_web_terminal/img/finished_job_with_terminal_open.png diff --git a/doc/ci/interactive_web_terminal/img/interactive_web_terminal_page.png b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_page.png Binary files differnew file mode 100644 index 00000000000..b59c1b6bc43 --- /dev/null +++ b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_page.png diff --git a/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png Binary files differnew file mode 100644 index 00000000000..f92c6df07a1 --- /dev/null +++ b/doc/ci/interactive_web_terminal/img/interactive_web_terminal_running_job.png diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md new file mode 100644 index 00000000000..507aceb27fa --- /dev/null +++ b/doc/ci/interactive_web_terminal/index.md @@ -0,0 +1,52 @@ +# Getting started with interactive web terminals + +> Introduced in GitLab 11.3. + +CAUTION: **Warning:** +Interactive web terminals are in beta, so they might not work properly and +lack features. For more information [follow issue #25990](https://gitlab.com/gitlab-org/gitlab-ce/issues/25990). + +Interactive web terminals give the user access to a terminal in GitLab for +running one-of commands for their CI pipeline. + +NOTE: **Note:** +This is not available for the shared Runners on GitLab.com. +To make use of this feature, you need to provide your +[own Runner](https://docs.gitlab.com/runner/install/) and properly +[configure it](#configuration). + +## Configuration + +Two things need to be configured for the interactive web terminal to work: + +- The Runner needs to have [`[session_server]` configured + properly][session-server] +- Web terminals need to be + [enabled](../../administration/integration/terminal.md#enabling-and-disabling-terminal-support) + +## Debugging a running job + +NOTE: **Note:** Not all executors are +[supported](https://docs.gitlab.com/runner/executors/#compatibility-chart). + +Sometimes, when a job is running, things don't go as you would expect, and it +would be helpful if one can have a shell to aid debugging. When a job is +running, on the right panel you can see a button `debug` that will open the terminal +for the current job. + + + +When clicked, a new tab will open to the terminal page where you can access +the terminal and type commands like a normal shell. + + + +If you have the terminal open and the job has finished with its tasks, the +terminal will block the job from finishing for the duration configured in +[`[session_server].terminal_max_retention_time`][session-server] until you +close the terminal window. + + + +[session-server]: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section diff --git a/doc/development/README.md b/doc/development/README.md index fed3903c771..ee9a9852205 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -55,7 +55,13 @@ description: 'Learn how to contribute to GitLab.' - [Merge request performance guidelines](merge_request_performance_guidelines.md) for ensuring merge requests do not negatively impact GitLab performance -## Databases guides +## Database guides + +### Tooling + +- [Understanding EXPLAIN plans](understanding_explain_plans.md) +- [explain.depesz.com](https://explain.depesz.com/) for visualising the output + of `EXPLAIN` ### Migrations diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md new file mode 100644 index 00000000000..adf8795a5e3 --- /dev/null +++ b/doc/development/understanding_explain_plans.md @@ -0,0 +1,676 @@ +# Understanding EXPLAIN plans + +PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This +command can be invaluable when trying to determine how a query will perform. +You can use this command directly in your SQL query, as long as the query starts +with it: + +```sql +EXPLAIN +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +When running this on GitLab.com, we are presented with the following output: + +``` +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +When using _just_ `EXPLAIN`, PostgreSQL won't actually execute our query, +instead it produces an _estimated_ execution plan based on the available +statistics. This means the actual plan can differ quite a bit. Fortunately, +PostgreSQL provides us with the option to execute the query as well. To do so, +we need to use `EXPLAIN ANALYZE` instead of just `EXPLAIN`: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This will produce: + +``` +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +As we can see this plan is quite different, and includes a lot more data. Let's +discuss this step by step. + +Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a +query that will write data or might time out. If the query modifies data, +consider wrapping it in a transaction that rolls back automatically like so: + +```sql +BEGIN; +EXPLAIN ANALYZE +DELETE FROM users WHERE id = 1; +ROLLBACK; +``` + +The `EXPLAIN` command also takes additional options, such as `BUFFERS`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This will then produce: + +``` +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +For more information, refer to the official [EXPLAIN +documentation](https://www.postgresql.org/docs/current/static/sql-explain.html). + +## Nodes + +Every query plan consists of nodes. Nodes can be nested, and are executed from +the inside out. This means that the innermost node is executed before an outer +node. This can be best thought of as nested function calls, returning their +results as they unwind. For example, a plan starting with an `Aggregate` +followed by a `Nested Loop`, followed by an `Index Only scan` can be thought of +as the following Ruby code: + +```ruby +aggregate( + nested_loop( + index_only_scan() + index_only_scan() + ) +) +``` + +Nodes are indicated using a `->` followed by the type of node taken. For +example: + +``` +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +Here the first node executed is `Seq scan on projects`. The `Filter:` is an +additional filter applied to the results of the node. A filter is very similar +to Ruby's `Array#select`: it takes the input rows, applies the filter, and +produces a new list of rows. Once the node is done, we perform the `Aggregate` +above it. + +Nested nodes will look like this: + +``` +Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) + Buffers: shared hit=155 + -> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +Planning time: 2.585 ms +Execution time: 0.310 ms +``` + +Here we first perform two separate "Index Only" scans, followed by performing a +"Nested Loop" on the result of these two scans. + +## Node statistics + +Each node in a plan has a set of associated statistics, such as the cost, the +number of rows produced, the number of loops performed, and more. For example: + +``` +Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) +``` + +Here we can see that our cost ranges from `0.00..908044.47` (we'll cover this in +a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN +ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` +statistics describes the estimated width of each row, in bytes. + +The `costs` field specifies how expensive a node was. The cost is measured in +arbitrary units determined by the query planner's cost parameters. What +influences the costs depends on a variety of settings, such as `seq_page_cost`, +`cpu_tuple_cost`, and various others. +The format of the costs field is as follows: + +``` +STARTUP COST..TOTAL COST +``` + +The startup cost states how expensive it was to start the node, with the total +cost describing how expensive the entire node was. In general: the greater the +values, the more expensive the node. + +When using `EXPLAIN ANALYZE`, these statistics will also include the actual time +(in milliseconds) spent, and other runtime statistics (e.g. the actual number of +produced rows): + +``` +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) +``` + +Here we can see we estimated 5,746,969 rows to be returned, but in reality we +returned 5,746,940 rows. We can also see that _just_ this sequential scan took +2.98 seconds to run. + +Using `EXPLAIN (ANALYZE, BUFFERS)` will also give us information about the +number of rows removed by a filter, the number of buffers used, and more. For +example: + +``` +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +``` + +Here we can see that our filter has to remove 65,677 rows, and that we use +208,846 buffers. Each buffer in PostgreSQL is 8 KB (8192 bytes), meaning our +above node uses *1.6 GB of buffers*. That's a lot! + +## Node types + +There are quite a few different types of nodes, so we only cover some of the +more common ones here. + +A full list of all the available nodes and their descriptions can be found in +the [PostgreSQL source file +"plannodes.h"](https://github.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h) + +### Seq Scan + +A sequential scan over (a chunk of) a database table. This is like using +`Array#each`, but on a database table. Sequential scans can be quite slow when +retrieving lots of rows, so it's best to avoid these for large tables. + +### Index Only Scan + +A scan on an index that did not require fetching anything from the table. In +certain cases an index only scan may still fetch data from the table, in this +case the node will include a `Heap Fetches:` statistic. + +### Index Scan + +A scan on an index that required retrieving some data from the table. + +### Bitmap Index Scan and Bitmap Heap scan + +Bitmap scans fall between sequential scans and index scans. These are typically +used when we would read too much data from an index scan, but too little to +perform a sequential scan. A bitmap scan uses what is known as a [bitmap +index](https://en.wikipedia.org/wiki/Bitmap_index) to perform its work. + +The [source code of PostgreSQL](https://github.com/postgres/postgres/blob/1c2cb2744bf3d8ad751cd5cf3b347f10f48492b3/src/include/nodes/plannodes.h#L446-L457) +states the following on bitmap scans: + +> Bitmap Index Scan delivers a bitmap of potential tuple locations; it does not +> access the heap itself. The bitmap is used by an ancestor Bitmap Heap Scan +> node, possibly after passing through intermediate Bitmap And and/or Bitmap Or +> nodes to combine it with the results of other Bitmap Index Scans. + +### Limit + +Applies a `LIMIT` on the input rows. + +### Sort + +Sorts the input rows as specified using an `ORDER BY` statement. + +### Nested Loop + +A nested loop will execute its child nodes for every row produced by a node that +precedes it. For example: + +``` +-> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +``` + +Here the first child node (`Index Only Scan using users_pkey on users users_1`) +produces 36 rows, and is executed once (`rows=36 loops=1`). The next node +produces 1 row (`rows=1`), but is repeated 36 times (`loops=36`). This is +because the previous node produced 36 rows. + +This means that nested loops can quickly slow the query down if the various +child nodes keep producing many rows. + +## Optimising queries + +With that out of the way, let's see how we can optimise a query. Let's use the +following query as an example: + +```sql +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This query simply counts the number of users that have a Twitter profile set. +Let's run this using `EXPLAIN (ANALYZE, BUFFERS)`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This will produce the following plan: + +``` +Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) + Buffers: shared hit=202662 + -> Seq Scan on users (cost=0.00..844969.99 rows=56087 width=0) (actual time=0.019..1265.883 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487813 + Buffers: shared hit=202662 +Planning time: 0.390 ms +Execution time: 1271.180 ms +``` + +From this query plan we can see the following: + +1. We need to perform a sequential scan on the `users` table. +1. This sequential scan filters out 2,487,813 rows using a `Filter`. +1. We use 202,622 buffers, which equals 1.58 GB of memory. +1. It takes us 1.2 seconds to do all of this. + +Considering we are just counting users, that's quite expensive! + +Before we start making any changes, let's see if there are any existing indexes +on the `users` table that we might be able to use. We can obtain this +information by running `\d users` in a `psql` console, then scrolling down to +the `Indexes:` section: + +``` +Indexes: + "users_pkey" PRIMARY KEY, btree (id) + "users_confirmation_token_key" UNIQUE CONSTRAINT, btree (confirmation_token) + "users_email_key" UNIQUE CONSTRAINT, btree (email) + "users_reset_password_token_key" UNIQUE CONSTRAINT, btree (reset_password_token) + "index_on_users_lower_email" btree (lower(email::text)) + "index_on_users_lower_username" btree (lower(username::text)) + "index_on_users_name_lower" btree (lower(name::text)) + "index_users_on_admin" btree (admin) + "index_users_on_created_at" btree (created_at) + "index_users_on_email_trigram" gin (email gin_trgm_ops) + "index_users_on_feed_token" btree (feed_token) + "index_users_on_ghost" btree (ghost) + "index_users_on_incoming_email_token" btree (incoming_email_token) + "index_users_on_name" btree (name) + "index_users_on_name_trigram" gin (name gin_trgm_ops) + "index_users_on_state" btree (state) + "index_users_on_state_and_internal_attrs" btree (state) WHERE ghost <> true AND support_bot <> true + "index_users_on_support_bot" btree (support_bot) + "index_users_on_username" btree (username) + "index_users_on_username_trigram" gin (username gin_trgm_ops) +``` + +Here we can see there is no index on the `twitter` column, which means +PostgreSQL has to perform a sequential scan in this case. Let's try to fix this +by adding the following index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +If we now re-run our query using `EXPLAIN (ANALYZE, BUFFERS)` we get the +following plan: + +``` +Aggregate (cost=61002.82..61002.83 rows=1 width=8) (actual time=297.311..297.312 rows=1 loops=1) + Buffers: shared hit=51854 dirtied=19 + -> Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 + Heap Fetches: 26037 + Buffers: shared hit=51854 dirtied=19 +Planning time: 0.191 ms +Execution time: 297.334 ms +``` + +Now it takes just under 300 milliseconds to get our data, instead of 1.2 +seconds. However, we still use 51,854 buffers, which is about 400 MB of memory. +300 milliseconds is also quite slow for such a simple query. To understand why +this query is still expensive, let's take a look at the following: + +``` +Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 +``` + +We start with an index only scan on our index, but we somehow still apply a +`Filter` that filters out 2,487,830 rows. Why is that? Well, let's look at how +we created the index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +We simply told PostgreSQL to index all possible values of the `twitter` column, +even empty strings. Our query in turn uses `WHERE twitter != ''`. This means +that the index does improve things, as we don't need to do a sequential scan, +but we may still encounter empty strings. This means PostgreSQL _has_ to apply a +Filter on the index results to get rid of those values. + +Fortunately, we can improve this even further using "partial indexes". Partial +indexes are indexes with a `WHERE` condition that is applied when indexing data. +For example: + +```sql +CREATE INDEX CONCURRENTLY some_index ON users (email) WHERE id < 100 +``` + +This index would only index the `email` value of rows that match `WHERE id < +100`. We can use partial indexes to change our Twitter index to the following: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; +``` + +Once created, if we run our query again we will be given the following plan: + +``` +Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) + Buffers: shared hit=44036 + -> Index Only Scan using twitter_test on users (cost=0.41..1479.71 rows=51420 width=0) (actual time=0.023..15.514 rows=51833 loops=1) + Heap Fetches: 1208 + Buffers: shared hit=44036 +Planning time: 0.123 ms +Execution time: 19.848 ms +``` + +That's _a lot_ better! Now it only takes 20 milliseconds to get the data, and we +only use about 344 MB of buffers (instead of the original 1.58 GB). The reason +this works is that now PostgreSQL no longer needs to apply a `Filter`, as the +index only contains `twitter` values that are not empty. + +Keep in mind that you shouldn't just add partial indexes every time you want to +optimise a query. Every index has to be updated for every write, and they may +require quite a bit of space, depending on the amount of indexed data. As a +result, first check if there are any existing indexes you may be able to reuse. +If there aren't any, check if you can perhaps slightly change an existing one to +fit both the existing and new queries. Only add a new index if none of the +existing indexes can be used in any way. + +## Queries that can't be optimised + +Now that we have seen how to optimise a query, let's look at another query that +we might not be able to optimise: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +The output of `EXPLAIN (ANALYZE, BUFFERS)` is as follows: + +``` +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +Looking at the output we see the following Filter: + +``` +Filter: (visibility_level = ANY ('{0,20}'::integer[])) +Rows Removed by Filter: 65677 +``` + +Looking at the number of rows removed by the filter, we may be tempted to add an +index on `projects.visibility_level` to somehow turn this Sequential scan + +filter into an index-only scan. + +Unfortunately, doing so is unlikely to improve anything. Contrary to what some +might believe, an index being present _does not guarantee_ that PostgreSQL will +actually use it. For example, when doing a `SELECT * FROM projects` it is much +cheaper to just scan the entire table, instead of using an index and then +fetching data from the table. In such cases PostgreSQL may decide to not use an +index. + +Second, let's think for a moment what our query does: it gets all projects with +visibility level 0 or 20. In the above plan we can see this produces quite a lot +of rows (5,745,940), but how much is that relative to the total? Let's find out +by running the following query: + +```sql +SELECT visibility_level, count(*) AS amount +FROM projects +GROUP BY visibility_level +ORDER BY visibility_level ASC; +``` + +For GitLab.com this produces: + +``` + visibility_level | amount +------------------+--------- + 0 | 5071325 + 10 | 65678 + 20 | 674801 +``` + +Here the total number of projects is 5,811,804, and 5,746,126 of those are of +level 0 or 20. That's 98% of the entire table! + +So no matter what we do, this query will retrieve 98% of the entire table. Since +most time is spent doing exactly that, there isn't really much we can do to +improve this query, other than _not_ running it at all. + +What is important here is that while some may recommend to straight up add an +index the moment you see a sequential scan, it is _much more important_ to first +understand what your query does, how much data it retrieves, and so on. After +all, you can not optimise something you do not understand. + +### Cardinality and selectivity + +Earlier we saw that our query had to retrieve 98% of the rows in the table. +There are two terms commonly used for databases: cardinality, and selectivity. +Cardinality refers to the number of unique values in a particular column in a +table. + +Selectivity is the number of unique values produced by an operation (e.g. an +index scan or filter), relative to the total number of rows. The higher the +selectivity, the more likely PostgreSQL is able to use an index. + +In the above example, there are only 3 unique values: 0, 10, and 20. This means +the cardinality is 3. The selectivity in turn is also very low: 0.0000003% (2 / +5,811,804), because our `Filter` only filters using two values (`0` and `20`). +With such a low selectivity value it's not surprising that PostgreSQL decides +using an index is not worth it, because it would produce almost no unique rows. + +## Rewriting queries + +So the above query can't really be optimised as-is, or at least not much. But +what if we slightly change the purpose of it? What if instead of retrieving all +projects with `visibility_level` 0 or 20, we retrieve those that a user +interacted with somehow? + +Fortunately, GitLab has an answer for this, and it's a table called +`user_interacted_projects`. This table has the following schema: + +``` +Table "public.user_interacted_projects" + Column | Type | Modifiers +------------+---------+----------- + user_id | integer | not null + project_id | integer | not null +Indexes: + "index_user_interacted_projects_on_project_id_and_user_id" UNIQUE, btree (project_id, user_id) + "index_user_interacted_projects_on_user_id" btree (user_id) +Foreign-key constraints: + "fk_rails_0894651f08" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +``` + +Let's rewrite our query to JOIN this table onto our projects, and get the +projects for a specific user: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +INNER JOIN user_interacted_projects ON user_interacted_projects.project_id = projects.id +WHERE projects.visibility_level IN (0, 20) +AND user_interacted_projects.user_id = 1; +``` + +What we do here is the following: + +1. Get our projects. +1. INNER JOIN `user_interacted_projects`, meaning we're only left with rows in + `projects` that have a corresponding row in `user_interacted_projects`. +1. Limit this to the projects with `visibility_level` of 0 or 20, and to + projects that the user with ID 1 interacted with. + +If we run this query we get the following plan: + +``` + Aggregate (cost=871.03..871.04 rows=1 width=8) (actual time=9.763..9.763 rows=1 loops=1) + -> Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) + -> Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) + Index Cond: (user_id = 1) + -> Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 + Planning time: 2.614 ms + Execution time: 9.809 ms +``` + +Here it only took us just under 10 milliseconds to get the data. We can also see +we're retrieving far fewer projects: + +``` +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 +``` + +Here we see we perform 145 loops (`loops=145`), with every loop producing 1 row +(`rows=1`). This is much less than before, and our query performs much better! + +If we look at the plan we also see our costs are very low: + +``` +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) +``` + +Here our cost is only 3.45, and it only takes us 0.050 milliseconds to do so. +The next index scan is a bit more expensive: + +``` +Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) +``` + +Here the cost is 160.71 (`cost=0.43..160.71`), taking about 2.5 milliseconds +(based on the output of `actual time=....`). + +The most expensive part here is the "Nested Loop" that acts upon the result of +these two index scans: + +``` +Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) +``` + +Here we had to perform 870.52 disk page fetches for 203 rows, 9.748 +milliseconds, producing 143 rows in a single loop. + +The key takeaway here is that sometimes you have to rewrite (parts of) a query +to make it better. Sometimes that means having to slightly change your feature +to accommodate for better performance. + +## What makes a bad plan + +This is a bit of a difficult question to answer, because the definition of "bad" +is relative to the problem you are trying to solve. However, some patterns are +best avoided in most cases, such as: + +* Sequential scans on large tables +* Filters that remove a lot of rows +* Performing a certain step (e.g. an index scan) that requires _a lot_ of + buffers (e.g. more than 512 MB for GitLab.com). + +As a general guideline, aim for a query that: + +1. Takes no more than 10 milliseconds. Our target time spent in SQL per request + is around 100 milliseconds, so every query should be as fast as possible. +1. Does not use an excessive number of buffers, relative to the workload. For + example, retrieving ten rows shouldn't require 1 GB of buffers. +1. Does not spend a long amount of time performing disk IO operations. The + setting `track_io_timing` must be enabled for this data to be included in the + output of `EXPLAIN ANALYZE`. +1. Applies a `LIMIT` when retrieving rows without aggregating them, such as + `SELECT * FROM users`. +1. Doesn't use a `Filter` to filter out too many rows, especially if the query + does not use a `LIMIT` to limit the number of returned rows. Filters can + usually be removed by adding a (partial) index. + +These are _guidelines_ and not hard requirements, as different needs may require +different queries. The only _rule_ is that you _must always measure_ your query +(preferably using a production-like database) using `EXPLAIN (ANALYZE, BUFFERS)` +and related tools such as: + +* <https://explain.depesz.com/> +* <http://tatiyants.com/postgres-query-plan-visualization/> + +GitLab employees can also use our chatops solution, available in Slack using the +`/chatops` slash command. You can use chatops to get a query plan by running the +following: + +``` +/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Visualising the plan using <https://explain.depesz.com/> is also supported: + +``` +/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Quoting the query is not necessary. + +For more information about the available options, run: + +``` +/chatops run explain --help +``` diff --git a/doc/install/kubernetes/gitlab_omnibus.md b/doc/install/kubernetes/gitlab_omnibus.md index 9aee6b9dc74..c2c8a7a92fd 100644 --- a/doc/install/kubernetes/gitlab_omnibus.md +++ b/doc/install/kubernetes/gitlab_omnibus.md @@ -120,7 +120,7 @@ gitlabConfigStorageSize: 1Gi Ingress routing and SSL are automatically configured within this Chart. An NGINX ingress is provisioned and configured, and will route traffic to any service. SSL certificates are automatically created and configured by [kube-lego](https://github.com/kubernetes/charts/tree/master/stable/kube-lego). > **Note:** -Let's Encrypt limits a single TLD to five certificate requests within a single week. This means that common DNS wildcard services like [nip.io](http://nip.io) and [nip.io](http://nip.io) are unlikely to work. +Let's Encrypt limits a single TLD to five certificate requests within a single week. This means that common DNS wildcard services like [nip.io](http://nip.io) are unlikely to work. ## Installing GitLab using the Helm Chart > **Note:** diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index ac827cbe1ca..bcbaf00e11b 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -45,7 +45,7 @@ module Gitlab end def ensure_default_member! - @project.project_members.destroy_all + @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) end diff --git a/rubocop/cop/destroy_all.rb b/rubocop/cop/destroy_all.rb new file mode 100644 index 00000000000..38b6cb40f91 --- /dev/null +++ b/rubocop/cop/destroy_all.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module RuboCop + module Cop + # Cop that blacklists the use of `destroy_all`. + class DestroyAll < RuboCop::Cop::Cop + MSG = 'Use `delete_all` instead of `destroy_all`. ' \ + '`destroy_all` will load the rows into memory, then execute a ' \ + '`DELETE` for every individual row.' + + def_node_matcher :destroy_all?, <<~PATTERN + (send {send ivar lvar const} :destroy_all ...) + PATTERN + + def on_send(node) + return unless destroy_all?(node) + + add_offense(node, location: :expression) + end + end + end +end diff --git a/rubocop/rubocop.rb b/rubocop/rubocop.rb index a427208cdab..88c9bbf24f4 100644 --- a/rubocop/rubocop.rb +++ b/rubocop/rubocop.rb @@ -27,3 +27,4 @@ require_relative 'cop/project_path_helper' require_relative 'cop/rspec/env_assignment' require_relative 'cop/rspec/factories_in_migration_specs' require_relative 'cop/sidekiq_options_queue' +require_relative 'cop/destroy_all' diff --git a/spec/controllers/omniauth_callbacks_controller_spec.rb b/spec/controllers/omniauth_callbacks_controller_spec.rb index b23f183fec8..d377d69457f 100644 --- a/spec/controllers/omniauth_callbacks_controller_spec.rb +++ b/spec/controllers/omniauth_callbacks_controller_spec.rb @@ -95,7 +95,7 @@ describe OmniauthCallbacksController, type: :controller do end it 'allows linking the disabled provider' do - user.identities.destroy_all + user.identities.destroy_all # rubocop: disable DestroyAll sign_in(user) expect { post provider }.to change { user.reload.identities.count }.by(1) diff --git a/spec/controllers/projects/releases_controller_spec.rb b/spec/controllers/projects/releases_controller_spec.rb index fc1619acec6..20a6beb3df8 100644 --- a/spec/controllers/projects/releases_controller_spec.rb +++ b/spec/controllers/projects/releases_controller_spec.rb @@ -14,7 +14,7 @@ describe Projects::ReleasesController do describe 'GET #edit' do it 'initializes a new release' do tag_id = release.tag - project.releases.destroy_all + project.releases.destroy_all # rubocop: disable DestroyAll get :edit, namespace_id: project.namespace, project_id: project, tag_id: tag_id diff --git a/spec/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys_spec.rb b/spec/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys_spec.rb index 26d48cc8201..f92acf61682 100644 --- a/spec/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys_spec.rb +++ b/spec/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys_spec.rb @@ -5,7 +5,7 @@ describe Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys, :migration let!(:gpg_key) { create(:gpg_key, key: GpgHelpers::User3.public_key) } before do - GpgKeySubkey.destroy_all + GpgKeySubkey.destroy_all # rubocop: disable DestroyAll end it 'generate the subkeys' do diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 35a6fc94753..17348b01006 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -542,7 +542,7 @@ describe Gitlab::Git::Repository, :seed_helper do Gitlab::Shell.new.remove_repository('default', 'my_project') end - shared_examples 'repository mirror fecthing' do + shared_examples 'repository mirror fetching' do it 'fetches a repository as a mirror remote' do subject @@ -569,11 +569,11 @@ describe Gitlab::Git::Repository, :seed_helper do end context 'with gitaly enabled' do - it_behaves_like 'repository mirror fecthing' + it_behaves_like 'repository mirror fetching' end context 'with gitaly enabled', :skip_gitaly_mock do - it_behaves_like 'repository mirror fecthing' + it_behaves_like 'repository mirror fetching' end def new_repository_path diff --git a/spec/migrations/schedule_create_gpg_key_subkeys_from_gpg_keys_spec.rb b/spec/migrations/schedule_create_gpg_key_subkeys_from_gpg_keys_spec.rb index 96bef107599..c4427910518 100644 --- a/spec/migrations/schedule_create_gpg_key_subkeys_from_gpg_keys_spec.rb +++ b/spec/migrations/schedule_create_gpg_key_subkeys_from_gpg_keys_spec.rb @@ -6,7 +6,7 @@ describe ScheduleCreateGpgKeySubkeysFromGpgKeys, :migration, :sidekiq do create(:gpg_key, id: 1, key: GpgHelpers::User1.public_key) # rubocop:disable RSpec/FactoriesInMigrationSpecs create(:gpg_key, id: 2, key: GpgHelpers::User3.public_key) # rubocop:disable RSpec/FactoriesInMigrationSpecs # Delete all subkeys so they can be recreated - GpgKeySubkey.destroy_all + GpgKeySubkey.destroy_all # rubocop: disable DestroyAll end it 'correctly schedules background migrations' do diff --git a/spec/models/fork_network_member_spec.rb b/spec/models/fork_network_member_spec.rb index 25bf596fddc..60d04562e6c 100644 --- a/spec/models/fork_network_member_spec.rb +++ b/spec/models/fork_network_member_spec.rb @@ -11,7 +11,7 @@ describe ForkNetworkMember do let(:fork_network) { fork_network_member.fork_network } it 'removes the fork network if it was the last member' do - fork_network.fork_network_members.destroy_all + fork_network.fork_network_members.destroy_all # rubocop: disable DestroyAll expect(ForkNetwork.count).to eq(0) end diff --git a/spec/models/hooks/system_hook_spec.rb b/spec/models/hooks/system_hook_spec.rb index 01129df1107..edd1cb455af 100644 --- a/spec/models/hooks/system_hook_spec.rb +++ b/spec/models/hooks/system_hook_spec.rb @@ -73,7 +73,7 @@ describe SystemHook do it "project_destroy hook" do project.add_maintainer(user) - project.project_members.destroy_all + project.project_members.destroy_all # rubocop: disable DestroyAll expect(WebMock).to have_requested(:post, system_hook.url).with( body: /user_remove_from_team/, @@ -110,7 +110,7 @@ describe SystemHook do it 'group member destroy hook' do group.add_maintainer(user) - group.group_members.destroy_all + group.group_members.destroy_all # rubocop: disable DestroyAll expect(WebMock).to have_requested(:post, system_hook.url).with( body: /user_remove_from_group/, diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 6258bfa232f..48f4e53b93e 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -1764,7 +1764,7 @@ describe MergeRequest do context 'with no discussions' do before do - merge_request.notes.destroy_all + merge_request.notes.destroy_all # rubocop: disable DestroyAll end it 'returns true' do diff --git a/spec/models/project_group_link_spec.rb b/spec/models/project_group_link_spec.rb index 1fccf92627a..5bea21427d4 100644 --- a/spec/models/project_group_link_spec.rb +++ b/spec/models/project_group_link_spec.rb @@ -41,7 +41,7 @@ describe ProjectGroupLink do project.project_group_links.create(group: group) group_users.each { |user| expect(user.authorized_projects).to include(project) } - project.project_group_links.destroy_all + project.project_group_links.destroy_all # rubocop: disable DestroyAll group_users.each { |user| expect(user.authorized_projects).not_to include(project) } end end diff --git a/spec/policies/group_policy_spec.rb b/spec/policies/group_policy_spec.rb index 35951251bc5..615fea11f26 100644 --- a/spec/policies/group_policy_spec.rb +++ b/spec/policies/group_policy_spec.rb @@ -205,7 +205,7 @@ describe GroupPolicy do nested_group.add_guest(developer) nested_group.add_guest(maintainer) - group.owners.destroy_all + group.owners.destroy_all # rubocop: disable DestroyAll group.add_guest(owner) nested_group.add_owner(owner) diff --git a/spec/rubocop/cop/destroy_all_spec.rb b/spec/rubocop/cop/destroy_all_spec.rb new file mode 100644 index 00000000000..b0bc40552b3 --- /dev/null +++ b/spec/rubocop/cop/destroy_all_spec.rb @@ -0,0 +1,43 @@ +require 'spec_helper' +require 'rubocop' +require 'rubocop/rspec/support' +require_relative '../../../rubocop/cop/destroy_all' + +describe RuboCop::Cop::DestroyAll do + include CopHelper + + subject(:cop) { described_class.new } + + it 'flags the use of destroy_all with a send receiver' do + inspect_source('foo.destroy_all # rubocop: disable DestroyAll') + + expect(cop.offenses.size).to eq(1) + end + + it 'flags the use of destroy_all with a constant receiver' do + inspect_source('User.destroy_all # rubocop: disable DestroyAll') + + expect(cop.offenses.size).to eq(1) + end + + it 'flags the use of destroy_all when passing arguments' do + inspect_source('User.destroy_all([])') + + expect(cop.offenses.size).to eq(1) + end + + it 'flags the use of destroy_all with a local variable receiver' do + inspect_source(<<~RUBY) + users = User.all + users.destroy_all # rubocop: disable DestroyAll + RUBY + + expect(cop.offenses.size).to eq(1) + end + + it 'does not flag the use of delete_all' do + inspect_source('foo.delete_all') + + expect(cop.offenses).to be_empty + end +end diff --git a/spec/services/merge_requests/create_service_spec.rb b/spec/services/merge_requests/create_service_spec.rb index 06fb61baf33..74bcc15f912 100644 --- a/spec/services/merge_requests/create_service_spec.rb +++ b/spec/services/merge_requests/create_service_spec.rb @@ -134,9 +134,11 @@ describe MergeRequests::CreateService do let!(:pipeline_3) { create(:ci_pipeline, project: project, ref: "other_branch", project_id: project.id) } before do + # rubocop: disable DestroyAll project.merge_requests .where(source_branch: opts[:source_branch], target_branch: opts[:target_branch]) .destroy_all + # rubocop: enable DestroyAll end it 'sets head pipeline' do diff --git a/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb b/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb index 1c632847940..6268c149fc6 100644 --- a/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb +++ b/spec/services/merge_requests/delete_non_latest_diffs_service_spec.rb @@ -46,10 +46,12 @@ describe MergeRequests::DeleteNonLatestDiffsService, :clean_gitlab_redis_shared_ end it 'schedules no removal if there is no non-latest diffs' do + # rubocop: disable DestroyAll merge_request .merge_request_diffs .where.not(id: merge_request.latest_merge_request_diff_id) .destroy_all + # rubocop: enable DestroyAll expect(DeleteDiffFilesWorker).not_to receive(:bulk_perform_in) diff --git a/spec/services/todo_service_spec.rb b/spec/services/todo_service_spec.rb index 9a51c873b30..1746721b0d0 100644 --- a/spec/services/todo_service_spec.rb +++ b/spec/services/todo_service_spec.rb @@ -280,7 +280,7 @@ describe TodoService do end it 'does not create a todo if unassigned' do - issue.assignees.destroy_all + issue.assignees.destroy_all # rubocop: disable DestroyAll should_not_create_any_todo { service.reassigned_issue(issue, author) } end diff --git a/spec/support/shared_examples/fast_destroy_all.rb b/spec/support/shared_examples/fast_destroy_all.rb index 5448ddcfe33..a8079b6d864 100644 --- a/spec/support/shared_examples/fast_destroy_all.rb +++ b/spec/support/shared_examples/fast_destroy_all.rb @@ -4,8 +4,8 @@ shared_examples_for 'fast destroyable' do expect(external_data_counter).to be > 0 expect(subjects.count).to be > 0 - expect { subjects.first.destroy }.to raise_error('`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`') - expect { subjects.destroy_all }.to raise_error('`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`') + expect { subjects.first.destroy }.to raise_error('`destroy` and `destroy_all` are forbidden. Please use `fast_destroy_all`') + expect { subjects.destroy_all }.to raise_error('`destroy` and `destroy_all` are forbidden. Please use `fast_destroy_all`') # rubocop: disable DestroyAll expect(subjects.count).to be > 0 expect(external_data_counter).to be > 0 diff --git a/spec/workers/repository_check/single_repository_worker_spec.rb b/spec/workers/repository_check/single_repository_worker_spec.rb index 22fc64c1536..f11875cffd1 100644 --- a/spec/workers/repository_check/single_repository_worker_spec.rb +++ b/spec/workers/repository_check/single_repository_worker_spec.rb @@ -6,7 +6,7 @@ describe RepositoryCheck::SingleRepositoryWorker do it 'skips when the project has no push events' do project = create(:project, :repository, :wiki_disabled) - project.events.destroy_all + project.events.destroy_all # rubocop: disable DestroyAll break_project(project) expect(worker).not_to receive(:git_fsck) |
