diff options
66 files changed, 1294 insertions, 297 deletions
diff --git a/.scss-lint.yml b/.scss-lint.yml index 835a4a88c44..9bfc18b9698 100644 --- a/.scss-lint.yml +++ b/.scss-lint.yml @@ -65,7 +65,7 @@ linters: # Reports when you have an empty rule set. EmptyRule: - enabled: false + enabled: true # Reports when you have an @extend directive. ExtendDirective: diff --git a/CHANGELOG b/CHANGELOG index 361e33eb6b3..d6353d34395 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,17 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.8.0 (unreleased) + - Remove future dates from contribution calendar graph. + - Fix error when visiting commit builds page before build was updated -v 8.7.0 (unreleased) +v 8.7.1 (unreleased) + - Fix .gitlab-ci.yml parsing issue when hidde job is a template without script definition. !3849 + - Fix license detection to detect all license files, not only known licenses. !3878 + - Use the `can?` helper instead of `current_user.can?`. !3882 + - Prevent users from deleting Webhooks via API they do not own + - Fix Error 500 due to stale cache when projects are renamed or transferred + +v 8.7.0 - Gitlab::GitAccess and Gitlab::GitAccessWiki are now instrumented - Fix vulnerability that made it possible to gain access to private labels and milestones - The number of InfluxDB points stored per UDP packet can now be configured @@ -109,6 +118,7 @@ v 8.7.0 (unreleased) - Show number sign on external issue reference text (Florent Baldino) - Updated print style for issues - Use GitHub Issue/PR number as iid to keep references + - Add 'l' shortcut to open Label dropdown on issuables and 'i' to create new issue on a project - Import GitHub labels - Add option to filter by "Owned projects" on dashboard page - Import GitHub milestones @@ -178,7 +178,7 @@ gem 'ruby-fogbugz', '~> 0.2.1' gem 'd3_rails', '~> 3.5.0' #cal-heatmap -gem 'cal-heatmap-rails', '~> 3.5.0' +gem 'cal-heatmap-rails', '~> 3.6.0' # underscore-rails gem "underscore-rails", "~> 1.8.0" @@ -217,7 +217,7 @@ gem 'font-awesome-rails', '~> 4.2' gem 'gitlab_emoji', '~> 0.3.0' gem 'gon', '~> 6.0.1' gem 'jquery-atwho-rails', '~> 1.3.2' -gem 'jquery-rails', '~> 4.0.0' +gem 'jquery-rails', '~> 4.1.0' gem 'jquery-scrollto-rails', '~> 1.4.3' gem 'jquery-ui-rails', '~> 5.0.0' gem 'raphael-rails', '~> 2.1.2' diff --git a/Gemfile.lock b/Gemfile.lock index b00d7b35c84..f6558fc44fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -103,7 +103,7 @@ GEM bundler (~> 1.2) thor (~> 0.18) byebug (8.2.1) - cal-heatmap-rails (3.5.1) + cal-heatmap-rails (3.6.0) capybara (2.6.2) addressable mime-types (>= 1.16) @@ -134,7 +134,7 @@ GEM execjs coffee-script-source (1.10.0) colorize (0.7.7) - concurrent-ruby (1.0.0) + concurrent-ruby (1.0.1) connection_pool (2.2.0) coveralls (0.8.13) json (~> 1.8) @@ -431,8 +431,8 @@ GEM json ipaddress (0.8.2) jquery-atwho-rails (1.3.2) - jquery-rails (4.0.5) - rails-dom-testing (~> 1.0) + jquery-rails (4.1.1) + rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) jquery-scrollto-rails (1.4.3) @@ -629,7 +629,7 @@ GEM recaptcha (1.0.2) json redcarpet (3.3.3) - redis (3.2.2) + redis (3.3.0) redis-actionpack (4.0.1) actionpack (~> 4) redis-rack (~> 1.5.0) @@ -736,10 +736,9 @@ GEM rack shoulda-matchers (2.8.0) activesupport (>= 3.0.0) - sidekiq (4.0.1) + sidekiq (4.1.1) concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) - json (~> 1.0) redis (~> 3.2, >= 3.2.1) sidekiq-cron (0.4.0) redis-namespace (>= 1.5.2) @@ -905,7 +904,7 @@ DEPENDENCIES bullet bundler-audit byebug - cal-heatmap-rails (~> 3.5.0) + cal-heatmap-rails (~> 3.6.0) capybara (~> 2.6.2) capybara-screenshot (~> 1.0.0) carrierwave (~> 0.10.0) @@ -953,7 +952,7 @@ DEPENDENCIES httparty (~> 0.13.3) influxdb (~> 0.2) jquery-atwho-rails (~> 1.3.2) - jquery-rails (~> 4.0.0) + jquery-rails (~> 4.1.0) jquery-scrollto-rails (~> 1.4.3) jquery-turbolinks (~> 2.1.0) jquery-ui-rails (~> 5.0.0) diff --git a/app/assets/javascripts/issuable_context.js.coffee b/app/assets/javascripts/issuable_context.js.coffee index 3a439b94c59..3c491ebfc4c 100644 --- a/app/assets/javascripts/issuable_context.js.coffee +++ b/app/assets/javascripts/issuable_context.js.coffee @@ -10,8 +10,8 @@ class @IssuableContext $(this).submit() $(document) - .off 'click', '.dropdown-content a' - .on 'click', '.dropdown-content a', (e) -> + .off 'click', '.issuable-sidebar .dropdown-content a' + .on 'click', '.issuable-sidebar .dropdown-content a', (e) -> e.preventDefault() $(document) diff --git a/app/assets/javascripts/labels_select.js.coffee b/app/assets/javascripts/labels_select.js.coffee index 516f7d19932..85517b18c5a 100644 --- a/app/assets/javascripts/labels_select.js.coffee +++ b/app/assets/javascripts/labels_select.js.coffee @@ -223,7 +223,7 @@ class @LabelsSelect fieldName: $dropdown.data('field-name') id: (label) -> if $dropdown.hasClass("js-filter-submit") and not label.isAny? - label.title + _.escape label.title else label.id diff --git a/app/assets/javascripts/right_sidebar.js.coffee b/app/assets/javascripts/right_sidebar.js.coffee index 67403554340..2d084b76cfe 100644 --- a/app/assets/javascripts/right_sidebar.js.coffee +++ b/app/assets/javascripts/right_sidebar.js.coffee @@ -1,10 +1,12 @@ class @Sidebar constructor: (currentUser) -> + @sidebar = $('aside') + @addEventListeners() addEventListeners: -> - $('aside').on('click', '.sidebar-collapsed-icon', @sidebarCollapseClicked) - $('.dropdown').on('hidden.gl.dropdown', @sidebarDropdownHidden) + @sidebar.on('click', '.sidebar-collapsed-icon', @, @sidebarCollapseClicked) + $('.dropdown').on('hidden.gl.dropdown', @, @onSidebarDropdownHidden) $('.dropdown').on('loading.gl.dropdown', @sidebarDropdownLoading) $('.dropdown').on('loaded.gl.dropdown', @sidebarDropdownLoaded) @@ -30,26 +32,56 @@ class @Sidebar else i.show() - sidebarCollapseClicked: (e) -> + sidebar = e.data e.preventDefault() $block = $(@).closest('.block') + sidebar.openDropdown($block); - $('aside') - .find('.gutter-toggle') - .trigger('click') - $editLink = $block.find('.edit-link') + openDropdown: (blockOrName) -> + $block = if _.isString(blockOrName) then @getBlock(blockOrName) else blockOrName + + $block.find('.edit-link').trigger('click') - if $editLink.length - $editLink.trigger('click') - $block.addClass('collapse-after-update') - $('.page-with-sidebar').addClass('with-overlay') + if not @isOpen() + @setCollapseAfterUpdate($block) + @toggleSidebar('open') - sidebarDropdownHidden: (e) -> + setCollapseAfterUpdate: ($block) -> + $block.addClass('collapse-after-update') + $('.page-with-sidebar').addClass('with-overlay') + + onSidebarDropdownHidden: (e) -> + sidebar = e.data + e.preventDefault() $block = $(@).closest('.block') + sidebar.sidebarDropdownHidden($block) + + sidebarDropdownHidden: ($block) -> if $block.hasClass('collapse-after-update') $block.removeClass('collapse-after-update') $('.page-with-sidebar').removeClass('with-overlay') - $('aside') - .find('.gutter-toggle') - .trigger('click')
\ No newline at end of file + @toggleSidebar('hide') + + triggerOpenSidebar: -> + @sidebar + .find('.js-sidebar-toggle') + .trigger('click') + + toggleSidebar: (action = 'toggle') -> + if action is 'toggle' + @triggerOpenSidebar() + + if action is 'open' + @triggerOpenSidebar() if not @isOpen() + + if action is 'hide' + @triggerOpenSidebar() is @isOpen() + + isOpen: -> + @sidebar.is('.right-sidebar-expanded') + + getBlock: (name) -> + @sidebar.find(".block.#{name}") + + diff --git a/app/assets/javascripts/shortcuts.js.coffee b/app/assets/javascripts/shortcuts.js.coffee index 100e3aac535..f3d66004138 100644 --- a/app/assets/javascripts/shortcuts.js.coffee +++ b/app/assets/javascripts/shortcuts.js.coffee @@ -2,34 +2,35 @@ class @Shortcuts constructor: -> @enabledHelp = [] Mousetrap.reset() - Mousetrap.bind('?', @selectiveHelp) + Mousetrap.bind('?', @onToggleHelp) Mousetrap.bind('s', Shortcuts.focusSearch) Mousetrap.bind(['ctrl+shift+p', 'command+shift+p'], @toggleMarkdownPreview) Mousetrap.bind('t', -> Turbolinks.visit(findFileURL)) if findFileURL? - selectiveHelp: (e) => - Shortcuts.showHelp(e, @enabledHelp) + onToggleHelp: (e) => + e.preventDefault() + @toggleHelp(@enabledHelp) toggleMarkdownPreview: (e) => $(document).triggerHandler('markdown-preview:toggle', [e]) - @showHelp: (e, location) -> - if $('#modal-shortcuts').length > 0 - $('#modal-shortcuts').modal('show') - else - url = '/help/shortcuts' - url = gon.relative_url_root + url if gon.relative_url_root? - $.ajax( - url: url, - dataType: 'script', - success: (e) -> - if location and location.length > 0 - $(l).show() for l in location - else - $('.hidden-shortcut').show() - $('.js-more-help-button').remove() - ) - e.preventDefault() + toggleHelp: (location) -> + $modal = $('#modal-shortcuts') + + if $modal.length + $modal.modal('toggle') + return + + $.ajax( + url: gon.shortcuts_path, + dataType: 'script', + success: (e) -> + if location and location.length > 0 + $(l).show() for l in location + else + $('.hidden-shortcut').show() + $('.js-more-help-button').remove() + ) @focusSearch: (e) -> $('#search').focus() diff --git a/app/assets/javascripts/shortcuts_issuable.coffee b/app/assets/javascripts/shortcuts_issuable.coffee index bbf02f1db24..ad9b3c1c6bf 100644 --- a/app/assets/javascripts/shortcuts_issuable.coffee +++ b/app/assets/javascripts/shortcuts_issuable.coffee @@ -4,18 +4,8 @@ class @ShortcutsIssuable extends ShortcutsNavigation constructor: (isMergeRequest) -> super() - Mousetrap.bind('a', -> - $('.block.assignee .edit-link').trigger('click') - return false - ) - Mousetrap.bind('m', -> - $('.block.milestone .edit-link').trigger('click') - return false - ) - Mousetrap.bind('r', => - @replyWithSelectedText() - return false - ) + Mousetrap.bind('a', @openSidebarDropdown.bind(@, 'assignee')) + Mousetrap.bind('m', @openSidebarDropdown.bind(@, 'milestone')) Mousetrap.bind('j', => @prevIssue() return false @@ -28,7 +18,7 @@ class @ShortcutsIssuable extends ShortcutsNavigation @editIssue() return false ) - + Mousetrap.bind('l', @openSidebarDropdown.bind(@, 'labels')) if isMergeRequest @enabledHelp.push('.hidden-shortcut.merge_requests') @@ -71,3 +61,7 @@ class @ShortcutsIssuable extends ShortcutsNavigation editIssue: -> $editBtn = $('.issuable-edit') Turbolinks.visit($editBtn.attr('href')) + + openSidebarDropdown: (name) -> + sidebar.openDropdown(name) + return false diff --git a/app/assets/javascripts/shortcuts_navigation.coffee b/app/assets/javascripts/shortcuts_navigation.coffee index 8decaedd87b..f39504e0645 100644 --- a/app/assets/javascripts/shortcuts_navigation.coffee +++ b/app/assets/javascripts/shortcuts_navigation.coffee @@ -14,6 +14,7 @@ class @ShortcutsNavigation extends Shortcuts Mousetrap.bind('g m', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-merge_requests')) Mousetrap.bind('g w', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-wiki')) Mousetrap.bind('g s', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-snippets')) + Mousetrap.bind('i', -> ShortcutsNavigation.findAndFollowLink('.shortcuts-new-issue')) @enabledHelp.push('.hidden-shortcut.project') @findAndFollowLink: (selector) -> diff --git a/app/assets/javascripts/todos.js.coffee b/app/assets/javascripts/todos.js.coffee index 10e698d6a54..10bef96f43d 100644 --- a/app/assets/javascripts/todos.js.coffee +++ b/app/assets/javascripts/todos.js.coffee @@ -102,7 +102,8 @@ class @Todos todoLink = $(this).data('url') return unless todoLink - if e.metaKey + # Allow Meta-Click or Mouse3-click to open in a new tab + if e.metaKey or e.which is 2 e.preventDefault() window.open(todoLink,'_blank') else diff --git a/app/assets/stylesheets/framework/calendar.scss b/app/assets/stylesheets/framework/calendar.scss index 0b3af592d4a..11f39d583bd 100644 --- a/app/assets/stylesheets/framework/calendar.scss +++ b/app/assets/stylesheets/framework/calendar.scss @@ -54,6 +54,10 @@ fill: #254e77 !important; } + .future { + visibility: hidden; + } + .domain-background { fill: none; shape-rendering: crispedges; diff --git a/app/assets/stylesheets/framework/files.scss b/app/assets/stylesheets/framework/files.scss index 32cc15113f2..61d9954c6c8 100644 --- a/app/assets/stylesheets/framework/files.scss +++ b/app/assets/stylesheets/framework/files.scss @@ -84,10 +84,6 @@ } } - &.blob_file { - - } - &.blob-no-preview { background: #eee; text-shadow: 0 1px 2px #fff; diff --git a/app/assets/stylesheets/framework/markdown_area.scss b/app/assets/stylesheets/framework/markdown_area.scss index 704fa1ff800..fd885b38680 100644 --- a/app/assets/stylesheets/framework/markdown_area.scss +++ b/app/assets/stylesheets/framework/markdown_area.scss @@ -95,7 +95,7 @@ &.md-preview-holder { code { white-space: pre-wrap; - word-break: break-all; + word-break: keep-all; } } } diff --git a/app/assets/stylesheets/framework/selects.scss b/app/assets/stylesheets/framework/selects.scss index b2fab387e17..eae5f062dda 100644 --- a/app/assets/stylesheets/framework/selects.scss +++ b/app/assets/stylesheets/framework/selects.scss @@ -121,9 +121,6 @@ } } -.select2-container-multi .select2-choices .select2-search-choice { -} - .select2-drop-active { margin-top: 6px; font-size: 14px; diff --git a/app/assets/stylesheets/highlight/monokai.scss b/app/assets/stylesheets/highlight/monokai.scss index 28253d4ccb4..80a509a7c1a 100644 --- a/app/assets/stylesheets/highlight/monokai.scss +++ b/app/assets/stylesheets/highlight/monokai.scss @@ -111,8 +111,6 @@ .vg { color: #f8f8f2 } /* Name.Variable.Global */ .vi { color: #f8f8f2 } /* Name.Variable.Instance */ .il { color: #ae81ff } /* Literal.Number.Integer.Long */ - - .gh { } /* Generic Heading & Diff Header */ .gu { color: #75715e; } /* Generic.Subheading & Diff Unified/Comment? */ .gd { color: #f92672; } /* Generic.Deleted & Diff Deleted */ .gi { color: #a6e22e; } /* Generic.Inserted & Diff Inserted */ diff --git a/app/assets/stylesheets/pages/detail_page.scss b/app/assets/stylesheets/pages/detail_page.scss index 751a5ab4d92..3438dbe4958 100644 --- a/app/assets/stylesheets/pages/detail_page.scss +++ b/app/assets/stylesheets/pages/detail_page.scss @@ -40,6 +40,7 @@ .wiki { code { white-space: pre-wrap; + word-break: keep-all; } } } diff --git a/app/assets/stylesheets/pages/graph.scss b/app/assets/stylesheets/pages/graph.scss index 4e5c4ed84b6..f7f9a9bb770 100644 --- a/app/assets/stylesheets/pages/graph.scss +++ b/app/assets/stylesheets/pages/graph.scss @@ -18,9 +18,6 @@ } .graphs { - .graph-author-commits-count { - } - .graph-author-email { float: right; color: #777; diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index d54abe9bc02..55b1ad97eb0 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -109,6 +109,10 @@ ul.notes { border-color: darken(#f5f5f5, 8%); margin: 10px 0; } + + code { + word-break: keep-all; + } } a { diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 18e28fc49fe..7d4fc361ce2 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -109,7 +109,7 @@ class Projects::IssuesController < Projects::ApplicationController end end format.json do - render json: @issue.to_json(include: [:milestone, labels: { methods: :text_color }, assignee: { methods: :avatar_url }]) + render json: @issue.to_json(include: { milestone: {}, assignee: { methods: :avatar_url }, labels: { methods: :text_color } }) end end end diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index a57fd2d4e34..9c147b3689e 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -154,7 +154,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController @merge_request.target_project, @merge_request]) end format.json do - render json: @merge_request.to_json(include: [:milestone, labels: { methods: :text_color }, assignee: { methods: :avatar_url }]) + render json: @merge_request.to_json(include: { milestone: {}, assignee: { methods: :avatar_url }, labels: { methods: :text_color } }) end end else diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index ab85694da3f..3d5e61d2c18 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -123,6 +123,18 @@ module ProjectsHelper end end + def license_short_name(project) + no_license_key = project.repository.license_key.nil? || + # Back-compat if cache contains 'no-license', can be removed in a few weeks + project.repository.license_key == 'no-license' + + return 'LICENSE' if no_license_key + + license = Licensee::License.new(project.repository.license_key) + + license.nickname || license.name + end + private def get_project_nav_tabs(project, current_user) @@ -320,14 +332,6 @@ module ProjectsHelper @ref || @repository.try(:root_ref) end - def license_short_name(project) - license = Licensee::License.new(project.repository.license_key) - - license.nickname || license.name - end - - private - def filename_path(project, filename) if project && blob = project.repository.send(filename) namespace_project_blob_path( diff --git a/app/models/project.rb b/app/models/project.rb index 7aa21b19e67..0420c6a61ae 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -820,13 +820,11 @@ class Project < ActiveRecord::Base wiki = Repository.new("#{old_path}.wiki", self) if repo.exists? - repo.expire_cache - repo.expire_emptiness_caches + repo.before_delete end if wiki.exists? - wiki.expire_cache - wiki.expire_emptiness_caches + wiki.before_delete end end diff --git a/app/models/repository.rb b/app/models/repository.rb index da751591103..61c8dce6060 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -466,8 +466,8 @@ class Repository return nil if !exists? || empty? cache.fetch(:license_blob) do - if licensee_project.license - blob_at_branch(root_ref, licensee_project.matched_file.filename) + tree(:head).blobs.find do |file| + file.name =~ /\A(licen[sc]e|copying)(\..+|\z)/i end end end @@ -476,7 +476,7 @@ class Repository return nil if !exists? || empty? cache.fetch(:license_key) do - licensee_project.license.try(:key) || 'no-license' + Licensee.license(path).try(:key) end end @@ -959,8 +959,4 @@ class Repository def cache @cache ||= RepositoryCache.new(path_with_namespace) end - - def licensee_project - @licensee_project ||= Licensee.project(path) - end end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 79a27f4af7e..111b3ec05ea 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -34,6 +34,8 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end + project.expire_caches_before_rename(old_path) + # Apply new namespace id and visibility level project.namespace = new_namespace project.visibility_level = new_namespace.visibility_level unless project.visibility_level_allowed_by_group? diff --git a/app/views/help/_shortcuts.html.haml b/app/views/help/_shortcuts.html.haml index da3c3711cdd..70e88da7aae 100644 --- a/app/views/help/_shortcuts.html.haml +++ b/app/views/help/_shortcuts.html.haml @@ -21,7 +21,7 @@ %tr %td.shortcut .key ? - %td Show this dialog + %td Show/hide this dialog %tr %td.shortcut - if browser.mac? @@ -169,6 +169,10 @@ %td.shortcut .key t %td Go to finding file + %tr + %td.shortcut + .key i + %td New issue .col-lg-4 %table.shortcut-mappings %tbody{ class: 'hidden-shortcut network', style: 'display:none' } @@ -241,6 +245,10 @@ %td.shortcut .key e %td Edit issue + %tr + %td.shortcut + .key l + %td Change Label %tbody{ class: 'hidden-shortcut merge_requests', style: 'display:none' } %tr %th @@ -261,3 +269,7 @@ %td.shortcut .key e %td Edit merge request + %tr + %td.shortcut + .key l + %td Change Label diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index a15b7758c4b..479bde33719 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -124,3 +124,8 @@ %li.hidden = link_to namespace_project_network_path(@project.namespace, @project, current_ref), title: 'Network', class: 'shortcuts-network' do Network + + -# Shortcut to create a new issue + %li.hidden + = link_to new_namespace_project_issue_path(@project.namespace, @project), class: 'shortcuts-new-issue' do + Create a new issue diff --git a/app/views/projects/commit/_ci_commit.html.haml b/app/views/projects/commit/_ci_commit.html.haml index 25714e6cb47..d3acd33116c 100644 --- a/app/views/projects/commit/_ci_commit.html.haml +++ b/app/views/projects/commit/_ci_commit.html.haml @@ -16,7 +16,7 @@ - if defined?(link_to_commit) && link_to_commit for commit = link_to ci_commit.short_sha, namespace_project_commit_path(@project.namespace, @project, ci_commit.sha), class: "monospace" - - if ci_commit.duration > 0 + - if ci_commit.duration in = time_interval_in_words ci_commit.duration diff --git a/app/views/projects/project_members/_shared_group_members.html.haml b/app/views/projects/project_members/_shared_group_members.html.haml index 62888e41935..ae13f8428f0 100644 --- a/app/views/projects/project_members/_shared_group_members.html.haml +++ b/app/views/projects/project_members/_shared_group_members.html.haml @@ -8,7 +8,7 @@ group, members with %strong #{group_links.human_access} role (#{shared_group_users_count}) - - if current_user.can?(:admin_group, shared_group) + - if can?(current_user, :admin_group, shared_group) .panel-head-actions = link_to group_group_members_path(shared_group), class: 'btn btn-sm' do %i.fa.fa-pencil-square-o diff --git a/app/views/shared/issuable/_sidebar.html.haml b/app/views/shared/issuable/_sidebar.html.haml index 04b834ba5d1..ed1b8a8da2a 100644 --- a/app/views/shared/issuable/_sidebar.html.haml +++ b/app/views/shared/issuable/_sidebar.html.haml @@ -166,5 +166,5 @@ new LabelsSelect(); new IssuableContext('#{escape_javascript(current_user.to_json(only: [:username, :id, :name]))}'); new Subscription('.subscription') - new Sidebar(); new DueDateSelect(); + sidebar = new Sidebar(); diff --git a/bin/rails b/bin/rails index 5191e6927af..0138d79b751 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,9 @@ #!/usr/bin/env ruby +begin + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') +end APP_PATH = File.expand_path('../../config/application', __FILE__) require_relative '../config/boot' require 'rails/commands' @@ -1,4 +1,9 @@ #!/usr/bin/env ruby +begin + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') +end require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/rspec b/bin/rspec index 20060ebd79c..6e6709219af 100755 --- a/bin/rspec +++ b/bin/rspec @@ -1,7 +1,8 @@ #!/usr/bin/env ruby begin - load File.expand_path("../spring", __FILE__) -rescue LoadError + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') end require 'bundler/setup' load Gem.bin_path('rspec-core', 'rspec') diff --git a/bin/spinach b/bin/spinach index a080e286cfe..474050e29d1 100755 --- a/bin/spinach +++ b/bin/spinach @@ -1,7 +1,8 @@ #!/usr/bin/env ruby begin - load File.expand_path("../spring", __FILE__) -rescue LoadError + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') end require 'bundler/setup' load Gem.bin_path('spinach', 'spinach') diff --git a/bin/spring b/bin/spring index 7b45d374fcd..7fe232c3aae 100755 --- a/bin/spring +++ b/bin/spring @@ -4,12 +4,12 @@ # It gets overwritten when you run the `spring binstub` command. unless defined?(Spring) - require "rubygems" - require "bundler" + require 'rubygems' + require 'bundler' - if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m) - Gem.paths = { "GEM_PATH" => [Bundler.bundle_path.to_s, *Gem.path].uniq } - gem "spring", match[1] - require "spring/binstub" + if (match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m)) + Gem.paths = { 'GEM_PATH' => [Bundler.bundle_path.to_s, *Gem.path].uniq.join(Gem.path_separator) } + gem 'spring', match[1] + require 'spring/binstub' end end diff --git a/bin/teaspoon b/bin/teaspoon new file mode 100755 index 00000000000..7c3b8dfc4ed --- /dev/null +++ b/bin/teaspoon @@ -0,0 +1,8 @@ +#!/usr/bin/env ruby +begin + load File.expand_path('../spring', __FILE__) +rescue LoadError => e + raise unless e.message.include?('spring') +end +require 'bundler/setup' +load Gem.bin_path('teaspoon', 'teaspoon') diff --git a/doc/README.md b/doc/README.md index e6ac4794827..e358da1c424 100644 --- a/doc/README.md +++ b/doc/README.md @@ -41,6 +41,8 @@ - [Git LFS configuration](workflow/lfs/lfs_administration.md) - [Housekeeping](administration/housekeeping.md) Keep your Git repository tidy and fast. - [GitLab Performance Monitoring](monitoring/performance/introduction.md) Configure GitLab and InfluxDB for measuring performance metrics +- [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md) Debug when Sidekiq appears hung and is not processing jobs +- [High Availability](administration/high_availability/README.md) Configure multiple servers for scaling or high availability ## Contributor documentation diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md new file mode 100644 index 00000000000..43d85ffb775 --- /dev/null +++ b/doc/administration/high_availability/README.md @@ -0,0 +1,35 @@ +# High Availability + +GitLab supports several different types of clustering and high-availability. +The solution you choose will be based on the level of scalability and +availability you require. The easiest solutions are scalable, but not necessarily +highly available. + +## Architecture + +### Active/Passive + +For pure high-availability/failover with no scaling you can use an +active/passive configuration. This utilizes DRBD (Distributed Replicated +Block Device) to keep all data in sync. DRBD requires a low latency link to +remain in sync. It is not advisable to attempt to run DRBD between data centers +or in different cloud availability zones. + +Components/Servers Required: + +- 2 servers/virtual machines (one active/one passive) + +### Active/Active + +This architecture scales easily because all application servers handle +user requests simultaneously. The database, Redis, and GitLab application are +all deployed on separate servers. The configuration is **only** highly-available +if the database, Redis and storage are also configured as such. + +**Steps to configure active/active:** + +1. [Configure the database](database.md) +1. [Configure Redis](redis.md) +1. [Configure NFS](nfs.md) +1. [Configure the GitLab application servers](gitlab.md) +1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md new file mode 100644 index 00000000000..538dada1bae --- /dev/null +++ b/doc/administration/high_availability/database.md @@ -0,0 +1,116 @@ +# Configuring a Database for GitLab HA + +You can choose to install and manage a database server (PostgreSQL/MySQL) +yourself, or you can use GitLab Omnibus packages to help. GitLab recommends +PostgreSQL. This is the database that will be installed if you use the +Omnibus package to manage your database. + +## Configure your own database server + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring GitLab for HA](gitlab.md) + +## Configure using Omnibus + +1. Download/install GitLab Omnibus using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + Be sure to change the `external_url` to match your eventual GitLab front-end + URL. + + ```ruby + external_url 'https://gitlab.example.com' + + # Disable all components except PostgreSQL + postgresql['enable'] = true + bootstrap['enable'] = false + nginx['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + redis['enable'] = false + gitlab_workhorse['enable'] = false + mailroom['enable'] = false + + # PostgreSQL configuration + postgresql['sql_password'] = 'DB password' + postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'] + postgresql['listen_address'] = '0.0.0.0' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to install and configure PostgreSQL. + + > **Note**: This `reconfigure` step will result in some errors. + That's OK - don't be alarmed. + +1. Open a database prompt: + + ``` + su - gitlab-psql + /bin/bash + psql -h /var/opt/gitlab/postgresql -d template1 + + # Output: + + psql (9.2.15) + Type "help" for help. + + template1=# + ``` + +1. Run the following command at the database prompt and you will be asked to + enter the new password for the PostgreSQL superuser. + + ``` + \password + + # Output: + + Enter new password: + Enter it again: + ``` + +1. Similarly, set the password for the `gitlab` database user. Use the same + password that you specified in the `/etc/gitlab/gitlab.rb` file for + `postgresql['sql_password']`. + + ``` + \password gitlab + + # Output: + + Enter new password: + Enter it again: + ``` + +1. Enable the `pg_trgm` extension: + ``` + CREATE EXTENSION pg_trgm; + + # Output: + + CREATE EXTENSION + ``` +1. Exit the database prompt by typing `\q` and Enter. +1. Exit the `gitlab-psql` user by running `exit` twice. +1. Run `sudo gitlab-ctl reconfigure` a final time. +1. Run `touch /etc/gitlab/skip-auto-migrations` to prevent database migrations + from running on upgrade. Only the primary GitLab application server should + handle migrations. + +--- + +Read more on high-availability configuration: + +1. [Configure Redis](redis.md) +1. [Configure NFS](nfs.md) +1. [Configure the GitLab application servers](gitlab.md) +1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md new file mode 100644 index 00000000000..8a881ce8863 --- /dev/null +++ b/doc/administration/high_availability/gitlab.md @@ -0,0 +1,131 @@ +# Configuring GitLab for HA + +Assuming you have already configured a database, Redis, and NFS, you can +configure the GitLab application server(s) now. Complete the steps below +for each GitLab application server in your environment. + +> **Note:** There is some additional configuration near the bottom for + secondary GitLab application servers. It's important to read and understand + these additional steps before proceeding with GitLab installation. + +1. If necessary, install the NFS client utility packages using the following + commands: + + ``` + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + +1. Specify the necessary NFS shares. Mounts are specified in + `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See [NFS documentation](nfs.md) for the various + options. Here is an example snippet to add to `/etc/fstab`: + + ``` + 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 + 10.1.1.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 + ``` + +1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ``` + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + +1. Download/install GitLab Omnibus using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + Be sure to change the `external_url` to match your eventual GitLab front-end + URL. Depending your the NFS configuration, you may need to change some GitLab + data locations. See [NFS documentation](nfs.md) for `/etc/gitlab/gitlab.rb` + configuration values for various scenarios. The example below assumes you've + added NFS mounts in the default data locations. + + ```ruby + external_url 'https://gitlab.example.com' + + # Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + # Disable components that will not be on the GitLab application server + postgresql['enable'] = false + redis['enable'] = false + + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + + # Redis connection details + gitlab_rails['redis_port'] = '6379' + gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server + gitlab_rails['redis_password'] = 'Redis Password' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. + +## Primary GitLab application server + +As a final step, run the setup rake task on the first GitLab application server. +It is not necessary to run this on additional application servers. + +1. Initialize the database by running `sudo gitlab-rake gitlab:setup`. + +> **WARNING:** Only run this setup task on **NEW** GitLab instances because it + will wipe any existing data. + +> **Note:** When you specify `https` in the `external_url`, as in the example + above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If + certificates are not present, Nginx will fail to start. See + [Nginx documentation](http://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) + for more information. + +## Additional configuration for secondary GitLab application servers + +Secondary GitLab servers (servers configured **after** the first GitLab server) +need some additional configuration. + +1. Configure shared secrets. These values can be obtained from the primary + GitLab server in `/etc/gitlab/gitlab-secrets.json`. Add these to + `/etc/gitlab/gitlab.rb` **prior to** running the first `reconfigure` in + the steps above. + + ```ruby + gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' + gitlab_rails['secret_token'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' + gitlab_ci['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' + gitlab_ci['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' + ``` + +1. Run `touch /etc/gitlab/skip-auto-migrations` to prevent database migrations + from running on upgrade. Only the primary GitLab application server should + handle migrations. + +## Troubleshooting + +- `mount: wrong fs type, bad option, bad superblock on` + +You have not installed the necessary NFS client utilities. See step 1 above. + +- `mount: mount point /var/opt/gitlab/... does not exist` + +This particular directory does not exist on the NFS server. Ensure +the share is exported and exists on the NFS server and try to remount. + +--- + +Read more on high-availability configuration: + +1. [Configure the database](database.md) +1. [Configure Redis](redis.md) +1. [Configure NFS](nfs.md) +1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/high_availability/load_balancer.md b/doc/administration/high_availability/load_balancer.md new file mode 100644 index 00000000000..b1fe34ed9a1 --- /dev/null +++ b/doc/administration/high_availability/load_balancer.md @@ -0,0 +1,63 @@ +# Load Balancer for GitLab HA + +In an active/active GitLab configuration, you will need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or the exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing HA systems like GitLab you have a load balancer of +choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, +and Citrix Net Scaler. This documentation will outline what ports and protocols +you need to use with GitLab. + +## Basic ports + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 80 | 80 | HTTP | +| 443 | 443 | HTTPS [^1] | +| 22 | 22 | TCP | + +## GitLab Pages Ports + +If you're using GitLab Pages you will need some additional port configurations. +GitLab Pages requires a separate VIP. Configure DNS to point the +`pages_external_url` from `/etc/gitlab/gitlab.rb` at the new VIP. See the +[GitLab Pages documentation][gitlab-pages] for more information. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 80 | Varies [^2] | HTTP | +| 443 | Varies [^2] | TCP [^3] | + +## Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that allows users +to use SSH on port 443. An alternate SSH hostname will require a new VIP +compared to the other GitLab HTTP configuration above. + +Configure DNS for an alternate SSH hostname such as altssh.gitlab.example.com. + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +--- + +Read more on high-availability configuration: + +1. [Configure the database](database.md) +1. [Configure Redis](redis.md) +1. [Configure NFS](nfs.md) +1. [Configure the GitLab application servers](gitlab.md) + +[^1]: When using HTTPS protocol for port 443, you will need to add an SSL + certificate to the load balancers. If you wish to terminate SSL at the + GitLab application server instead, use TCP protocol. +[^2]: The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + setting. See [GitLab Pages documentation][gitlab-pages] for more details. +[^3]: Port 443 for GitLab Pages should always use the TCP protocol. Users can + configure custom domains with custom SSL, which would not be possible + if SSL was terminated at the load balancer. + +[gitlab-pages]: http://doc.gitlab.com/ee/pages/administration.html diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md new file mode 100644 index 00000000000..e4e124e200a --- /dev/null +++ b/doc/administration/high_availability/nfs.md @@ -0,0 +1,116 @@ +# NFS + +## Required NFS Server features + +**File locking**: GitLab **requires** file locking which is only supported +natively in NFS version 4. NFSv3 also supports locking as long as +Linux Kernel 2.6.5+ is used. We recommend using version 4 and do not +specifically test NFSv3. + +**no_root_squash**: NFS normally changes the `root` user to `nobody`. This is +a good security measure when NFS shares will be accessed by many different +users. However, in this case only GitLab will use the NFS share so it +is safe. GitLab requires the `no_root_squash` setting because we need to +manage file permissions automatically. Without the setting you will receive +errors when the Omnibus package tries to alter permissions. Note that GitLab +and other bundled components do **not** run as `root` but as non-privileged +users. The requirement for `no_root_squash` is to allow the Omnibus package to +set ownership and permissions on files, as needed. + +### Recommended options + +When you define your NFS exports, we recommend you also add the following +options: + +- `sync` - Force synchronous behavior. Default is asynchronous and under certain + circumstances it could lead to data loss if a failure occurs before data has + synced. + +## Client mount options + +Below is an example of an NFS mount point we use on GitLab.com: + +``` +10.1.1.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nobootwait,lookupcache=positive 0 2 +``` + +Notice several options that you should consider using: + +| Setting | Description | +| ------- | ----------- | +| `nobootwait` | Don't halt boot process waiting for this mount to become available +| `lookupcache=positive` | Tells the NFS client to honor `positive` cache results but invalidates any `negative` cache results. Negative cache results cause problems with Git. Specifically, a `git push` can fail to register uniformly across all NFS clients. The negative cache causes the clients to 'remember' that the files did not exist previously. + +## Mount locations + +When using default Omnibus configuration you will need to share 5 data locations +between all GitLab cluster nodes. No other locations should be shared. The +following are the 5 locations you need to mount: + +| Location | Description | +| -------- | ----------- | +| `/var/opt/gitlab/git-data` | Git repository data. This will account for a large portion of your data +| `/var/opt/gitlab/.ssh` | SSH `authorized_keys` file and keys used to import repositories from some other Git services +| `/var/opt/gitlab/gitlab-rails/uploads` | User uploaded attachments +| `/var/opt/gitlab/gitlab-rails/shared` | Build artifacts, GitLab Pages, LFS objects, temp files, etc. If you're using LFS this may also account for a large portion of your data +| `/var/opt/gitlab/gitlab-ci/builds` | GitLab CI build traces + +Other GitLab directories should not be shared between nodes. They contain +node-specific files and GitLab code that does not need to be shared. To ship +logs to a central location consider using remote syslog. GitLab Omnibus packages +provide configuration for [UDP log shipping][udp-log-shipping]. + +### Consolidating mount points + +If you don't want to configure 5-6 different NFS mount points, you have a few +alternative options. + +#### Change default file locations + +Omnibus allows you to configure the file locations. With custom configuration +you can specify just one main mountpoint and have all of these locations +as subdirectories. Mount `/gitlab-data` then use the following Omnibus +configuration to move each data location to a subdirectory: + +```ruby +user['home'] = '/gitlab-data/home' +git_data_dir '/gitlab-data/git-data' +gitlab_rails['shared_path'] = '/gitlab-data/shared' +gitlab_rails['uploads_directory'] = "/gitlab-data/uploads" +gitlab_ci['builds_directory'] = '/gitlab-data/builds' +``` + +To move the `git` home directory, all GitLab services must be stopped. Run +`gitlab-ctl stop && initctl stop gitlab-runsvdir`. Then continue with the +reconfigure. + +Run `sudo gitlab-ctl reconfigure` to start using the central location. Please +be aware that if you had existing data you will need to manually copy/rsync it +to these new locations and then restart GitLab. + +#### Bind mounts + +Bind mounts provide a way to specify just one NFS mount and then +bind the default GitLab data locations to the NFS mount. Start by defining your +single NFS mount point as you normally would in `/etc/fstab`. Let's assume your +NFS mount point is `/gitlab-data`. Then, add the following bind mounts in +`/etc/fstab`: + +```bash +/gitlab-data/git-data /var/opt/gitlab/git-data none bind 0 0 +/gitlab-data/.ssh /var/opt/gitlab/.ssh none bind 0 0 +/gitlab-data/uploads /var/opt/gitlab/gitlab-rails/uploads none bind 0 0 +/gitlab-data/shared /var/opt/gitlab/gitlab-rails/shared none bind 0 0 +/gitlab-data/builds /var/opt/gitlab/gitlab-ci/builds none bind 0 0 +``` + +--- + +Read more on high-availability configuration: + +1. [Configure the database](database.md) +1. [Configure Redis](redis.md) +1. [Configure the GitLab application servers](gitlab.md) +1. [Configure the load balancers](load_balancer.md) + +[udp-log-shipping]: http://doc.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only "UDP log shipping" diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md new file mode 100644 index 00000000000..d89a1e582ca --- /dev/null +++ b/doc/administration/high_availability/redis.md @@ -0,0 +1,62 @@ +# Configuring Redis for GitLab HA + +You can choose to install and manage Redis yourself, or you can use GitLab +Omnibus packages to help. + +## Configure your own Redis server + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for Redis. For example, AWS offers a managed ElastiCache service +that runs Redis. + +> **Note:** Redis does not require authentication by default. See + [Redis Security](http://redis.io/topics/security) documentation for more + information. We recommend using a combination of a Redis password and tight + firewall rules to secure your Redis service. + +## Configure using Omnibus + +1. Download/install GitLab Omnibus using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + Be sure to change the `external_url` to match your eventual GitLab front-end + URL. + + ```ruby + external_url 'https://gitlab.example.com' + + # Disable all components except PostgreSQL + redis['enable'] = true + bootstrap['enable'] = false + nginx['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + postgresql['enable'] = false + gitlab_workhorse['enable'] = false + mailroom['enable'] = false + + # Redis configuration + redis['port'] = 6379 + redis['bind'] = '0.0.0.0' + + # If you wish to use Redis authentication (recommended) + redis['password'] = 'Redis Password' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to install and configure PostgreSQL. + + > **Note**: This `reconfigure` step will result in some errors. + That's OK - don't be alarmed. +1. Run `touch /etc/gitlab/skip-auto-migrations` to prevent database migrations + from running on upgrade. Only the primary GitLab application server should + handle migrations. + +--- + +Read more on high-availability configuration: + +1. [Configure the database](database.md) +1. [Configure NFS](nfs.md) +1. [Configure the GitLab application servers](gitlab.md) +1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md new file mode 100644 index 00000000000..134a7583762 --- /dev/null +++ b/doc/administration/troubleshooting/sidekiq.md @@ -0,0 +1,162 @@ +# Troubleshooting Sidekiq + +Sidekiq is the background job processor GitLab uses to asynchronously run +tasks. When things go wrong it can be difficult to troubleshoot. These +situations also tend to be high-pressure because a production system job queue +may be filling up. Users will notice when this happens because new branches +may not show up and merge requests may not be updated. The following are some +troubleshooting steps that will help you diagnose the bottleneck. + +> **Note:** GitLab administrators/users should consider working through these +debug steps with GitLab Support so the backtraces can be analyzed by our team. +It may reveal a bug or necessary improvement in GitLab. + +> **Note:** In any of the backtraces, be weary of suspecting cases where every + thread appears to be waiting in the database, Redis, or waiting to acquire + a mutex. This **may** mean there's contention in the database, for example, + but look for one thread that is different than the rest. This other thread + may be using all available CPU, or have a Ruby Global Interpreter Lock, + preventing other threads from continuing. + +## Thread dump + +Send the Sidekiq process ID the `TTIN` signal and it will output thread +backtraces in the log file. + +``` +kill -TTIN <sidekiq_pid> +``` + +Check in `/var/log/gitlab/sidekiq/current` or `$GITLAB_HOME/log/sidekiq.log` for +the backtrace output. The backtraces will be lengthy and generally start with +several `WARN` level messages. Here's an example of a single thread's backtrace: + +``` +2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: ActiveRecord::RecordNotFound: Couldn't find Note with 'id'=3375386 +2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/activerecord-4.2.5.2/lib/active_record/core.rb:155:in `find' +/opt/gitlab/embedded/service/gitlab-rails/app/workers/new_note_worker.rb:7:in `perform' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:150:in `execute_job' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:132:in `block (2 levels) in process' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:127:in `block in invoke' +/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/memory_killer.rb:17:in `call' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:129:in `block in invoke' +/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/arguments_logger.rb:6:in `call' +... +``` + +In some cases Sidekiq may be hung and unable to respond to the `TTIN` signal. +Move on to other troubleshooting methods if this happens. + +## Process profiling with `perf` + +Linux has a process profiling tool called `perf` that is helpful when a certain +process is eating up a lot of CPU. If you see high CPU usage and Sidekiq won't +respond to the `TTIN` signal, this is a good next step. + +If `perf` is not installed on your system, install it with `apt-get` or `yum`: + +``` +# Debian +sudo apt-get install linux-tools + +# Ubuntu (may require these additional Kernel packages) +sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` + +# Red Hat/CentOS +sudo yum install perf +``` + +Run perf against the Sidekiq PID: + +``` +sudo perf record -p <sidekiq_pid> +``` + +Let this run for 30-60 seconds and then press Ctrl-C. Then view the perf report: + +``` +sudo perf report + +# Sample output +Samples: 348K of event 'cycles', Event count (approx.): 280908431073 + 97.69% ruby nokogiri.so [.] xmlXPathNodeSetMergeAndClear + 0.18% ruby libruby.so.2.1.0 [.] objspace_malloc_increase + 0.12% ruby libc-2.12.so [.] _int_malloc + 0.10% ruby libc-2.12.so [.] _int_free +``` + +Above you see sample output from a perf report. It shows that 97% of the CPU is +being spent inside Nokogiri and `xmlXPathNodeSetMergeAndClear`. For something +this obvious you should then go investigate what job in GitLab would use +Nokogiri and XPath. Combine with `TTIN` or `gdb` output to show the +corresponding Ruby code where this is happening. + +## The GNU Project Debugger (gdb) + +`gdb` can be another effective tool for debugging Sidekiq. It gives you a little +more interactive way to look at each thread and see what's causing problems. + +> **Note:** Attaching to a process with `gdb` will suspends the normal operation + of the process (Sidekiq will not process jobs while `gdb` is attached). + +Start by attaching to the Sidekiq PID: + +``` +gdb -p <sidekiq_pid> +``` + +Then gather information on all the threads: + +``` +info threads + +# Example output +30 Thread 0x7fe5fbd63700 (LWP 26060) 0x0000003f7cadf113 in poll () from /lib64/libc.so.6 +29 Thread 0x7fe5f2b3b700 (LWP 26533) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +28 Thread 0x7fe5f2a3a700 (LWP 26534) 0x0000003f7ce0ba5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +27 Thread 0x7fe5f2939700 (LWP 26535) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +26 Thread 0x7fe5f2838700 (LWP 26537) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +25 Thread 0x7fe5f2737700 (LWP 26538) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +24 Thread 0x7fe5f2535700 (LWP 26540) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +23 Thread 0x7fe5f2434700 (LWP 26541) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +22 Thread 0x7fe5f2232700 (LWP 26543) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +21 Thread 0x7fe5f2131700 (LWP 26544) 0x00007fe5f7b570f0 in xmlXPathNodeSetMergeAndClear () +from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +... +``` + +If you see a suspicious thread, like the Nokogiri one above, you may want +to get more information: + +``` +thread 21 +bt + +# Example output +#0 0x00007ff0d6afe111 in xmlXPathNodeSetMergeAndClear () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#1 0x00007ff0d6b0b836 in xmlXPathNodeCollectAndTest () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#2 0x00007ff0d6b09037 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#3 0x00007ff0d6b09017 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#4 0x00007ff0d6b092e0 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#5 0x00007ff0d6b0bc37 in xmlXPathRunEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#6 0x00007ff0d6b0be5f in xmlXPathEvalExpression () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#7 0x00007ff0d6a97dc3 in evaluate (argc=2, argv=0x1022d058, self=<value optimized out>) at xml_xpath_context.c:221 +#8 0x00007ff0daeab0ea in vm_call_cfunc_with_frame (th=0x1022a4f0, reg_cfp=0x1032b810, ci=<value optimized out>) at vm_insnhelper.c:1510 +``` + +To output a backtrace from all threads at once: + +``` +apply all thread bt +``` + +## Check for blocking queries + +Sometimes the speed at which Sidekiq processes jobs can be so fast that it can +cause database contention. Check for blocking queries when backtraces above +show that many threads are stuck in the database adapter. + +The PostgreSQL wiki has details on the query you can run to see blocking +queries. The query is different based on PostgreSQL version. See +[Lock Monitoring](https://wiki.postgresql.org/wiki/Lock_Monitoring) for +the query details. diff --git a/doc/monitoring/performance/grafana_configuration.md b/doc/monitoring/performance/grafana_configuration.md index a79c8d48d3b..168bd85c26a 100644 --- a/doc/monitoring/performance/grafana_configuration.md +++ b/doc/monitoring/performance/grafana_configuration.md @@ -59,34 +59,53 @@ This will drop you in to an InfluxDB interactive session. Copy the entire contents below and paste it in to the interactive session: ``` -CREATE RETENTION POLICY gitlab_30d ON gitlab DURATION 30d REPLICATION 1 DEFAULT -CREATE RETENTION POLICY seven_days ON gitlab DURATION 7d REPLICATION 1 -CREATE CONTINUOUS QUERY rails_transaction_counts_seven_days ON gitlab BEGIN SELECT count("duration") AS "count" INTO gitlab.seven_days.rails_transaction_counts FROM rails_transactions GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_transaction_counts_seven_days ON gitlab BEGIN SELECT count("duration") AS "count" INTO gitlab.seven_days.sidekiq_transaction_counts FROM sidekiq_transactions GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_method_call_timings_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS "duration_95th", percentile("duration", 99.000) AS "duration_99th", mean("duration") AS "duration_mean" INTO gitlab.seven_days.rails_method_call_timings FROM rails_method_calls GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_method_call_timings_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS "duration_95th", percentile("duration", 99.000) AS "duration_99th", mean("duration") AS "duration_mean" INTO gitlab.seven_days.sidekiq_method_call_timings FROM sidekiq_method_calls GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_method_call_timings_per_method_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS duration_95th, percentile("duration", 99.000) AS duration_99th, mean("duration") AS duration_mean INTO gitlab.seven_days.rails_method_call_timings_per_method FROM rails_method_calls GROUP BY time(1m), method END; -CREATE CONTINUOUS QUERY sidekiq_method_call_timings_per_method_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS duration_95th, percentile("duration", 99.000) AS duration_99th, mean("duration") AS duration_mean INTO gitlab.seven_days.sidekiq_method_call_timings_per_method FROM sidekiq_method_calls GROUP BY time(1m), method END; -CREATE CONTINUOUS QUERY rails_memory_usage_per_minute ON gitlab BEGIN SELECT percentile(value, 95.000) AS memory_95th, percentile(value, 99.000) AS memory_99th, mean(value) AS memory_mean INTO gitlab.seven_days.rails_memory_usage_per_minute FROM rails_memory_usage GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_memory_usage_per_minute ON gitlab BEGIN SELECT percentile(value, 95.000) AS memory_95th, percentile(value, 99.000) AS memory_99th, mean(value) AS memory_mean INTO gitlab.seven_days.sidekiq_memory_usage_per_minute FROM sidekiq_memory_usage GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_file_descriptors_per_minute ON gitlab BEGIN SELECT sum(value) AS value INTO gitlab.seven_days.sidekiq_file_descriptors_per_minute FROM sidekiq_file_descriptors GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_file_descriptors_per_minute ON gitlab BEGIN SELECT sum(value) AS value INTO gitlab.seven_days.rails_file_descriptors_per_minute FROM rails_file_descriptors GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_gc_counts_per_minute ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.seven_days.rails_gc_counts_per_minute FROM rails_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_gc_counts_per_minute ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.seven_days.sidekiq_gc_counts_per_minute FROM sidekiq_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_gc_timings_per_minute ON gitlab BEGIN SELECT percentile(total_time, 95.000) AS duration_95th, percentile(total_time, 99.000) AS duration_99th, mean(total_time) AS duration_mean INTO gitlab.seven_days.rails_gc_timings_per_minute FROM rails_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_gc_timings_per_minute ON gitlab BEGIN SELECT percentile(total_time, 95.000) AS duration_95th, percentile(total_time, 99.000) AS duration_99th, mean(total_time) AS duration_mean INTO gitlab.seven_days.sidekiq_gc_timings_per_minute FROM sidekiq_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_gc_major_minor_per_minute ON gitlab BEGIN SELECT sum(major_gc_count) AS major, sum(minor_gc_count) AS minor INTO gitlab.seven_days.rails_gc_major_minor_per_minute FROM rails_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_gc_major_minor_per_minute ON gitlab BEGIN SELECT sum(major_gc_count) AS major, sum(minor_gc_count) AS minor INTO gitlab.seven_days.sidekiq_gc_major_minor_per_minute FROM sidekiq_gc_statistics GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_allowed_request_counts_per_minute ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.seven_days.grape_internal_allowed_request_counts_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/allowed' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_allowed_request_timings_per_minute ON gitlab BEGIN SELECT percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean("duration") AS duration_mean INTO gitlab.seven_days.grape_internal_allowed_request_timings_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/allowed' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_allowed_sql_timings_per_minute ON gitlab BEGIN SELECT percentile(sql_duration, 95) AS duration_95th, percentile(sql_duration, 99) AS duration_99th, mean(sql_duration) AS duration_mean INTO gitlab.seven_days.grape_internal_allowed_sql_timings_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/allowed' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_authorized_keys_request_counts_per_minute ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.seven_days.grape_internal_authorized_keys_request_counts_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/authorized_keys' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_authorized_keys_request_timings_per_minute ON gitlab BEGIN SELECT percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean("duration") AS duration_mean INTO gitlab.seven_days.grape_internal_authorized_keys_request_timings_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/authorized_keys' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_internal_authorized_keys_sql_timings_per_minute ON gitlab BEGIN SELECT percentile(sql_duration, 95) AS duration_95th, percentile(sql_duration, 99) AS duration_99th, mean(sql_duration) AS duration_mean INTO gitlab.seven_days.grape_internal_authorized_keys_sql_timings_per_minute FROM rails_transactions WHERE request_uri = '/api/v3/internal/authorized_keys' GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY rails_transaction_timings_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS duration_95th, percentile("duration", 99.000) AS duration_99th, mean("duration") AS duration_mean, percentile(sql_duration, 95.000) AS sql_duration_95th, percentile(sql_duration, 99.000) AS sql_duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(view_duration, 95.000) AS view_duration_95th, percentile(view_duration, 99.000) AS view_duration_99th, mean(view_duration) AS view_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(cache_duration) AS cache_duration_mean INTO gitlab.seven_days.rails_transaction_timings FROM rails_transactions GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY sidekiq_transaction_timings_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS duration_95th, percentile("duration", 99.000) AS duration_99th, mean("duration") AS duration_mean, percentile(sql_duration, 95.000) AS sql_duration_95th, percentile(sql_duration, 99.000) AS sql_duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(view_duration, 95.000) AS view_duration_95th, percentile(view_duration, 99.000) AS view_duration_99th, mean(view_duration) AS view_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(cache_duration) AS cache_duration_mean INTO gitlab.seven_days.sidekiq_transaction_timings FROM sidekiq_transactions GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_transaction_counts_seven_days ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.seven_days.grape_transaction_counts FROM rails_transactions WHERE action !~ /.+/ GROUP BY time(1m) END; -CREATE CONTINUOUS QUERY grape_transaction_timings_seven_days ON gitlab BEGIN SELECT percentile("duration", 95.000) AS duration_95th, percentile("duration", 99.000) AS duration_99th, mean("duration") AS duration_mean, percentile(sql_duration, 95.000) AS sql_duration_95th, percentile(sql_duration, 99.000) AS sql_duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(cache_duration) AS cache_duration_mean INTO gitlab.seven_days.grape_transaction_timings FROM rails_transactions WHERE action !~ /.+/ GROUP BY time(1m) END; +CREATE RETENTION POLICY default ON gitlab DURATION 1h REPLICATION 1 DEFAULT +CREATE RETENTION POLICY downsampled ON gitlab DURATION 7d REPLICATION 1 +CREATE CONTINUOUS QUERY grape_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.grape_git_timings_per_action FROM gitlab."default".rails_method_calls WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY grape_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.grape_markdown_render_timings_overall FROM gitlab."default".rails_transactions WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY grape_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.grape_markdown_render_timings_per_action FROM gitlab."default".rails_transactions WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY grape_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.grape_markdown_timings_overall FROM gitlab."default".rails_method_calls WHERE (action !~ /.+/ OR action =~ /^Grape#/) AND method =~ /^Banzai/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY grape_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.grape_method_call_timings_per_action_and_method FROM gitlab."default".rails_method_calls WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), method, action END; +CREATE CONTINUOUS QUERY grape_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.grape_method_call_timings_per_method FROM gitlab."default".rails_method_calls WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), method END; +CREATE CONTINUOUS QUERY grape_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.grape_transaction_counts_overall FROM gitlab."default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY grape_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.grape_transaction_counts_per_action FROM gitlab."default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY grape_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.grape_transaction_timings_overall FROM gitlab."default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY grape_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.grape_transaction_timings_per_action FROM gitlab."default".rails_transactions WHERE action !~ /.+/ OR action =~ /^Grape#/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY rails_file_descriptor_counts ON gitlab BEGIN SELECT sum(value) AS count INTO gitlab.downsampled.rails_file_descriptor_counts FROM gitlab."default".rails_file_descriptors GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_gc_counts ON gitlab BEGIN SELECT sum(count) AS total, sum(minor_gc_count) AS minor, sum(major_gc_count) AS major INTO gitlab.downsampled.rails_gc_counts FROM gitlab."default".rails_gc_statistics GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_gc_timings ON gitlab BEGIN SELECT mean(total_time) AS duration_mean, percentile(total_time, 95) AS duration_95th, percentile(total_time, 99) AS duration_99th INTO gitlab.downsampled.rails_gc_timings FROM gitlab."default".rails_gc_statistics GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.rails_git_timings_per_action FROM gitlab."default".rails_method_calls WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY rails_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.rails_markdown_render_timings_overall FROM gitlab."default".rails_transactions WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.rails_markdown_render_timings_per_action FROM gitlab."default".rails_transactions WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY rails_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.rails_markdown_timings_overall FROM gitlab."default".rails_method_calls WHERE (action =~ /.+/ AND action !~ /^Grape#/) AND method =~ /^Banzai/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_memory_usage_overall ON gitlab BEGIN SELECT mean(value) AS memory_mean, percentile(value, 95) AS memory_95th, percentile(value, 99) AS memory_99th INTO gitlab.downsampled.rails_memory_usage_overall FROM gitlab."default".rails_memory_usage GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.rails_method_call_timings_per_action_and_method FROM gitlab."default".rails_method_calls WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), method, action END; +CREATE CONTINUOUS QUERY rails_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.rails_method_call_timings_per_method FROM gitlab."default".rails_method_calls WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), method END; +CREATE CONTINUOUS QUERY rails_object_counts_overall ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.downsampled.rails_object_counts_overall FROM gitlab."default".rails_object_counts GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_object_counts_per_type ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.downsampled.rails_object_counts_per_type FROM gitlab."default".rails_object_counts GROUP BY time(1m), type END; +CREATE CONTINUOUS QUERY rails_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.rails_transaction_counts_overall FROM gitlab."default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.rails_transaction_counts_per_action FROM gitlab."default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY rails_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.rails_transaction_timings_overall FROM gitlab."default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY rails_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.rails_transaction_timings_per_action FROM gitlab."default".rails_transactions WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY rails_view_timings_per_action_and_view ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.rails_view_timings_per_action_and_view FROM gitlab."default".rails_views WHERE action =~ /.+/ AND action !~ /^Grape#/ GROUP BY time(1m), action, view END; +CREATE CONTINUOUS QUERY sidekiq_file_descriptor_counts ON gitlab BEGIN SELECT sum(value) AS count INTO gitlab.downsampled.sidekiq_file_descriptor_counts FROM gitlab."default".sidekiq_file_descriptors GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_gc_counts ON gitlab BEGIN SELECT sum(count) AS total, sum(minor_gc_count) AS minor, sum(major_gc_count) AS major INTO gitlab.downsampled.sidekiq_gc_counts FROM gitlab."default".sidekiq_gc_statistics GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_gc_timings ON gitlab BEGIN SELECT mean(total_time) AS duration_mean, percentile(total_time, 95) AS duration_95th, percentile(total_time, 99) AS duration_99th INTO gitlab.downsampled.sidekiq_gc_timings FROM gitlab."default".sidekiq_gc_statistics GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_git_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.sidekiq_git_timings_per_action FROM gitlab."default".sidekiq_method_calls WHERE method =~ /^(Rugged|Gitlab::Git)/ GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY sidekiq_markdown_render_timings_overall ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.sidekiq_markdown_render_timings_overall FROM gitlab."default".sidekiq_transactions WHERE (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_markdown_render_timings_per_action ON gitlab BEGIN SELECT mean(banzai_cached_render_real_time) AS cached_real_mean, percentile(banzai_cached_render_real_time, 95) AS cached_real_95th, percentile(banzai_cached_render_real_time, 99) AS cached_real_99th, mean(banzai_cached_render_cpu_time) AS cached_cpu_mean, percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th, percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th, sum(banzai_cached_render_call_count) AS cached_call_count, mean(banzai_cacheless_render_real_time) AS cacheless_real_mean, percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th, percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th, mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean, percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th, percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th, sum(banzai_cacheless_render_call_count) AS cacheless_call_count INTO gitlab.downsampled.sidekiq_markdown_render_timings_per_action FROM gitlab."default".sidekiq_transactions WHERE (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0) GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY sidekiq_markdown_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.sidekiq_markdown_timings_overall FROM gitlab."default".sidekiq_method_calls WHERE method =~ /^Banzai/ GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_memory_usage_overall ON gitlab BEGIN SELECT mean(value) AS memory_mean, percentile(value, 95) AS memory_95th, percentile(value, 99) AS memory_99th INTO gitlab.downsampled.sidekiq_memory_usage_overall FROM gitlab."default".sidekiq_memory_usage GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_method_call_timings_per_action_and_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.sidekiq_method_call_timings_per_action_and_method FROM gitlab."default".sidekiq_method_calls GROUP BY time(1m), method, action END; +CREATE CONTINUOUS QUERY sidekiq_method_call_timings_per_method ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.sidekiq_method_call_timings_per_method FROM gitlab."default".sidekiq_method_calls GROUP BY time(1m), method END; +CREATE CONTINUOUS QUERY sidekiq_object_counts_overall ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.downsampled.sidekiq_object_counts_overall FROM gitlab."default".sidekiq_object_counts GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_object_counts_per_type ON gitlab BEGIN SELECT sum(count) AS count INTO gitlab.downsampled.sidekiq_object_counts_per_type FROM gitlab."default".sidekiq_object_counts GROUP BY time(1m), type END; +CREATE CONTINUOUS QUERY sidekiq_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.sidekiq_transaction_counts_overall FROM gitlab."default".sidekiq_transactions GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_transaction_counts_per_action ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.sidekiq_transaction_counts_per_action FROM gitlab."default".sidekiq_transactions GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY sidekiq_transaction_timings_overall ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.sidekiq_transaction_timings_overall FROM gitlab."default".sidekiq_transactions GROUP BY time(1m) END; +CREATE CONTINUOUS QUERY sidekiq_transaction_timings_per_action ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th, mean(sql_duration) AS sql_duration_mean, percentile(sql_duration, 95) AS sql_duration_95th, percentile(sql_duration, 99) AS sql_duration_99th, mean(view_duration) AS view_duration_mean, percentile(view_duration, 95) AS view_duration_95th, percentile(view_duration, 99) AS view_duration_99th, mean(cache_read_duration) AS cache_read_duration_mean, percentile(cache_read_duration, 99) AS cache_read_duration_99th, percentile(cache_read_duration, 95) AS cache_read_duration_95th, mean(cache_write_duration) AS cache_write_duration_mean, percentile(cache_write_duration, 99) AS cache_write_duration_99th, percentile(cache_write_duration, 95) AS cache_write_duration_95th, mean(cache_delete_duration) AS cache_delete_duration_mean, percentile(cache_delete_duration, 99) AS cache_delete_duration_99th, percentile(cache_delete_duration, 95) AS cache_delete_duration_95th, mean(cache_exists_duration) AS cache_exists_duration_mean, percentile(cache_exists_duration, 99) AS cache_exists_duration_99th, percentile(cache_exists_duration, 95) AS cache_exists_duration_95th, mean(cache_duration) AS cache_duration_mean, percentile(cache_duration, 99) AS cache_duration_99th, percentile(cache_duration, 95) AS cache_duration_95th, mean(method_duration) AS method_duration_mean, percentile(method_duration, 99) AS method_duration_99th, percentile(method_duration, 95) AS method_duration_95th INTO gitlab.downsampled.sidekiq_transaction_timings_per_action FROM gitlab."default".sidekiq_transactions GROUP BY time(1m), action END; +CREATE CONTINUOUS QUERY sidekiq_view_timings_per_action_and_view ON gitlab BEGIN SELECT mean("duration") AS duration_mean, percentile("duration", 95) AS duration_95th, percentile("duration", 99) AS duration_99th INTO gitlab.downsampled.sidekiq_view_timings_per_action_and_view FROM gitlab."default".sidekiq_views GROUP BY time(1m), action, view END; +CREATE CONTINUOUS QUERY web_transaction_counts_overall ON gitlab BEGIN SELECT count("duration") AS count INTO gitlab.downsampled.web_transaction_counts_overall FROM gitlab."default".rails_transactions GROUP BY time(1m) END; ``` ## Import Dashboards diff --git a/doc/update/8.6-to-8.7.md b/doc/update/8.6-to-8.7.md index 8599133a726..4a2c6ea91d2 100644 --- a/doc/update/8.6-to-8.7.md +++ b/doc/update/8.6-to-8.7.md @@ -45,8 +45,8 @@ sudo -u git -H git checkout 8-7-stable-ee ```bash cd /home/git/gitlab-shell -sudo -u git -H git fetch --all -sudo -u git -H git checkout v2.7.0 +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v2.7.2 ``` ### 5. Update gitlab-workhorse diff --git a/doc/workflow/shortcuts.png b/doc/workflow/shortcuts.png Binary files differindex 83e562d6929..beb6c53ec77 100644 --- a/doc/workflow/shortcuts.png +++ b/doc/workflow/shortcuts.png diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index cf9938d25a7..ccca65cbe1c 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -103,10 +103,10 @@ module API required_attributes! [:hook_id] begin - @hook = ProjectHook.find(params[:hook_id]) - @hook.destroy + @hook = user_project.hooks.destroy(params[:hook_id]) rescue # ProjectHook can raise Error if hook_id not found + not_found!("Error deleting hook #{params[:hook_id]}") end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index ff9887cba1e..504d3df9d34 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -61,23 +61,21 @@ module Ci @stages = @config[:stages] || @config[:types] @variables = @config[:variables] || {} @cache = @config[:cache] + @jobs = {} + @config.except!(*ALLOWED_YAML_KEYS) + @config.each { |name, param| add_job(name, param) } - # anything that doesn't have script is considered as unknown - @config.each do |name, param| - raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) && param.has_key?(:script) - end + raise ValidationError, "Please define at least one job" if @jobs.none? + end - unless @config.values.any?{|job| job.is_a?(Hash)} - raise ValidationError, "Please define at least one job" - end + def add_job(name, job) + return if name.to_s.start_with?('.') - @jobs = {} - @config.each do |key, job| - next if key.to_s.start_with?('.') - stage = job[:stage] || job[:type] || DEFAULT_STAGE - @jobs[key] = { stage: stage }.merge(job) - end + raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script) + + stage = job[:stage] || job[:type] || DEFAULT_STAGE + @jobs[name] = { stage: stage }.merge(job) end def build_job(name, job) @@ -112,8 +110,6 @@ module Ci true end - private - def validate_global! unless validate_array_of_strings(@before_script) raise ValidationError, "before_script should be an array of strings" diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5ebaad6ca6e..ab900b641c4 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -6,6 +6,7 @@ module Gitlab gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.shortcuts_path = help_shortcuts_path gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class if current_user diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 1324e4cd267..d521de28e8a 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -61,7 +61,8 @@ server { error_page 422 /422.html; error_page 500 /500.html; error_page 502 /502.html; - location ~ ^/(404|422|500|502)\.html$ { + error_page 503 /503.html; + location ~ ^/(404|422|500|502|503)\.html$ { root /home/git/gitlab/public; internal; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index af6ea9ed706..bf014b56cf6 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -105,7 +105,8 @@ server { error_page 422 /422.html; error_page 500 /500.html; error_page 502 /502.html; - location ~ ^/(404|422|500|502)\.html$ { + error_page 503 /503.html; + location ~ ^/(404|422|500|502|503)\.html$ { root /home/git/gitlab/public; internal; } diff --git a/public/503.html b/public/503.html new file mode 100644 index 00000000000..6ab1185658d --- /dev/null +++ b/public/503.html @@ -0,0 +1,54 @@ +<!DOCTYPE html> +<html> +<head> + <title>GitLab is not responding (503)</title> + <style> + body { + color: #666; + text-align: center; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + margin: 0; + width: 800px; + margin: auto; + font-size: 14px; + } + + h1 { + font-size: 56px; + line-height: 100px; + font-weight: normal; + color: #456; + } + + h2 { + font-size: 24px; + color: #666; + line-height: 1.5em; + } + + h3 { + color: #456; + font-size: 20px; + font-weight: normal; + line-height: 28px; + } + + hr { + margin: 18px 0; + border: 0; + border-top: 1px solid #EEE; + border-bottom: 1px solid white; + } + </style> +</head> +<body> + <h1> + <img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjEwIiBoZWlnaHQ9IjIxMCIgdmlld0JveD0iMCAwIDIxMCAyMTAiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CiAgPHBhdGggZD0iTTEwNS4wNjE0IDIwMy42NTVsMzguNjQtMTE4LjkyMWgtNzcuMjhsMzguNjQgMTE4LjkyMXoiIGZpbGw9IiNlMjQzMjkiLz4KICA8cGF0aCBkPSJNMTA1LjA2MTQgMjAzLjY1NDhsLTM4LjY0LTExOC45MjFoLTU0LjE1M2w5Mi43OTMgMTE4LjkyMXoiIGZpbGw9IiNmYzZkMjYiLz4KICA8cGF0aCBkPSJNMTIuMjY4NSA4NC43MzQxbC0xMS43NDIgMzYuMTM5Yy0xLjA3MSAzLjI5Ni4xMDIgNi45MDcgMi45MDYgOC45NDRsMTAxLjYyOSA3My44MzgtOTIuNzkzLTExOC45MjF6IiBmaWxsPSIjZmNhMzI2Ii8+CiAgPHBhdGggZD0iTTEyLjI2ODUgODQuNzM0Mmg1NC4xNTNsLTIzLjI3My03MS42MjVjLTEuMTk3LTMuNjg2LTYuNDExLTMuNjg1LTcuNjA4IDBsLTIzLjI3MiA3MS42MjV6IiBmaWxsPSIjZTI0MzI5Ii8+CiAgPHBhdGggZD0iTTEwNS4wNjE0IDIwMy42NTQ4bDM4LjY0LTExOC45MjFoNTQuMTUzbC05Mi43OTMgMTE4LjkyMXoiIGZpbGw9IiNmYzZkMjYiLz4KICA8cGF0aCBkPSJNMTk3Ljg1NDQgODQuNzM0MWwxMS43NDIgMzYuMTM5YzEuMDcxIDMuMjk2LS4xMDIgNi45MDctMi45MDYgOC45NDRsLTEwMS42MjkgNzMuODM4IDkyLjc5My0xMTguOTIxeiIgZmlsbD0iI2ZjYTMyNiIvPgogIDxwYXRoIGQ9Ik0xOTcuODU0NCA4NC43MzQyaC01NC4xNTNsMjMuMjczLTcxLjYyNWMxLjE5Ny0zLjY4NiA2LjQxMS0zLjY4NSA3LjYwOCAwbDIzLjI3MiA3MS42MjV6IiBmaWxsPSIjZTI0MzI5Ii8+Cjwvc3ZnPgo=" alt="GitLab Logo"/><br /> + 503 + </h1> + <h3>Whoops, GitLab is currently unavailable.</h3> + <hr/> + <p>Try refreshing the page, or going back and attempting the action again.</p> + <p>Please contact your GitLab administrator if this problem persists.</p> +</body> +</html> diff --git a/spec/features/dashboard/user_filters_projects_spec.rb b/spec/features/dashboard/user_filters_projects_spec.rb new file mode 100644 index 00000000000..cf86e2c85e9 --- /dev/null +++ b/spec/features/dashboard/user_filters_projects_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe "Dashboard > User filters projects", feature: true do + + describe 'filtering personal projects' do + before do + user = create(:user) + project = create(:project, name: "Victorialand", namespace: user.namespace) + project.team << [user, :master] + + user2 = create(:user) + project2 = create(:project, name: "Treasure", namespace: user2.namespace) + project2.team << [user, :developer] + + login_as(user) + visit dashboard_projects_path + end + + it 'filters by projects "Owned by me"' do + click_link "Owned by me" + + expect(page).to have_css('.is-active', text: 'Owned by me') + expect(page).to have_content('Victorialand') + expect(page).not_to have_content('Treasure') + end + end +end diff --git a/spec/features/dashboard_filter_spec.rb b/spec/features/dashboard_filter_spec.rb deleted file mode 100644 index 3d20813161a..00000000000 --- a/spec/features/dashboard_filter_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -require 'spec_helper' - -describe "Dashboard projects filters", feature: true, js: true do - - context 'filtering personal projects' do - before do - user = create(:user) - project = create(:project, name: "Victorialand", namespace: user.namespace) - project.team << [user, :master] - - login_as(user) - visit dashboard_projects_path - - open_filter_dropdown - click_link "Owned by me" - end - - it 'filters by projects "Owned by me"' do - sleep 1 - open_filter_dropdown - page.within('ul.dropdown-menu.dropdown-menu-align-right') do - expect(page).to have_css('.is-active', text: 'Owned by me') - end - end - end - - def open_filter_dropdown - find('button.dropdown-toggle.btn').click - end -end diff --git a/spec/features/issues_spec.rb b/spec/features/issues_spec.rb index b1d49875ac4..b57131f68d5 100644 --- a/spec/features/issues_spec.rb +++ b/spec/features/issues_spec.rb @@ -317,6 +317,27 @@ describe 'Issues', feature: true do expect(issue.reload.assignee).to be_nil end + + it 'allows user to select an assignee', js: true do + issue2 = create(:issue, project: project, author: @user) + visit namespace_project_issue_path(project.namespace, project, issue2) + + page.within('.assignee') do + expect(page).to have_content "No assignee" + end + + page.within '.assignee' do + click_link 'Edit' + end + + page.within '.dropdown-menu-user' do + click_link @user.name + end + + page.within('.assignee') do + expect(page).to have_content @user.name + end + end end context 'by unauthorized user' do diff --git a/spec/features/project/shortcuts_spec.rb b/spec/features/project/shortcuts_spec.rb new file mode 100644 index 00000000000..2595c4181e5 --- /dev/null +++ b/spec/features/project/shortcuts_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +feature 'Project shortcuts', feature: true do + let(:project) { create(:project) } + let(:user) { create(:user) } + + describe 'On a project', js: true do + before do + project.team << [user, :master] + login_as user + visit namespace_project_path(project.namespace, project) + end + + describe 'pressing "i"' do + it 'redirects to new issue page' do + find('body').native.send_key('i') + expect(page).to have_content('New Issue') + end + end + end +end diff --git a/spec/features/projects/commit/builds_spec.rb b/spec/features/projects/commit/builds_spec.rb new file mode 100644 index 00000000000..40ba0bdc115 --- /dev/null +++ b/spec/features/projects/commit/builds_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +feature 'project commit builds' do + given(:project) { create(:project) } + + background do + user = create(:user) + project.team << [user, :master] + login_as(user) + end + + context 'when no builds triggered yet' do + background do + create(:ci_commit, project: project, + sha: project.commit.sha, + ref: 'master') + end + + scenario 'user views commit builds page' do + visit builds_namespace_project_commit_path(project.namespace, + project, project.commit.sha) + + + expect(page).to have_content('Builds') + end + end +end diff --git a/spec/features/project/commits/cherry_pick_spec.rb b/spec/features/projects/commits/cherry_pick_spec.rb index 0559b02f321..0559b02f321 100644 --- a/spec/features/project/commits/cherry_pick_spec.rb +++ b/spec/features/projects/commits/cherry_pick_spec.rb diff --git a/spec/features/projects/members/anonymous_user_sees_members_spec.rb b/spec/features/projects/members/anonymous_user_sees_members_spec.rb new file mode 100644 index 00000000000..c5e3d143d91 --- /dev/null +++ b/spec/features/projects/members/anonymous_user_sees_members_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +feature 'Projects > Members > Anonymous user sees members', feature: true do + let(:user) { create(:user) } + let(:group) { create(:group, :public) } + let(:project) { create(:empty_project, :public) } + + background do + project.team << [user, :master] + create(:project_group_link, project: project, group: group) + end + + scenario "anonymous user visits the project's members page and sees the list of members" do + visit namespace_project_project_members_path(project.namespace, project) + + expect(current_path).to eq( + namespace_project_project_members_path(project.namespace, project)) + expect(page).to have_content(user.name) + end +end diff --git a/spec/features/todos/todos_spec.rb b/spec/features/todos/todos_spec.rb index 113d4c40cfc..248e004ba6e 100644 --- a/spec/features/todos/todos_spec.rb +++ b/spec/features/todos/todos_spec.rb @@ -1,12 +1,10 @@ require 'spec_helper' describe 'Dashboard Todos', feature: true do - let(:user){ create(:user) } - let(:author){ create(:user) } - let(:project){ create(:project) } - let(:issue){ create(:issue) } - let(:todos_per_page){ Todo.default_per_page } - let(:todos_total){ todos_per_page + 1 } + let(:user) { create(:user) } + let(:author) { create(:user) } + let(:project) { create(:project) } + let(:issue) { create(:issue) } describe 'GET /dashboard/todos' do context 'User does not have todos' do @@ -46,31 +44,35 @@ describe 'Dashboard Todos', feature: true do end context 'User has multiple pages of Todos' do - let(:todo_total_pages){ (todos_total.to_f/todos_per_page).ceil } - before do - todos_total.times do - create(:todo, :mentioned, user: user, project: project, target: issue, author: author) - end + allow(Todo).to receive(:default_per_page).and_return(1) + + # Create just enough records to cause us to paginate + create_list(:todo, 2, :mentioned, user: user, project: project, target: issue, author: author) login_as(user) - visit dashboard_todos_path end it 'is paginated' do + visit dashboard_todos_path + expect(page).to have_selector('.gl-pagination') end it 'is has the right number of pages' do - expect(page).to have_selector('.gl-pagination .page', count: todo_total_pages) + visit dashboard_todos_path + + expect(page).to have_selector('.gl-pagination .page', count: 2) end - describe 'deleting last todo from last page', js: true do + describe 'completing last todo from last page', js: true do it 'redirects to the previous page' do - page.within('.gl-pagination') do - click_link todo_total_pages.to_s - end - first('.done-todo').click + visit dashboard_todos_path(page: 2) + expect(page).to have_content(Todo.first.body) + + click_link('Done') + + expect(current_path).to eq dashboard_todos_path expect(page).to have_content(Todo.last.body) end end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index c258cfebd73..62389188d2c 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -105,4 +105,30 @@ describe ProjectsHelper do end end end + + describe '#license_short_name' do + let(:project) { create(:project) } + + context 'when project.repository has a license_key' do + it 'returns the nickname of the license if present' do + allow(project.repository).to receive(:license_key).and_return('agpl-3.0') + + expect(helper.license_short_name(project)).to eq('GNU AGPLv3') + end + + it 'returns the name of the license if nickname is not present' do + allow(project.repository).to receive(:license_key).and_return('mit') + + expect(helper.license_short_name(project)).to eq('MIT License') + end + end + + context 'when project.repository has no license_key but a license_blob' do + it 'returns LICENSE' do + allow(project.repository).to receive(:license_key).and_return(nil) + + expect(helper.license_short_name(project)).to eq('LICENSE') + end + end + end end diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index 643acf0303c..c7ab3185378 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -648,70 +648,131 @@ module Ci end describe "Hidden jobs" do - let(:config) do - YAML.dump({ - '.hidden_job' => { script: 'test' }, - 'normal_job' => { script: 'test' } - }) + let(:config_processor) { GitlabCiYamlProcessor.new(config) } + subject { config_processor.builds_for_stage_and_ref("test", "master") } + + shared_examples 'hidden_job_handling' do + it "doesn't create jobs that start with dot" do + expect(subject.size).to eq(1) + expect(subject.first).to eq({ + except: nil, + stage: "test", + stage_idx: 1, + name: :normal_job, + only: nil, + commands: "test", + tag_list: [], + options: {}, + when: "on_success", + allow_failure: false + }) + end end - let(:config_processor) { GitlabCiYamlProcessor.new(config) } + context 'when hidden job have a script definition' do + let(:config) do + YAML.dump({ + '.hidden_job' => { image: 'ruby:2.1', script: 'test' }, + 'normal_job' => { script: 'test' } + }) + end - subject { config_processor.builds_for_stage_and_ref("test", "master") } + it_behaves_like 'hidden_job_handling' + end - it "doesn't create jobs that starts with dot" do - expect(subject.size).to eq(1) - expect(subject.first).to eq({ - except: nil, - stage: "test", - stage_idx: 1, - name: :normal_job, - only: nil, - commands: "test", - tag_list: [], - options: {}, - when: "on_success", - allow_failure: false - }) + context "when hidden job doesn't have a script definition" do + let(:config) do + YAML.dump({ + '.hidden_job' => { image: 'ruby:2.1' }, + 'normal_job' => { script: 'test' } + }) + end + + it_behaves_like 'hidden_job_handling' end end describe "YAML Alias/Anchor" do - it "is correctly supported for jobs" do - config = <<EOT + let(:config_processor) { GitlabCiYamlProcessor.new(config) } + subject { config_processor.builds_for_stage_and_ref("build", "master") } + + shared_examples 'job_templates_handling' do + it "is correctly supported for jobs" do + expect(subject.size).to eq(2) + expect(subject.first).to eq({ + except: nil, + stage: "build", + stage_idx: 0, + name: :job1, + only: nil, + commands: "execute-script-for-job", + tag_list: [], + options: {}, + when: "on_success", + allow_failure: false + }) + expect(subject.second).to eq({ + except: nil, + stage: "build", + stage_idx: 0, + name: :job2, + only: nil, + commands: "execute-script-for-job", + tag_list: [], + options: {}, + when: "on_success", + allow_failure: false + }) + end + end + + context 'when template is a job' do + let(:config) do + <<EOT job1: &JOBTMPL + stage: build script: execute-script-for-job job2: *JOBTMPL EOT + end - config_processor = GitlabCiYamlProcessor.new(config) + it_behaves_like 'job_templates_handling' + end - expect(config_processor.builds_for_stage_and_ref("test", "master").size).to eq(2) - expect(config_processor.builds_for_stage_and_ref("test", "master").first).to eq({ - except: nil, - stage: "test", - stage_idx: 1, - name: :job1, - only: nil, - commands: "execute-script-for-job", - tag_list: [], - options: {}, - when: "on_success", - allow_failure: false - }) - expect(config_processor.builds_for_stage_and_ref("test", "master").second).to eq({ - except: nil, - stage: "test", - stage_idx: 1, - name: :job2, - only: nil, - commands: "execute-script-for-job", - tag_list: [], - options: {}, - when: "on_success", - allow_failure: false - }) + context 'when template is a hidden job' do + let(:config) do + <<EOT +.template: &JOBTMPL + stage: build + script: execute-script-for-job + +job1: *JOBTMPL + +job2: *JOBTMPL +EOT + end + + it_behaves_like 'job_templates_handling' + end + + context 'when job adds its own keys to a template definition' do + let(:config) do + <<EOT +.template: &JOBTMPL + stage: build + +job1: + <<: *JOBTMPL + script: execute-script-for-job + +job2: + <<: *JOBTMPL + script: execute-script-for-job +EOT + end + + it_behaves_like 'job_templates_handling' end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index becc743de31..e33c7d62ff4 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -719,11 +719,8 @@ describe Project, models: true do with('foo.wiki', project). and_return(wiki) - expect(repo).to receive(:expire_cache) - expect(repo).to receive(:expire_emptiness_caches) - - expect(wiki).to receive(:expire_cache) - expect(wiki).to receive(:expire_emptiness_caches) + expect(repo).to receive(:before_delete) + expect(wiki).to receive(:before_delete) project.expire_caches_before_rename('foo') end diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index b561aa663d1..c19524a01f8 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -132,7 +132,6 @@ describe Repository, models: true do it { expect(subject.basename).to eq('a/b/c') } end end - end describe '#license_blob' do @@ -148,39 +147,18 @@ describe Repository, models: true do expect(repository.license_blob).to be_nil end - it 'favors license file with no extension' do - repository.commit_file(user, 'LICENSE', Licensee::License.new('mit').content, 'Add LICENSE', 'master', false) - repository.commit_file(user, 'LICENSE.md', Licensee::License.new('mit').content, 'Add LICENSE.md', 'master', false) - - expect(repository.license_blob.name).to eq('LICENSE') - end - - it 'favors .md file to .txt' do - repository.commit_file(user, 'LICENSE.md', Licensee::License.new('mit').content, 'Add LICENSE.md', 'master', false) - repository.commit_file(user, 'LICENSE.txt', Licensee::License.new('mit').content, 'Add LICENSE.txt', 'master', false) - - expect(repository.license_blob.name).to eq('LICENSE.md') - end - - it 'favors LICENCE to LICENSE' do - repository.commit_file(user, 'LICENSE', Licensee::License.new('mit').content, 'Add LICENSE', 'master', false) - repository.commit_file(user, 'LICENCE', Licensee::License.new('mit').content, 'Add LICENCE', 'master', false) - - expect(repository.license_blob.name).to eq('LICENCE') - end - - it 'favors LICENSE to COPYING' do - repository.commit_file(user, 'LICENSE', Licensee::License.new('mit').content, 'Add LICENSE', 'master', false) - repository.commit_file(user, 'COPYING', Licensee::License.new('mit').content, 'Add COPYING', 'master', false) + it 'detects license file with no recognizable open-source license content' do + repository.commit_file(user, 'LICENSE', 'Copyright!', 'Add LICENSE', 'master', false) expect(repository.license_blob.name).to eq('LICENSE') end - it 'favors LICENCE to COPYING' do - repository.commit_file(user, 'LICENCE', Licensee::License.new('mit').content, 'Add LICENCE', 'master', false) - repository.commit_file(user, 'COPYING', Licensee::License.new('mit').content, 'Add COPYING', 'master', false) + %w[LICENSE LICENCE LiCensE LICENSE.md LICENSE.foo COPYING COPYING.md].each do |filename| + it "detects '#{filename}'" do + repository.commit_file(user, filename, Licensee::License.new('mit').content, "Add #{filename}", 'master', false) - expect(repository.license_blob.name).to eq('LICENCE') + expect(repository.license_blob.name).to eq(filename) + end end end @@ -190,8 +168,14 @@ describe Repository, models: true do repository.remove_file(user, 'LICENSE', 'Remove LICENSE', 'master') end - it 'returns "no-license" when no license is detected' do - expect(repository.license_key).to eq('no-license') + it 'returns nil when no license is detected' do + expect(repository.license_key).to be_nil + end + + it 'detects license file with no recognizable open-source license content' do + repository.commit_file(user, 'LICENSE', 'Copyright!', 'Add LICENSE', 'master', false) + + expect(repository.license_key).to be_nil end it 'returns the license key' do diff --git a/spec/requests/api/project_hooks_spec.rb b/spec/requests/api/project_hooks_spec.rb index 142b637d291..ffb93bbb120 100644 --- a/spec/requests/api/project_hooks_spec.rb +++ b/spec/requests/api/project_hooks_spec.rb @@ -148,14 +148,24 @@ describe API::API, 'ProjectHooks', api: true do expect(response.status).to eq(200) end - it "should return success when deleting non existent hook" do + it "should return a 404 error when deleting non existent hook" do delete api("/projects/#{project.id}/hooks/42", user) - expect(response.status).to eq(200) + expect(response.status).to eq(404) end it "should return a 405 error if hook id not given" do delete api("/projects/#{project.id}/hooks", user) expect(response.status).to eq(405) end + + it "shold return a 404 if a user attempts to delete project hooks he/she does not own" do + test_user = create(:user) + other_project = create(:project) + other_project.team << [test_user, :master] + + delete api("/projects/#{other_project.id}/hooks/#{hook.id}", test_user) + expect(response.status).to eq(404) + expect(WebHook.exists?(hook.id)).to be_truthy + end end end |