diff options
108 files changed, 3451 insertions, 158 deletions
diff --git a/CHANGELOG b/CHANGELOG index 9df1994665b..effa46399ab 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -21,6 +21,7 @@ v 8.12.0 (unreleased) - Pass the "Remember me" value to the U2F authentication form - Display stages in valid order in stages dropdown on build page - Only update projects.last_activity_at once per hour when creating a new event + - Cycle analytics (first iteration) !5986 - Remove vendor prefixes for linear-gradient CSS (ClemMakesApps) - Move pushes_since_gc from the database to Redis - Add font color contrast to external label in admin area (ClemMakesApps) @@ -50,6 +51,7 @@ v 8.12.0 (unreleased) - Expose `sha` and `merge_commit_sha` in merge request API (Ben Boeckel) - Set path for all JavaScript cookies to honor GitLab's subdirectory setting !5627 (Mike Greiling) - Fix blame table layout width + - Spec testing if issue authors can read issues on private projects - Fix bug where pagination is still displayed despite all todos marked as done (ClemMakesApps) - Request only the LDAP attributes we need !6187 - Center build stage columns in pipeline overview (ClemMakesApps) @@ -76,6 +78,8 @@ v 8.12.0 (unreleased) - Require confirmation when not logged in for unsubscribe links !6223 (Maximiliano Perez Coto) - Add `wiki_page_events` to project hook APIs (Ben Boeckel) - Remove Gitorious import + - Loads GFM autocomplete source only when required + - Fix issue with slash commands not loading on new issue page - Fix inconsistent background color for filter input field (ClemMakesApps) - Remove prefixes from transition CSS property (ClemMakesApps) - Add Sentry logging to API calls diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 1545d966571..40c341bdcdb 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -3.5.0 +3.6.0 @@ -135,8 +135,7 @@ gem 'after_commit_queue', '~> 1.3.0' gem 'acts-as-taggable-on', '~> 3.4' # Background jobs -gem 'sinatra', '~> 1.4.4', require: false -gem 'sidekiq', '~> 4.0' +gem 'sidekiq', '~> 4.2' gem 'sidekiq-cron', '~> 0.4.0' gem 'redis-namespace', '~> 1.5.2' @@ -320,6 +319,7 @@ group :test do gem 'webmock', '~> 1.21.0' gem 'test_after_commit', '~> 0.4.2' gem 'sham_rack', '~> 1.3.6' + gem 'timecop', '~> 0.8.0' end group :production do diff --git a/Gemfile.lock b/Gemfile.lock index 8e26429df14..c9b4b30c1f2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -673,11 +673,11 @@ GEM rack shoulda-matchers (2.8.0) activesupport (>= 3.0.0) - sidekiq (4.1.4) + sidekiq (4.2.1) concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) + rack-protection (~> 1.5) redis (~> 3.2, >= 3.2.1) - sinatra (>= 1.4.7) sidekiq-cron (0.4.0) redis-namespace (>= 1.5.2) rufus-scheduler (>= 2.0.24) @@ -687,10 +687,6 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.0) - sinatra (1.4.7) - rack (~> 1.5) - rack-protection (~> 1.4) - tilt (>= 1.3, < 3) slack-notifier (1.2.1) slop (3.6.0) spinach (0.8.10) @@ -960,10 +956,9 @@ DEPENDENCIES settingslogic (~> 2.0.9) sham_rack (~> 1.3.6) shoulda-matchers (~> 2.8.0) - sidekiq (~> 4.0) + sidekiq (~> 4.2) sidekiq-cron (~> 0.4.0) simplecov (= 0.12.0) - sinatra (~> 1.4.4) slack-notifier (~> 1.2.0) spinach-rails (~> 0.2.1) spinach-rerun-reporter (~> 0.0.2) @@ -980,6 +975,7 @@ DEPENDENCIES teaspoon-jasmine (~> 2.2.0) test_after_commit (~> 0.4.2) thin (~> 1.7.0) + timecop (~> 0.8.0) turbolinks (~> 2.5.0) u2f (~> 0.2.1) uglifier (~> 2.7.2) @@ -995,4 +991,4 @@ DEPENDENCIES wikicloth (= 0.8.1) BUNDLED WITH - 1.12.5 + 1.13.0 diff --git a/app/assets/javascripts/api.js b/app/assets/javascripts/api.js index 6df2ecf57a2..1cd2302111e 100644 --- a/app/assets/javascripts/api.js +++ b/app/assets/javascripts/api.js @@ -16,9 +16,6 @@ .replace(':id', group_id); return $.ajax({ url: url, - data: { - private_token: gon.api_token - }, dataType: "json" }).done(function(group) { return callback(group); @@ -31,7 +28,6 @@ return $.ajax({ url: url, data: { - private_token: gon.api_token, search: query, per_page: 20 }, @@ -46,7 +42,6 @@ return $.ajax({ url: url, data: { - private_token: gon.api_token, search: query, per_page: 20 }, @@ -61,7 +56,6 @@ return $.ajax({ url: url, data: { - private_token: gon.api_token, search: query, order_by: order, per_page: 20 @@ -74,7 +68,6 @@ newLabel: function(project_id, data, callback) { var url = Api.buildUrl(Api.labelsPath) .replace(':id', project_id); - data.private_token = gon.api_token; return $.ajax({ url: url, type: "POST", @@ -93,7 +86,6 @@ return $.ajax({ url: url, data: { - private_token: gon.api_token, search: query, per_page: 20 }, diff --git a/app/assets/javascripts/cycle-analytics.js.es6 b/app/assets/javascripts/cycle-analytics.js.es6 new file mode 100644 index 00000000000..afaed7c4f60 --- /dev/null +++ b/app/assets/javascripts/cycle-analytics.js.es6 @@ -0,0 +1,92 @@ +((global) => { + + const COOKIE_NAME = 'cycle_analytics_help_dismissed'; + + gl.CycleAnalytics = class CycleAnalytics { + constructor() { + const that = this; + + this.isHelpDismissed = $.cookie(COOKIE_NAME); + this.vue = new Vue({ + el: '#cycle-analytics', + name: 'CycleAnalytics', + created: this.fetchData(), + data: this.decorateData({ isLoading: true }), + methods: { + dismissLanding() { + that.dismissLanding(); + } + } + }); + } + + fetchData(options) { + options = options || { startDate: 30 }; + + $.ajax({ + url: $('#cycle-analytics').data('request-path'), + method: 'GET', + dataType: 'json', + contentType: 'application/json', + data: { start_date: options.startDate } + }).done((data) => { + this.vue.$data = this.decorateData(data); + this.initDropdown(); + }) + .error((data) => { + this.handleError(data); + }) + .always(() => { + this.vue.isLoading = false; + }) + } + + decorateData(data) { + data.summary = data.summary || []; + data.stats = data.stats || []; + data.isHelpDismissed = this.isHelpDismissed; + data.isLoading = data.isLoading || false; + + data.summary.forEach((item) => { + item.value = item.value || '-'; + }); + + data.stats.forEach((item) => { + item.value = item.value || '- - -'; + }) + + return data; + } + + handleError(data) { + this.vue.$data = { + hasError: true, + isHelpDismissed: this.isHelpDismissed + }; + + new Flash('There was an error while fetching cycle analytics data.', 'alert'); + } + + dismissLanding() { + this.vue.isHelpDismissed = true; + $.cookie(COOKIE_NAME, true); + } + + initDropdown() { + const $dropdown = $('.js-ca-dropdown'); + const $label = $dropdown.find('.dropdown-label'); + + $dropdown.find('li a').off('click').on('click', (e) => { + e.preventDefault(); + const $target = $(e.currentTarget); + const value = $target.data('value'); + + $label.text($target.text().trim()); + this.vue.isLoading = true; + this.fetchData({ startDate: value }); + }) + } + + } + +})(window.gl || (window.gl = {})); diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index 99b16f7d59b..ddf11ecf34c 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -94,6 +94,11 @@ break; case "projects:merge_requests:conflicts": window.mcui = new MergeConflictResolver() + break; + case 'projects:merge_requests:index': + shortcut_handler = new ShortcutsNavigation(); + Issuable.init(); + break; case 'dashboard:activity': new Activities(); break; @@ -185,6 +190,9 @@ new gl.ProtectedBranchCreate(); new gl.ProtectedBranchEditList(); break; + case 'projects:cycle_analytics:show': + new gl.CycleAnalytics(); + break; } switch (path.first()) { case 'admin': diff --git a/app/assets/javascripts/issuable.js.es6 b/app/assets/javascripts/issuable.js.es6 index 53faaa38a0c..81d89a48227 100644 --- a/app/assets/javascripts/issuable.js.es6 +++ b/app/assets/javascripts/issuable.js.es6 @@ -16,11 +16,11 @@ }, initSearch: function() { this.timer = null; - return $('#issue_search').off('keyup').on('keyup', function() { + return $('#issuable_search').off('keyup').on('keyup', function() { clearTimeout(this.timer); return this.timer = setTimeout(function() { var $form, $input, $search; - $search = $('#issue_search'); + $search = $('#issuable_search'); $form = $('.js-filter-form'); $input = $("input[name='" + ($search.attr('name')) + "']", $form); if ($input.length === 0) { diff --git a/app/assets/stylesheets/pages/cycle_analytics.scss b/app/assets/stylesheets/pages/cycle_analytics.scss new file mode 100644 index 00000000000..21e19c97632 --- /dev/null +++ b/app/assets/stylesheets/pages/cycle_analytics.scss @@ -0,0 +1,121 @@ +#cycle-analytics { + margin: 24px auto 0; + width: 800px; + position: relative; + + .panel { + + .content-block { + padding: 24px 0; + border-bottom: none; + position: relative; + } + + .column { + text-align: center; + + .header { + font-size: 30px; + line-height: 38px; + font-weight: normal; + margin: 0; + } + + .text { + color: $layout-link-gray; + margin: 0; + } + + &:last-child { + text-align: right; + } + } + + .dropdown { + position: relative; + top: 13px; + } + } + + .bordered-box { + border: 1px solid $border-color; + @include border-radius($border-radius-default); + position: relative; + } + + .content-list { + li { + padding: 18px $gl-padding $gl-padding; + + .container-fluid { + padding: 0; + } + } + + .title-col { + p { + margin: 0; + + &.title { + line-height: 19px; + font-size: 15px; + font-weight: 600; + } + &:text { + color: #8c8c8c; + } + } + } + + .value-col { + text-align: right; + + span { + line-height: 42px; + } + } + } + + .landing { + margin-bottom: $gl-padding; + overflow: hidden; + + .dismiss-icon { + position: absolute; + right: $gl-padding; + cursor: pointer; + color: #b2b2b2; + } + + svg { + margin: 0 20px; + float: left; + width: 136px; + height: 136px; + } + + .inner-content { + width: 480px; + float: left; + + h4 { + color: $gl-text-color; + font-size: 17px; + } + + p { + color: #8c8c8c; + margin-bottom: $gl-padding; + } + } + } + + .fa-spinner { + font-size: 28px; + position: relative; + margin-left: -20px; + left: 50%; + margin-top: 36px; + } + +} diff --git a/app/controllers/concerns/issuable_collections.rb b/app/controllers/concerns/issuable_collections.rb index b5e79099e39..4a447735fa7 100644 --- a/app/controllers/concerns/issuable_collections.rb +++ b/app/controllers/concerns/issuable_collections.rb @@ -13,10 +13,18 @@ module IssuableCollections issues_finder.execute end + def all_issues_collection + IssuesFinder.new(current_user, filter_params_all).execute + end + def merge_requests_collection merge_requests_finder.execute end + def all_merge_requests_collection + MergeRequestsFinder.new(current_user, filter_params_all).execute + end + def issues_finder @issues_finder ||= issuable_finder_for(IssuesFinder) end @@ -54,6 +62,10 @@ module IssuableCollections @filter_params end + def filter_params_all + @filter_params_all ||= filter_params.merge(state: 'all', sort: nil) + end + def set_default_scope params[:scope] = 'all' if params[:scope].blank? end diff --git a/app/controllers/concerns/issues_action.rb b/app/controllers/concerns/issues_action.rb index b89fb94be6e..eced9d9d678 100644 --- a/app/controllers/concerns/issues_action.rb +++ b/app/controllers/concerns/issues_action.rb @@ -10,6 +10,8 @@ module IssuesAction .preload(:author, :project) .page(params[:page]) + @all_issues = all_issues_collection.non_archived + respond_to do |format| format.html format.atom { render layout: false } diff --git a/app/controllers/concerns/merge_requests_action.rb b/app/controllers/concerns/merge_requests_action.rb index a1b0eee37f9..729763169e2 100644 --- a/app/controllers/concerns/merge_requests_action.rb +++ b/app/controllers/concerns/merge_requests_action.rb @@ -9,5 +9,7 @@ module MergeRequestsAction .non_archived .preload(:author, :target_project) .page(params[:page]) + + @all_merge_requests = all_merge_requests_collection.non_archived end end diff --git a/app/controllers/jwt_controller.rb b/app/controllers/jwt_controller.rb index 06d96774754..34d5d99558e 100644 --- a/app/controllers/jwt_controller.rb +++ b/app/controllers/jwt_controller.rb @@ -11,10 +11,8 @@ class JwtController < ApplicationController service = SERVICES[params[:service]] return head :not_found unless service - @authentication_result ||= Gitlab::Auth::Result.new - result = service.new(@authentication_result.project, @authentication_result.actor, auth_params). - execute(authentication_abilities: @authentication_result.authentication_abilities) + execute(authentication_abilities: @authentication_result.authentication_abilities || []) render json: result, status: result[:http_status] end @@ -22,6 +20,8 @@ class JwtController < ApplicationController private def authenticate_project_or_user + @authentication_result = Gitlab::Auth::Result.new + authenticate_with_http_basic do |login, password| @authentication_result = Gitlab::Auth.find_for_git_client(login, password, project: nil, ip: request.ip) diff --git a/app/controllers/projects/cycle_analytics_controller.rb b/app/controllers/projects/cycle_analytics_controller.rb new file mode 100644 index 00000000000..16a7b1fc6e2 --- /dev/null +++ b/app/controllers/projects/cycle_analytics_controller.rb @@ -0,0 +1,67 @@ +class Projects::CycleAnalyticsController < Projects::ApplicationController + include ActionView::Helpers::DateHelper + include ActionView::Helpers::TextHelper + + before_action :authorize_read_cycle_analytics! + + def show + @cycle_analytics = CycleAnalytics.new(@project, from: parse_start_date) + + respond_to do |format| + format.html + format.json { render json: cycle_analytics_json } + end + end + + private + + def parse_start_date + case cycle_analytics_params[:start_date] + when '30' then 30.days.ago + when '90' then 90.days.ago + else 90.days.ago + end + end + + def cycle_analytics_params + return {} unless params[:cycle_analytics].present? + + { start_date: params[:cycle_analytics][:start_date] } + end + + def cycle_analytics_json + cycle_analytics_view_data = [[:issue, "Issue", "Time before an issue gets scheduled"], + [:plan, "Plan", "Time before an issue starts implementation"], + [:code, "Code", "Time until first merge request"], + [:test, "Test", "Total test time for all commits/merges"], + [:review, "Review", "Time between merge request creation and merge/close"], + [:staging, "Staging", "From merge request merge until deploy to production"], + [:production, "Production", "From issue creation until deploy to production"]] + + stats = cycle_analytics_view_data.reduce([]) do |stats, (stage_method, stage_text, stage_description)| + value = @cycle_analytics.send(stage_method).presence + + stats << { + title: stage_text, + description: stage_description, + value: value && !value.zero? ? distance_of_time_in_words(value) : nil + } + stats + end + + issues = @cycle_analytics.summary.new_issues + commits = @cycle_analytics.summary.commits + deploys = @cycle_analytics.summary.deploys + + summary = [ + { title: "New Issue".pluralize(issues), value: issues }, + { title: "Commit".pluralize(commits), value: commits }, + { title: "Deploy".pluralize(deploys), value: deploys } + ] + + { + summary: summary, + stats: stats + } + end +end diff --git a/app/controllers/projects/git_http_client_controller.rb b/app/controllers/projects/git_http_client_controller.rb index cbfd3cab3dd..383e184d796 100644 --- a/app/controllers/projects/git_http_client_controller.rb +++ b/app/controllers/projects/git_http_client_controller.rb @@ -32,11 +32,11 @@ class Projects::GitHttpClientController < Projects::ApplicationController return # Allow access end elsif allow_kerberos_spnego_auth? && spnego_provided? - user = find_kerberos_user + kerberos_user = find_kerberos_user - if user + if kerberos_user @authentication_result = Gitlab::Auth::Result.new( - user, nil, :kerberos, Gitlab::Auth.full_authentication_abilities) + kerberos_user, nil, :kerberos, Gitlab::Auth.full_authentication_abilities) send_final_spnego_response return # Allow access diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index de02e28e384..19b8b1576c4 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -23,20 +23,13 @@ class Projects::IssuesController < Projects::ApplicationController respond_to :html def index - terms = params['issue_search'] @issues = issues_collection - - if terms.present? - if terms =~ /\A#(\d+)\z/ - @issues = @issues.where(iid: $1) - else - @issues = @issues.full_search(terms) - end - end - @issues = @issues.page(params[:page]) + @labels = @project.labels.where(title: params[:label_name]) + @all_issues = all_issues_collection + respond_to do |format| format.html format.atom { render layout: false } diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index 0288ee87717..e972376df4c 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -31,22 +31,14 @@ class Projects::MergeRequestsController < Projects::ApplicationController before_action :authorize_can_resolve_conflicts!, only: [:conflicts, :resolve_conflicts] def index - terms = params['issue_search'] @merge_requests = merge_requests_collection - - if terms.present? - if terms =~ /\A[#!](\d+)\z/ - @merge_requests = @merge_requests.where(iid: $1) - else - @merge_requests = @merge_requests.full_search(terms) - end - end - @merge_requests = @merge_requests.page(params[:page]) @merge_requests = @merge_requests.preload(:target_project) @labels = @project.labels.where(title: params[:label_name]) + @all_merge_requests = all_merge_requests_collection + respond_to do |format| format.html format.json do diff --git a/app/finders/issuable_finder.rb b/app/finders/issuable_finder.rb index 60996b181f2..8f9ef8f725c 100644 --- a/app/finders/issuable_finder.rb +++ b/app/finders/issuable_finder.rb @@ -216,7 +216,14 @@ class IssuableFinder end def by_search(items) - items = items.search(search) if search + if search + items = + if search =~ iid_pattern + items.where(iid: $~[:iid]) + else + items.full_search(search) + end + end items end diff --git a/app/finders/issues_finder.rb b/app/finders/issues_finder.rb index c2befa5a5b3..be00a219205 100644 --- a/app/finders/issues_finder.rb +++ b/app/finders/issues_finder.rb @@ -25,4 +25,8 @@ class IssuesFinder < IssuableFinder def init_collection Issue.visible_to_user(current_user) end + + def iid_pattern + @iid_pattern ||= %r{\A#{Regexp.escape(Issue.reference_prefix)}(?<iid>\d+)\z} + end end diff --git a/app/finders/merge_requests_finder.rb b/app/finders/merge_requests_finder.rb index b258216d0d4..3b254e7d9d5 100644 --- a/app/finders/merge_requests_finder.rb +++ b/app/finders/merge_requests_finder.rb @@ -19,4 +19,14 @@ class MergeRequestsFinder < IssuableFinder def klass MergeRequest end + + private + + def iid_pattern + @iid_pattern ||= %r{\A[ + #{Regexp.escape(MergeRequest.reference_prefix)} + #{Regexp.escape(Issue.reference_prefix)} + ](?<iid>\d+)\z + }x + end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 5f3765cad0d..ed41bf04fc0 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -249,7 +249,7 @@ module ApplicationHelper milestone_title: params[:milestone_title], assignee_id: params[:assignee_id], author_id: params[:author_id], - issue_search: params[:issue_search], + search: params[:search], label_name: params[:label_name] } @@ -280,23 +280,14 @@ module ApplicationHelper end end - def state_filters_text_for(entity, project) + def state_filters_text_for(state, records) titles = { opened: "Open" } - entity_title = titles[entity] || entity.to_s.humanize - - count = - if project.nil? - nil - elsif current_controller?(:issues) - project.issues.visible_to_user(current_user).send(entity).count - elsif current_controller?(:merge_requests) - project.merge_requests.send(entity).count - end - - html = content_tag :span, entity_title + state_title = titles[state] || state.to_s.humanize + count = records.public_send(state).size + html = content_tag :span, state_title if count.present? html += " " diff --git a/app/helpers/gitlab_routing_helper.rb b/app/helpers/gitlab_routing_helper.rb index a322a90cc4e..5b71113feb9 100644 --- a/app/helpers/gitlab_routing_helper.rb +++ b/app/helpers/gitlab_routing_helper.rb @@ -46,6 +46,10 @@ module GitlabRoutingHelper namespace_project_environments_path(project.namespace, project, *args) end + def project_cycle_analytics_path(project, *args) + namespace_project_cycle_analytics_path(project.namespace, project, *args) + end + def project_builds_path(project, *args) namespace_project_builds_path(project.namespace, project, *args) end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index dd984aef318..cb87b43f6be 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -493,8 +493,11 @@ module Ci end def hide_secrets(trace) - trace = Ci::MaskSecret.mask(trace, project.runners_token) if project - trace = Ci::MaskSecret.mask(trace, token) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) trace end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 895eac1a258..663c5b1e231 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -56,6 +56,16 @@ module Ci pipeline.finished_at = Time.now end + after_transition [:created, :pending] => :running do |pipeline| + MergeRequest::Metrics.where(merge_request_id: pipeline.merge_requests.map(&:id)). + update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: nil) + end + + after_transition any => [:success] do |pipeline| + MergeRequest::Metrics.where(merge_request_id: pipeline.merge_requests.map(&:id)). + update_all(latest_build_finished_at: pipeline.finished_at) + end + before_transition do |pipeline| pipeline.update_duration end @@ -280,6 +290,16 @@ module Ci project.execute_services(data, :pipeline_hooks) end + # Merge requests for which the current pipeline is running against + # the merge request's latest commit. + def merge_requests + @merge_requests ||= + begin + project.merge_requests.where(source_branch: self.ref). + select { |merge_request| merge_request.pipeline.try(:id) == self.id } + end + end + private def pipeline_data diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 22231b2e0f0..1650ac9fcbe 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -28,10 +28,13 @@ module Issuable loaded? && to_a.all? { |note| note.association(:award_emoji).loaded? } end end + has_many :label_links, as: :target, dependent: :destroy has_many :labels, through: :label_links has_many :todos, as: :target, dependent: :destroy + has_one :metrics + validates :author, presence: true validates :title, presence: true, length: { within: 0..255 } @@ -81,6 +84,7 @@ module Issuable acts_as_paranoid after_save :update_assignee_cache_counts, if: :assignee_id_changed? + after_save :record_metrics def update_assignee_cache_counts # make sure we flush the cache for both the old *and* new assignee @@ -286,4 +290,9 @@ module Issuable def can_move?(*) false end + + def record_metrics + metrics = self.metrics || create_metrics + metrics.record! + end end diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb new file mode 100644 index 00000000000..be295487fd2 --- /dev/null +++ b/app/models/cycle_analytics.rb @@ -0,0 +1,97 @@ +class CycleAnalytics + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + def initialize(project, from:) + @project = project + @from = from + end + + def summary + @summary ||= Summary.new(@project, from: @from) + end + + def issue + calculate_metric(:issue, + Issue.arel_table[:created_at], + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]]) + end + + def plan + calculate_metric(:plan, + [Issue::Metrics.arel_table[:first_associated_with_milestone_at], + Issue::Metrics.arel_table[:first_added_to_board_at]], + Issue::Metrics.arel_table[:first_mentioned_in_commit_at]) + end + + def code + calculate_metric(:code, + Issue::Metrics.arel_table[:first_mentioned_in_commit_at], + MergeRequest.arel_table[:created_at]) + end + + def test + calculate_metric(:test, + MergeRequest::Metrics.arel_table[:latest_build_started_at], + MergeRequest::Metrics.arel_table[:latest_build_finished_at]) + end + + def review + calculate_metric(:review, + MergeRequest.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:merged_at]) + end + + def staging + calculate_metric(:staging, + MergeRequest::Metrics.arel_table[:merged_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end + + def production + calculate_metric(:production, + Issue.arel_table[:created_at], + MergeRequest::Metrics.arel_table[:first_deployed_to_production_at]) + end + + private + + def calculate_metric(name, start_time_attrs, end_time_attrs) + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query, end_time_attrs, start_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + # Join table with a row for every <issue,merge_request> pair (where the merge request + # closes the given issue) with issue and merge request metrics included. The metrics + # are loaded with an inner join, so issues / merge requests without metrics are + # automatically excluded. + def base_query + arel_table = MergeRequestsClosingIssues.arel_table + + # Load issues + query = arel_table.join(Issue.arel_table).on(Issue.arel_table[:id].eq(arel_table[:issue_id])). + join(Issue::Metrics.arel_table).on(Issue.arel_table[:id].eq(Issue::Metrics.arel_table[:issue_id])). + where(Issue.arel_table[:project_id].eq(@project.id)). + where(Issue.arel_table[:deleted_at].eq(nil)). + where(Issue.arel_table[:created_at].gteq(@from)) + + # Load merge_requests + query = query.join(MergeRequest.arel_table, Arel::Nodes::OuterJoin). + on(MergeRequest.arel_table[:id].eq(arel_table[:merge_request_id])). + join(MergeRequest::Metrics.arel_table). + on(MergeRequest.arel_table[:id].eq(MergeRequest::Metrics.arel_table[:merge_request_id])) + + # Limit to merge requests that have been deployed to production after `@from` + query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from)) + end +end diff --git a/app/models/cycle_analytics/summary.rb b/app/models/cycle_analytics/summary.rb new file mode 100644 index 00000000000..53b2cacb131 --- /dev/null +++ b/app/models/cycle_analytics/summary.rb @@ -0,0 +1,24 @@ +class CycleAnalytics + class Summary + def initialize(project, from:) + @project = project + @from = from + end + + def new_issues + @project.issues.created_after(@from).count + end + + def commits + repository = @project.repository.raw_repository + + if @project.default_branch + repository.log(ref: @project.default_branch, after: @from).count + end + end + + def deploys + @project.deployments.where("created_at > ?", @from).count + end + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 1e338889714..07d7e19e70d 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -42,4 +42,38 @@ class Deployment < ActiveRecord::Base project.repository.is_ancestor?(commit.id, sha) end + + def update_merge_request_metrics! + return unless environment.update_merge_request_metrics? + + merge_requests = project.merge_requests. + joins(:metrics). + where(target_branch: self.ref, merge_request_metrics: { first_deployed_to_production_at: nil }). + where("merge_request_metrics.merged_at <= ?", self.created_at) + + if previous_deployment + merge_requests = merge_requests.where("merge_request_metrics.merged_at >= ?", previous_deployment.created_at) + end + + # Need to use `map` instead of `select` because MySQL doesn't allow `SELECT`ing from the same table + # that we're updating. + merge_request_ids = + if Gitlab::Database.postgresql? + merge_requests.select(:id) + elsif Gitlab::Database.mysql? + merge_requests.map(&:id) + end + + MergeRequest::Metrics. + where(merge_request_id: merge_request_ids, first_deployed_to_production_at: nil). + update_all(first_deployed_to_production_at: self.created_at) + end + + def previous_deployment + @previous_deployment ||= + project.deployments.joins(:environment). + where(environments: { name: self.environment.name }, ref: self.ref). + where.not(id: self.id). + take + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 33c9abf382a..49e0a20640c 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -43,4 +43,8 @@ class Environment < ActiveRecord::Base last_deployment.includes_commit?(commit) end + + def update_merge_request_metrics? + self.name == "production" + end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 788611305fe..abd58e0454a 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -23,6 +23,8 @@ class Issue < ActiveRecord::Base has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + validates :project, presence: true scope :cared, ->(user) { where(assignee_id: user) } @@ -36,6 +38,8 @@ class Issue < ActiveRecord::Base scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } + scope :created_after, -> (datetime) { where("created_at >= ?", datetime) } + attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true diff --git a/app/models/issue/metrics.rb b/app/models/issue/metrics.rb new file mode 100644 index 00000000000..012d545c440 --- /dev/null +++ b/app/models/issue/metrics.rb @@ -0,0 +1,21 @@ +class Issue::Metrics < ActiveRecord::Base + belongs_to :issue + + def record! + if issue.milestone_id.present? && self.first_associated_with_milestone_at.blank? + self.first_associated_with_milestone_at = Time.now + end + + if issue_assigned_to_list_label? && self.first_added_to_board_at.blank? + self.first_added_to_board_at = Time.now + end + + self.save + end + + private + + def issue_assigned_to_list_label? + issue.labels.any? { |label| label.lists.present? } + end +end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 75f48fd4ba5..616efaf3c42 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -16,6 +16,8 @@ class MergeRequest < ActiveRecord::Base has_many :events, as: :target, dependent: :destroy + has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all + serialize :merge_params, Hash after_create :ensure_merge_request_diff, unless: :importing? @@ -501,6 +503,19 @@ class MergeRequest < ActiveRecord::Base target_project end + # If the merge request closes any issues, save this information in the + # `MergeRequestsClosingIssues` model. This is a performance optimization. + # Calculating this information for a number of merge requests requires + # running `ReferenceExtractor` on each of them separately. + def cache_merge_request_closes_issues!(current_user = self.author) + transaction do + self.merge_requests_closing_issues.delete_all + closes_issues(current_user).each do |issue| + self.merge_requests_closing_issues.create!(issue: issue) + end + end + end + def closes_issue?(issue) closes_issues.include?(issue) end @@ -508,7 +523,8 @@ class MergeRequest < ActiveRecord::Base # Return the set of issues that will be closed if this merge request is accepted. def closes_issues(current_user = self.author) if target_branch == project.default_branch - messages = commits.map(&:safe_message) << description + messages = [description] + messages.concat(commits.map(&:safe_message)) if merge_request_diff Gitlab::ClosingIssueExtractor.new(project, current_user). closed_by_message(messages.join("\n")) diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb new file mode 100644 index 00000000000..99c49a020c9 --- /dev/null +++ b/app/models/merge_request/metrics.rb @@ -0,0 +1,11 @@ +class MergeRequest::Metrics < ActiveRecord::Base + belongs_to :merge_request + + def record! + if merge_request.merged? && self.merged_at.blank? + self.merged_at = Time.now + end + + self.save + end +end diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb new file mode 100644 index 00000000000..ab597c37947 --- /dev/null +++ b/app/models/merge_requests_closing_issues.rb @@ -0,0 +1,7 @@ +class MergeRequestsClosingIssues < ActiveRecord::Base + belongs_to :merge_request + belongs_to :issue + + validates :merge_request_id, uniqueness: { scope: :issue_id }, presence: true + validates :issue_id, presence: true +end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 00c4c7b1440..be25c750d67 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -46,6 +46,7 @@ class ProjectPolicy < BasePolicy can! :create_issue can! :create_note can! :upload_file + can! :read_cycle_analytics end def reporter_access! @@ -204,6 +205,7 @@ class ProjectPolicy < BasePolicy can! :read_commit_status can! :read_container_image can! :download_code + can! :read_cycle_analytics # NOTE: may be overridden by IssuePolicy can! :read_issue diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 98da6563947..38ac6631228 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -5,7 +5,7 @@ module Auth AUDIENCE = 'container_registry' def execute(authentication_abilities:) - @authentication_abilities = authentication_abilities || [] + @authentication_abilities = authentication_abilities return error('not found', 404) unless registry.enabled diff --git a/app/services/create_deployment_service.rb b/app/services/create_deployment_service.rb index e6667132e27..799ad3e1bd0 100644 --- a/app/services/create_deployment_service.rb +++ b/app/services/create_deployment_service.rb @@ -4,7 +4,7 @@ class CreateDeploymentService < BaseService def execute(deployable = nil) environment = find_or_create_environment - project.deployments.create( + deployment = project.deployments.create( environment: environment, ref: params[:ref], tag: params[:tag], @@ -12,6 +12,10 @@ class CreateDeploymentService < BaseService user: current_user, deployable: deployable ) + + deployment.update_merge_request_metrics! + + deployment end private diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index 948041063c0..c499427605a 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -134,6 +134,7 @@ class GitPushService < BaseService end commit.create_cross_references!(authors[commit], closed_issues) + update_issue_metrics(commit, authors) end end @@ -186,4 +187,11 @@ class GitPushService < BaseService def branch_name @branch_name ||= Gitlab::Git.ref_name(params[:ref]) end + + def update_issue_metrics(commit, authors) + mentioned_issues = commit.all_references(authors[commit]).issues + + Issue::Metrics.where(issue_id: mentioned_issues.map(&:id), first_mentioned_in_commit_at: nil). + update_all(first_mentioned_in_commit_at: commit.committed_date) + end end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 4c8d93999a7..fbce46769f7 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -157,6 +157,10 @@ class IssuableBaseService < BaseService # To be overridden by subclasses end + def after_update(issuable) + # To be overridden by subclasses + end + def update_issuable(issuable, attributes) issuable.with_transaction_returning_status do issuable.update(attributes.merge(updated_by: current_user)) @@ -182,6 +186,7 @@ class IssuableBaseService < BaseService end handle_changes(issuable, old_labels: old_labels, old_mentioned_users: old_mentioned_users) + after_update(issuable) issuable.create_new_cross_references!(current_user) execute_hooks(issuable, 'update') end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 73247e62421..b0ae2dfe4ce 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -20,6 +20,7 @@ module MergeRequests event_service.open_mr(issuable, current_user) notification_service.new_merge_request(issuable, current_user) todo_service.new_merge_request(issuable, current_user) + issuable.cache_merge_request_closes_issues!(current_user) end end end diff --git a/app/services/merge_requests/refresh_service.rb b/app/services/merge_requests/refresh_service.rb index 5cedd6f11d9..22596b4014a 100644 --- a/app/services/merge_requests/refresh_service.rb +++ b/app/services/merge_requests/refresh_service.rb @@ -13,6 +13,7 @@ module MergeRequests reload_merge_requests reset_merge_when_build_succeeds mark_pending_todos_done + cache_merge_requests_closing_issues # Leave a system note if a branch was deleted/added if branch_added? || branch_removed? @@ -141,6 +142,14 @@ module MergeRequests end end + # If the merge requests closes any issues, save this information in the + # `MergeRequestsClosingIssues` model (as a performance optimization). + def cache_merge_requests_closing_issues + @project.merge_requests.where(source_branch: @branch_name).each do |merge_request| + merge_request.cache_merge_request_closes_issues!(@current_user) + end + end + def filter_merge_requests(merge_requests) merge_requests.uniq.select(&:source_project) end diff --git a/app/services/merge_requests/update_service.rb b/app/services/merge_requests/update_service.rb index 398ec47f0ea..f14f9e4b327 100644 --- a/app/services/merge_requests/update_service.rb +++ b/app/services/merge_requests/update_service.rb @@ -77,5 +77,9 @@ module MergeRequests def close_service MergeRequests::CloseService end + + def after_update(issuable) + issuable.cache_merge_request_closes_issues!(current_user) + end end end diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index 8e4937b7aa0..e44a2bfed9d 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -47,7 +47,7 @@ Repository - if project_nav_tab? :pipelines - = nav_link(controller: [:pipelines, :builds, :environments]) do + = nav_link(controller: [:pipelines, :builds, :environments, :cycle_analytics]) do = link_to project_pipelines_path(@project), title: 'Pipelines', class: 'shortcuts-pipelines' do %span Pipelines diff --git a/app/views/layouts/project.html.haml b/app/views/layouts/project.html.haml index 9fe94291db7..277eb71ea73 100644 --- a/app/views/layouts/project.html.haml +++ b/app/views/layouts/project.html.haml @@ -14,9 +14,6 @@ window.project_uploads_path = "#{namespace_project_uploads_path project.namespace,project}"; window.preview_markdown_path = "#{preview_markdown_path}"; -- content_for :scripts_body do - = render "layouts/init_auto_complete" if current_user - - content_for :header_content do .js-dropdown-menu-projects .dropdown-menu.dropdown-select.dropdown-menu-projects diff --git a/app/views/projects/_zen.html.haml b/app/views/projects/_zen.html.haml index 3978fa60d66..cb97181b9e1 100644 --- a/app/views/projects/_zen.html.haml +++ b/app/views/projects/_zen.html.haml @@ -7,3 +7,6 @@ = text_area_tag attr, nil, class: classes, placeholder: placeholder %a.zen-control.zen-control-leave.js-zen-leave{ href: "#" } = icon('compress') + +- content_for :scripts_body do + = render "layouts/init_auto_complete" if current_user && (@target_project || @project) diff --git a/app/views/projects/cycle_analytics/show.html.haml b/app/views/projects/cycle_analytics/show.html.haml new file mode 100644 index 00000000000..5dcb2a17873 --- /dev/null +++ b/app/views/projects/cycle_analytics/show.html.haml @@ -0,0 +1,57 @@ +- @no_container = true +- page_title "Cycle Analytics" += render "projects/pipelines/head" + +#cycle-analytics{"v-cloak" => "true", data: { request_path: project_cycle_analytics_path(@project)}} + + .bordered-box.landing.content-block{"v-if" => "!isHelpDismissed"} + = icon('times', class: 'dismiss-icon', "@click": "dismissLanding()") + = custom_icon('icon_cycle_analytics_splash') + .inner-content + %h4 + Introducing Cycle Analytics + %p + Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project. + + = link_to "Read more", help_page_path('user/project/cycle_analytics'), target: '_blank', class: 'btn' + + = icon("spinner spin", "v-show" => "isLoading") + + .wrapper{"v-show" => "!isLoading && !hasError"} + .panel.panel-default + .panel-heading + Pipeline Health + + .content-block + .container-fluid + .row + .col-xs-3.column{"v-for" => "item in summary"} + %h3.header {{item.value}} + %p.text {{item.title}} + + .col-xs-3.column + .dropdown.inline.js-ca-dropdown + %button.dropdown-menu-toggle{"data-toggle" => "dropdown", :type => "button"} + %span.dropdown-label Last 30 days + %i.fa.fa-chevron-down + %ul.dropdown-menu.dropdown-menu-align-right + %li + %a{'href' => "#", 'data-value' => '30'} + Last 30 days + %li + %a{'href' => "#", 'data-value' => '90'} + Last 90 days + + .bordered-box + %ul.content-list + %li{"v-for" => "item in stats"} + .container-fluid + .row + .col-xs-10.title-col + %p.title + {{item.title}} + %p.text + {{item.description}} + .col-xs-2.value-col + %span + {{item.value}} diff --git a/app/views/projects/pipelines/_head.html.haml b/app/views/projects/pipelines/_head.html.haml index f611ddc8f5f..5f571499e80 100644 --- a/app/views/projects/pipelines/_head.html.haml +++ b/app/views/projects/pipelines/_head.html.haml @@ -19,3 +19,9 @@ = link_to project_environments_path(@project), title: 'Environments', class: 'shortcuts-environments' do %span Environments + + - if can?(current_user, :read_cycle_analytics, @project) + = nav_link(controller: %w(cycle_analytics)) do + = link_to project_cycle_analytics_path(@project), title: 'Cycle Analytics' do + %span + Cycle Analytics diff --git a/app/views/shared/icons/_icon_cycle_analytics_splash.svg b/app/views/shared/icons/_icon_cycle_analytics_splash.svg new file mode 100644 index 00000000000..eb5a962d651 --- /dev/null +++ b/app/views/shared/icons/_icon_cycle_analytics_splash.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 99 102" xmlns:xlink="http://www.w3.org/1999/xlink"><defs><path id="0" d="m35.12 56.988c4.083-4.385 5.968-12.155 5.968-24.04 0-20.2-15.874-32.16-15.874-32.16-1.114-.954-2.929-.979-4.04 0 0 0-15.874 11.957-15.874 32.16 0 11.882 1.884 19.652 5.968 24.04h23.848"/><mask id="1" width="35.783" height="56.924" x="0" y="0" fill="#fff"><use xlink:href="#0"/></mask></defs><g fill="none" fill-rule="evenodd" transform="translate(0-4)"><g transform="translate(32.15 3.976)"><g fill="#6b4fbb"><path d="m11.928 56.988l1.325-1.325v3.313c0 .737.59 1.325 1.325 1.325h17.229c.736 0 1.325-.59 1.325-1.325v-3.313l1.325 1.325h-22.53m22.53-1.325v3.313c0 1.464-1.18 2.651-2.651 2.651h-17.229c-1.464 0-2.651-1.178-2.651-2.651v-3.313h22.53m-5.964 7.361h.663c0 3.294-2.67 5.964-5.964 5.964-3.294 0-5.964-2.67-5.964-5.964h.663.663c0 2.562 2.077 4.639 4.639 4.639 2.562 0 4.639-2.077 4.639-4.639h.663"/><path d="m5.816 42.535c-.346-2.839-.515-6.03-.515-9.584 0-20.2 15.874-32.16 15.874-32.16 1.106-.979 2.921-.954 4.04 0 0 0 15.874 11.957 15.874 32.16 0 11.882-1.884 19.652-5.968 24.04h-23.848c-2.861-3.073-4.643-7.807-5.453-14.453-.06-.493-.115-.997-.164-1.511l-4.04 2.884c-.891.637-1.614 2.041-1.614 3.137v14.581c0 1.465.971 1.958 2.165 1.106l8.691-6.208c-.282-.332-.553-.681-.813-1.048l-8.648 6.177c-.147.105-.069.152-.069-.027v-14.581c0-.668.516-1.671 1.059-2.059l3.432-2.451m38.4 20.2c1.193.852 2.165.359 2.165-1.106v-14.581c0-1.096-.723-2.5-1.614-3.137l-4.04-2.884c-.049.514-.104 1.018-.164 1.511l3.432 2.451c.543.388 1.059 1.391 1.059 2.059v14.581c0 .179.078.132-.069.027l-8.648-6.177c-.26.367-.531.716-.813 1.048l8.691 6.208"/></g><use fill="#fff" stroke="#6b4fbb" stroke-width="2.651" mask="url(#1)" xlink:href="#0"/><g fill="#b5a7dd"><path d="m30.482 28.494c0-4.03-3.263-7.289-7.289-7.289-4.03 0-7.289 3.263-7.289 7.289 0 4.03 3.263 7.289 7.289 7.289 4.03 0 7.289-3.263 7.289-7.289m-15.904 0c0-4.758 3.857-8.614 8.614-8.614 4.758 0 8.614 3.857 8.614 8.614 0 4.758-3.857 8.614-8.614 8.614-4.758 0-8.614-3.857-8.614-8.614"/><path d="m27.17 28.494c0-2.196-1.78-3.976-3.976-3.976-2.196 0-3.976 1.78-3.976 3.976 0 2.196 1.78 3.976 3.976 3.976 2.196 0 3.976-1.78 3.976-3.976m-9.277 0c0-2.928 2.373-5.301 5.301-5.301 2.928 0 5.301 2.373 5.301 5.301 0 2.928-2.373 5.301-5.301 5.301-2.928 0-5.301-2.373-5.301-5.301"/></g><path fill="#6b4fbb" d="m34.458 87.47c0 1.098.89 1.988 1.988 1.988 1.098 0 1.988-.89 1.988-1.988 0-.366.297-.663.663-.663.366 0 .663.297.663.663 0 1.83-1.483 3.313-3.313 3.313-1.826 0-3.307-1.478-3.313-3.302 0-.002 0-.003 0-.005v-2.663c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.657m-21.2-6.615c0-.002 0-.003 0-.005v-2.663c0-.358-.297-.657-.663-.657-.369 0-.663.294-.663.657v2.657c0 1.098-.89 1.988-1.988 1.988-1.098 0-1.988-.89-1.988-1.988 0-.366-.297-.663-.663-.663-.366 0-.663.297-.663.663 0 1.83 1.483 3.313 3.313 3.313 1.826 0 3.307-1.477 3.313-3.302m5.301 7.285c0-.001 0-.002 0-.003v-16.576c0-.362-.297-.658-.663-.658-.369 0-.663.295-.663.658v16.571c0 2.01-1.632 3.645-3.645 3.645-2.01 0-3.645-1.632-3.645-3.645 0-.366-.297-.663-.663-.663-.366 0-.663.297-.663.663 0 2.745 2.225 4.97 4.97 4.97 2.742 0 4.966-2.221 4.97-4.963m10.602 8.607v-18.555c0-.365-.297-.661-.663-.661-.369 0-.663.296-.663.661v18.557c0 0 0 0 0 .001.001 2.744 2.226 4.968 4.97 4.968 2.745 0 4.97-2.225 4.97-4.97 0-.366-.297-.663-.663-.663-.366 0-.663.297-.663.663 0 2.01-1.632 3.645-3.645 3.645-2.01 0-3.645-1.632-3.645-3.645m3.976-25.19c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m0 6.627c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m-10.602-6.627c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m5.301 0c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m-5.301 6.627c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m0 6.627c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663m-10.602-13.253c0-.363.294-.657.663-.657.366 0 .663.299.663.657v2.663c0 .363-.294.657-.663.657-.366 0-.663-.299-.663-.657v-2.663"/></g><path fill="#e2ddf2" d="m97.75 76.54c0-2.745-2.225-4.97-4.97-4.97-2.745 0-4.97 2.225-4.97 4.97 0 2.745 2.225 4.97 4.97 4.97 2.745 0 4.97-2.225 4.97-4.97m-8.614 0c0-2.01 1.632-3.645 3.645-3.645 2.01 0 3.645 1.632 3.645 3.645 0 2.01-1.632 3.645-3.645 3.645-2.01 0-3.645-1.632-3.645-3.645m-60.964-57.651c0-2.745-2.225-4.97-4.97-4.97-2.745 0-4.97 2.225-4.97 4.97 0 2.745 2.225 4.97 4.97 4.97 2.745 0 4.97-2.225 4.97-4.97m-8.614 0c0-2.01 1.632-3.645 3.645-3.645 2.01 0 3.645 1.632 3.645 3.645 0 2.01-1.632 3.645-3.645 3.645-2.01 0-3.645-1.632-3.645-3.645"/><path fill="#b5a7dd" d="m98.41 34.458c0-1.83-1.483-3.313-3.313-3.313-1.83 0-3.313 1.483-3.313 3.313 0 1.83 1.483 3.313 3.313 3.313 1.83 0 3.313-1.483 3.313-3.313m-5.301 0c0-1.098.89-1.988 1.988-1.988 1.098 0 1.988.89 1.988 1.988 0 1.098-.89 1.988-1.988 1.988-1.098 0-1.988-.89-1.988-1.988m-86.14 20.542c0-1.83-1.483-3.313-3.313-3.313-1.83 0-3.313 1.483-3.313 3.313 0 1.83 1.483 3.313 3.313 3.313 1.83 0 3.313-1.483 3.313-3.313m-5.301 0c0-1.098.89-1.988 1.988-1.988 1.098 0 1.988.89 1.988 1.988 0 1.098-.89 1.988-1.988 1.988-1.098 0-1.988-.89-1.988-1.988"/></g></svg> diff --git a/app/views/shared/issuable/_filter.html.haml b/app/views/shared/issuable/_filter.html.haml index 93c4d5c3d30..cf26197f7d7 100644 --- a/app/views/shared/issuable/_filter.html.haml +++ b/app/views/shared/issuable/_filter.html.haml @@ -2,9 +2,9 @@ .issues-filters .issues-details-filters.row-content-block.second-block - = form_tag page_filter_path(without: [:assignee_id, :author_id, :milestone_title, :label_name, :issue_search]), method: :get, class: 'filter-form js-filter-form' do - - if params[:issue_search].present? - = hidden_field_tag :issue_search, params[:issue_search] + = form_tag page_filter_path(without: [:assignee_id, :author_id, :milestone_title, :label_name, :search]), method: :get, class: 'filter-form js-filter-form' do + - if params[:search].present? + = hidden_field_tag :search, params[:search] - if @bulk_edit .check-all-holder = check_box_tag "check_all_issues", nil, false, @@ -29,7 +29,7 @@ = render "shared/issuable/label_dropdown" .filter-item.inline.reset-filters - %a{href: page_filter_path(without: [:assignee_id, :author_id, :milestone_title, :label_name, :issue_search])} Reset filters + %a{href: page_filter_path(without: [:assignee_id, :author_id, :milestone_title, :label_name, :search])} Reset filters .pull-right - if boards_page diff --git a/app/views/shared/issuable/_nav.html.haml b/app/views/shared/issuable/_nav.html.haml index 1d9b09a5ef1..fb592c2b1e2 100644 --- a/app/views/shared/issuable/_nav.html.haml +++ b/app/views/shared/issuable/_nav.html.haml @@ -1,25 +1,27 @@ %ul.nav-links.issues-state-filters - if defined?(type) && type == :merge_requests - page_context_word = 'merge requests' + - records = @all_merge_requests - else - page_context_word = 'issues' + - records = @all_issues %li{class: ("active" if params[:state] == 'opened')} = link_to page_filter_path(state: 'opened', label: true), title: "Filter by #{page_context_word} that are currently opened." do - #{state_filters_text_for(:opened, @project)} + #{state_filters_text_for(:opened, records)} - if defined?(type) && type == :merge_requests %li{class: ("active" if params[:state] == 'merged')} = link_to page_filter_path(state: 'merged', label: true), title: 'Filter by merge requests that are currently merged.' do - #{state_filters_text_for(:merged, @project)} + #{state_filters_text_for(:merged, records)} %li{class: ("active" if params[:state] == 'closed')} = link_to page_filter_path(state: 'closed', label: true), title: 'Filter by merge requests that are currently closed and unmerged.' do - #{state_filters_text_for(:closed, @project)} + #{state_filters_text_for(:closed, records)} - else %li{class: ("active" if params[:state] == 'closed')} = link_to page_filter_path(state: 'closed', label: true), title: 'Filter by issues that are currently closed.' do - #{state_filters_text_for(:closed, @project)} + #{state_filters_text_for(:closed, records)} %li{class: ("active" if params[:state] == 'all')} = link_to page_filter_path(state: 'all', label: true), title: "Show all #{page_context_word}." do - #{state_filters_text_for(:all, @project)} + #{state_filters_text_for(:all, records)} diff --git a/app/views/shared/issuable/_search_form.html.haml b/app/views/shared/issuable/_search_form.html.haml index 186963b32b8..2c89217cadd 100644 --- a/app/views/shared/issuable/_search_form.html.haml +++ b/app/views/shared/issuable/_search_form.html.haml @@ -1,2 +1,2 @@ -= form_tag(path, method: :get, id: "issue_search_form", class: 'issue-search-form') do - = search_field_tag :issue_search, params[:issue_search], { placeholder: 'Filter by name ...', class: 'form-control issue_search search-text-input input-short', spellcheck: false } += form_tag(path, method: :get, id: "issuable_search_form", class: 'issuable-search-form') do + = search_field_tag :search, params[:search], { id: 'issuable_search', placeholder: 'Filter by name ...', class: 'form-control issuable_search search-text-input input-short', spellcheck: false } diff --git a/config/routes.rb b/config/routes.rb index 068c92d1400..c4eee59e7aa 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -780,6 +780,8 @@ Rails.application.routes.draw do resources :environments + resource :cycle_analytics, only: [:show] + resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do collection do post :cancel_all diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb new file mode 100644 index 00000000000..e882a492757 --- /dev/null +++ b/db/fixtures/development/17_cycle_analytics.rb @@ -0,0 +1,246 @@ +require 'sidekiq/testing' +require './spec/support/test_env' + +class Gitlab::Seeder::CycleAnalytics + def initialize(project, perf: false) + @project = project + @user = User.order(:id).last + @issue_count = perf ? 1000 : 5 + stub_git_pre_receive! + end + + # The GitLab API needn't be running for the fixtures to be + # created. Since we're performing a number of git actions + # here (like creating a branch or committing a file), we need + # to disable the `pre_receive` hook in order to remove this + # dependency on the GitLab API. + def stub_git_pre_receive! + GitHooksService.class_eval do + def run_hook(name) + [true, ''] + end + end + end + + def seed_metrics! + @issue_count.times do |index| + # Issue + Timecop.travel 5.days.from_now + title = "#{FFaker::Product.brand}-#{FFaker::Product.brand}-#{rand(1000)}" + issue = Issue.create(project: @project, title: title, author: @user) + issue_metrics = issue.metrics + + # Milestones / Labels + Timecop.travel 5.days.from_now + if index.even? + issue_metrics.first_associated_with_milestone_at = rand(6..12).hours.from_now + else + issue_metrics.first_added_to_board_at = rand(6..12).hours.from_now + end + + # Commit + Timecop.travel 5.days.from_now + issue_metrics.first_mentioned_in_commit_at = rand(6..12).hours.from_now + + # MR + Timecop.travel 5.days.from_now + branch_name = "#{FFaker::Product.brand}-#{FFaker::Product.brand}-#{rand(1000)}" + @project.repository.add_branch(@user, branch_name, 'master') + merge_request = MergeRequest.create(target_project: @project, source_project: @project, source_branch: branch_name, target_branch: 'master', title: branch_name, author: @user) + merge_request_metrics = merge_request.metrics + + # MR closing issues + Timecop.travel 5.days.from_now + MergeRequestsClosingIssues.create!(issue: issue, merge_request: merge_request) + + # Merge + Timecop.travel 5.days.from_now + merge_request_metrics.merged_at = rand(6..12).hours.from_now + + # Start build + Timecop.travel 5.days.from_now + merge_request_metrics.latest_build_started_at = rand(6..12).hours.from_now + + # Finish build + Timecop.travel 5.days.from_now + merge_request_metrics.latest_build_finished_at = rand(6..12).hours.from_now + + # Deploy to production + Timecop.travel 5.days.from_now + merge_request_metrics.first_deployed_to_production_at = rand(6..12).hours.from_now + + issue_metrics.save! + merge_request_metrics.save! + + print '.' + end + end + + def seed! + Sidekiq::Testing.inline! do + issues = create_issues + puts '.' + + # Stage 1 + Timecop.travel 5.days.from_now + add_milestones_and_list_labels(issues) + print '.' + + # Stage 2 + Timecop.travel 5.days.from_now + branches = mention_in_commits(issues) + print '.' + + # Stage 3 + Timecop.travel 5.days.from_now + merge_requests = create_merge_requests_closing_issues(issues, branches) + print '.' + + # Stage 4 + Timecop.travel 5.days.from_now + run_builds(merge_requests) + print '.' + + # Stage 5 + Timecop.travel 5.days.from_now + merge_merge_requests(merge_requests) + print '.' + + # Stage 6 / 7 + Timecop.travel 5.days.from_now + deploy_to_production(merge_requests) + print '.' + end + + print '.' + end + + private + + def create_issues + Array.new(@issue_count) do + issue_params = { + title: "Cycle Analytics: #{FFaker::Lorem.sentence(6)}", + description: FFaker::Lorem.sentence, + state: 'opened', + assignee: @project.team.users.sample + } + + Issues::CreateService.new(@project, @project.team.users.sample, issue_params).execute + end + end + + def add_milestones_and_list_labels(issues) + issues.shuffle.map.with_index do |issue, index| + Timecop.travel 12.hours.from_now + + if index.even? + issue.update(milestone: @project.milestones.sample) + else + label_name = "#{FFaker::Product.brand}-#{FFaker::Product.brand}-#{rand(1000)}" + list_label = FactoryGirl.create(:label, title: label_name, project: issue.project) + FactoryGirl.create(:list, board: FactoryGirl.create(:board, project: issue.project), label: list_label) + issue.update(labels: [list_label]) + end + + issue + end + end + + def mention_in_commits(issues) + issues.map do |issue| + Timecop.travel 12.hours.from_now + + branch_name = filename = "#{FFaker::Product.brand}-#{FFaker::Product.brand}-#{rand(1000)}" + + issue.project.repository.add_branch(@user, branch_name, 'master') + + options = { + committer: issue.project.repository.user_to_committer(@user), + author: issue.project.repository.user_to_committer(@user), + commit: { message: "Commit for ##{issue.iid}", branch: branch_name, update_ref: true }, + file: { content: "content", path: filename, update: false } + } + + commit_sha = Gitlab::Git::Blob.commit(issue.project.repository, options) + issue.project.repository.commit(commit_sha) + + + GitPushService.new(issue.project, + @user, + oldrev: issue.project.repository.commit("master").sha, + newrev: commit_sha, + ref: 'refs/heads/master').execute + + branch_name + end + end + + def create_merge_requests_closing_issues(issues, branches) + issues.zip(branches).map do |issue, branch| + Timecop.travel 12.hours.from_now + + opts = { + title: 'Cycle Analytics merge_request', + description: "Fixes #{issue.to_reference}", + source_branch: branch, + target_branch: 'master' + } + + MergeRequests::CreateService.new(issue.project, @user, opts).execute + end + end + + def run_builds(merge_requests) + merge_requests.each do |merge_request| + Timecop.travel 12.hours.from_now + + service = Ci::CreatePipelineService.new(merge_request.project, + @user, + ref: "refs/heads/#{merge_request.source_branch}") + pipeline = service.execute(ignore_skip_ci: true, save_on_errors: false) + + pipeline.run! + Timecop.travel rand(1..6).hours.from_now + pipeline.succeed! + end + end + + def merge_merge_requests(merge_requests) + merge_requests.each do |merge_request| + Timecop.travel 12.hours.from_now + + MergeRequests::MergeService.new(merge_request.project, @user).execute(merge_request) + end + end + + def deploy_to_production(merge_requests) + merge_requests.each do |merge_request| + Timecop.travel 12.hours.from_now + + CreateDeploymentService.new(merge_request.project, @user, { + environment: 'production', + ref: 'master', + tag: false, + sha: @project.repository.commit('master').sha + }).execute + end + end +end + +Gitlab::Seeder.quiet do + if ENV['SEED_CYCLE_ANALYTICS'] + Project.all.each do |project| + seeder = Gitlab::Seeder::CycleAnalytics.new(project) + seeder.seed! + end + elsif ENV['CYCLE_ANALYTICS_PERF_TEST'] + seeder = Gitlab::Seeder::CycleAnalytics.new(Project.order(:id).first, perf: true) + seeder.seed! + elsif ENV['CYCLE_ANALYTICS_POPULATE_METRICS_DIRECTLY'] + seeder = Gitlab::Seeder::CycleAnalytics.new(Project.order(:id).first, perf: true) + seeder.seed_metrics! + else + puts "Not running the cycle analytics seed file. Use the `SEED_CYCLE_ANALYTICS` environment variable to enable it." + end +end diff --git a/db/migrate/20160824124900_add_table_issue_metrics.rb b/db/migrate/20160824124900_add_table_issue_metrics.rb new file mode 100644 index 00000000000..e9bb79b3c62 --- /dev/null +++ b/db/migrate/20160824124900_add_table_issue_metrics.rb @@ -0,0 +1,37 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddTableIssueMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = true + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + DOWNTIME_REASON = 'Adding foreign key' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :issue_metrics do |t| + t.references :issue, index: { name: "index_issue_metrics" }, foreign_key: { on_delete: :cascade }, null: false + + t.datetime 'first_mentioned_in_commit_at' + t.datetime 'first_associated_with_milestone_at' + t.datetime 'first_added_to_board_at' + + t.timestamps null: false + end + end +end diff --git a/db/migrate/20160825052008_add_table_merge_request_metrics.rb b/db/migrate/20160825052008_add_table_merge_request_metrics.rb new file mode 100644 index 00000000000..e01cc5038b9 --- /dev/null +++ b/db/migrate/20160825052008_add_table_merge_request_metrics.rb @@ -0,0 +1,38 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddTableMergeRequestMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = true + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + DOWNTIME_REASON = 'Adding foreign key' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :merge_request_metrics do |t| + t.references :merge_request, index: { name: "index_merge_request_metrics" }, foreign_key: { on_delete: :cascade }, null: false + + t.datetime 'latest_build_started_at' + t.datetime 'latest_build_finished_at' + t.datetime 'first_deployed_to_production_at', index: true + t.datetime 'merged_at' + + t.timestamps null: false + end + end +end diff --git a/db/migrate/20160915042921_create_merge_requests_closing_issues.rb b/db/migrate/20160915042921_create_merge_requests_closing_issues.rb new file mode 100644 index 00000000000..94874a853da --- /dev/null +++ b/db/migrate/20160915042921_create_merge_requests_closing_issues.rb @@ -0,0 +1,34 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateMergeRequestsClosingIssues < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = true + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + DOWNTIME_REASON = 'Adding foreign keys' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :merge_requests_closing_issues do |t| + t.references :merge_request, foreign_key: { on_delete: :cascade }, index: true, null: false + t.references :issue, foreign_key: { on_delete: :cascade }, index: true, null: false + + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 3567908de03..fc98694e2eb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160913212128) do +ActiveRecord::Schema.define(version: 20160915081353) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -439,6 +439,17 @@ ActiveRecord::Schema.define(version: 20160913212128) do add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree + create_table "issue_metrics", force: :cascade do |t| + t.integer "issue_id", null: false + t.datetime "first_associated_with_milestone_at" + t.datetime "first_added_to_board_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.datetime "first_mentioned_in_commit_at" + end + + add_index "issue_metrics", ["issue_id"], name: "index_issue_metrics", using: :btree + create_table "issues", force: :cascade do |t| t.string "title" t.integer "assignee_id" @@ -581,6 +592,18 @@ ActiveRecord::Schema.define(version: 20160913212128) do add_index "merge_request_diffs", ["merge_request_id"], name: "index_merge_request_diffs_on_merge_request_id", using: :btree + create_table "merge_request_metrics", force: :cascade do |t| + t.integer "merge_request_id", null: false + t.datetime "latest_build_started_at" + t.datetime "latest_build_finished_at" + t.datetime "first_deployed_to_production_at" + t.datetime "merged_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "merge_request_metrics", ["merge_request_id"], name: "index_merge_request_metrics", using: :btree + create_table "merge_requests", force: :cascade do |t| t.string "target_branch", null: false t.string "source_branch", null: false @@ -622,6 +645,16 @@ ActiveRecord::Schema.define(version: 20160913212128) do add_index "merge_requests", ["title"], name: "index_merge_requests_on_title", using: :btree add_index "merge_requests", ["title"], name: "index_merge_requests_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"} + create_table "merge_requests_closing_issues", force: :cascade do |t| + t.integer "merge_request_id", null: false + t.integer "issue_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "merge_requests_closing_issues", ["issue_id"], name: "index_merge_requests_closing_issues_on_issue_id", using: :btree + add_index "merge_requests_closing_issues", ["merge_request_id"], name: "index_merge_requests_closing_issues_on_merge_request_id", using: :btree + create_table "milestones", force: :cascade do |t| t.string "title", null: false t.integer "project_id", null: false @@ -1147,8 +1180,12 @@ ActiveRecord::Schema.define(version: 20160913212128) do add_index "web_hooks", ["project_id"], name: "index_web_hooks_on_project_id", using: :btree add_foreign_key "boards", "projects" + add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "lists", "boards" add_foreign_key "lists", "labels" + add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade + add_foreign_key "merge_requests_closing_issues", "issues", on_delete: :cascade + add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade add_foreign_key "personal_access_tokens", "users" add_foreign_key "protected_branch_merge_access_levels", "protected_branches" add_foreign_key "protected_branch_push_access_levels", "protected_branches" diff --git a/doc/api/README.md b/doc/api/README.md index 7661e1eea02..6e3295e0e0c 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -55,11 +55,12 @@ The following documentation is for the [internal CI API](ci/README.md): ## Authentication -All API requests require authentication via a token. There are three types of tokens -available: private tokens, OAuth 2 tokens, and personal access tokens. +All API requests require authentication via a session cookie or token. There are +three types of tokens available: private tokens, OAuth 2 tokens, and personal +access tokens. -If a token is invalid or omitted, an error message will be returned with -status code `401`: +If authentication information is invalid or omitted, an error message will be +returned with status code `401`: ```json { @@ -98,6 +99,13 @@ that needs access to the GitLab API. Once you have your token, pass it to the API using either the `private_token` parameter or the `PRIVATE-TOKEN` header. + +###Â Session cookie + +When signing in to GitLab as an ordinary user, a `_gitlab_session` cookie is +set. The API will use this cookie for authentication if it is present, but using +the API to generate a new session cookie is currently not supported. + ## Basic Usage API requests should be prefixed with `api` and the API version. The API version diff --git a/doc/user/project/cycle_analytics.md b/doc/user/project/cycle_analytics.md new file mode 100644 index 00000000000..e1fe1d256fd --- /dev/null +++ b/doc/user/project/cycle_analytics.md @@ -0,0 +1,112 @@ +# Cycle Analytics + +> [Introduced][ce-5986] in GitLab 8.12. +> +> **Note:** +This the first iteration of Cycle Analytics, you can follow the following issue +to track the changes that are coming to this feature: [#20975][ce-20975]. + +Cycle Analytics measures the time it takes to go from an idea to production for +each project you have. This is achieved by not only indicating the total time it +takes to reach at that point, but the total time is broken down into the +multiple stages an idea has to pass through to be shipped. + +Cycle Analytics is that it is tightly coupled with the [GitLab flow] and +calculates a separate median for each stage. + +## Overview + +You can find the Cycle Analytics page under your project's **Pipelines > Cycle +Analytics** tab. + + + +You can see that there are seven stages in total: + +- **Issue** (Tracker) + - Median time from issue creation until given a milestone or list label + (first assignment, any milestone, milestone date or assignee is not required) +- **Plan** (Board) + - Median time from giving an issue a milestone or label until pushing the + first commit +- **Code** (IDE) + - Median time from the first commit until the merge request is created +- **Test** (CI) + - Total test time for all commits/merges +- **Review** (Merge Request/MR) + - Median time from merge request creation until the merge request is merged + (closed merge requests won't be taken into account) +- **Staging** (Continuous Deployment) + - Median time from when the merge request got merged until the deploy to + production (production is last stage/environment) +- **Production** (Total) + - Sum of all the above stages excluding the Test (CI) time + +## How the data is measured + +Cycle Analytics records cycle time so only data on the issues that have been +deployed to production are measured. In case you just started a new project and +you have not pushed anything to production, then you will not be able to +properly see the Cycle Analytics of your project. + +Specifically, if your CI is not set up and you have not defined a `production` +[environment], then you will not have any data. + +Below you can see in more detail what the various stages of Cycle Analytics mean. + +| **Stage** | **Description** | +| --------- | --------------- | +| Issue | Measures the median time between creating an issue and taking action to solve it, by either labeling it or adding it to a milestone, whatever comes first. The label will be tracked only if it already has an [Issue Board list][board] created for it. | +| Plan | Measures the median time between the action you took for the previous stage, and pushing the first commit to the repository. To make this change tracked, the commit needs to be pushed that contains the issue closing pattern `Closes #xxx`, where `xxx` is the number of the issue related to this commit. If the commit does not contain the issue closing pattern, it is not considered to the measure time of the stage. | +| Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request related to that commit. The key to keep the process tracked is include the issue closing pattern to the description of the merge request. | +| Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. `master` is not excluded. It does not attempt to track time for any particular stages. | +| Review | Measures the median time taken to review the merge request, between its creation and until it's merged. | +| Staging | Measures the median time between merging the merge request until the very first deployment of the to production. It's tracked by the [environment] set to `production` in your GitLab CI configuration. If there isn't a `production` environment, this is not tracked. | +| Production| The sum of all time taken to run the entire process, from issue creation to deploying the code to production. | + +--- + +Here's a little explanation of how this works behind the scenes: + +1. Issues and merge requests are grouped together in pairs, such that for each + `<issue, merge request>` pair, the merge request has `Fixes #xxx` for the + corresponding issue. All other issues and merge requests are **not** considered. + +1. Then the <issue, merge request> pairs are filtered out. Any merge request + that has **not** been deployed to production in the last XX days (specified + by the UI - default is 90 days) prohibits these pairs from being considered. + +1. For the remaining `<issue, merge request>` pairs, we check the information that + we need for the stages, like issue creation date, merge request merge time, + etc. + +To sum up, anything that doesn't follow the [GitLab flow] won't be tracked at all. +So, if a merge request doesn't close an issue or an issue is not labeled with a +label present in the Issue Board or assigned a milestone or a project has no +`production` environment, the Cycle Analytics dashboard won't present any data +at all. + +## Permissions + +The current permissions on the Cycle Analytics dashboard are: + +- Public projects - anyone can access +- Private/internal projects - any member (guest level and above) can access + +You can [read more about permissions][permissions] in general. + +## More resources + +Learn more about Cycle Analytics in the following resources: + +- [Cycle Analytics feature page](https://about.gitlab.com/solutions/cycle-analytics/) +- [Cycle Analytics feature preview](https://about.gitlab.com/2016/09/16/feature-preview-introducing-cycle-analytics/) +- [Cycle Analytics feature highlight](https://about.gitlab.com/2016-09-19-cycle-analytics-feature-highlight.html) + + +[ce-5986]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5986 +[ce-20975]: https://gitlab.com/gitlab-org/gitlab-ce/issues/20975 +[GitLab flow]: ../../workflow/gitlab_flow.md +[permissions]: ../permissions.md +[environment]: ../../ci/yaml/README.md#environment +[board]: issue_board.md#creating-a-new-list diff --git a/doc/user/project/img/cycle_analytics_landing_page.png b/doc/user/project/img/cycle_analytics_landing_page.png Binary files differnew file mode 100644 index 00000000000..4fa42c87395 --- /dev/null +++ b/doc/user/project/img/cycle_analytics_landing_page.png diff --git a/doc/workflow/README.md b/doc/workflow/README.md index dcb9c32ad58..e8ecb8d8fb4 100644 --- a/doc/workflow/README.md +++ b/doc/workflow/README.md @@ -1,6 +1,7 @@ # Workflow - [Change your time zone](timezone.md) +- [Cycle Analytics](../user/project/cycle_analytics.md) - [Description templates](../user/project/description_templates.md) - [Feature branch workflow](workflow.md) - [GitLab Flow](gitlab_flow.md) diff --git a/features/steps/project/issues/issues.rb b/features/steps/project/issues/issues.rb index e21f76d00d9..ed7241679ee 100644 --- a/features/steps/project/issues/issues.rb +++ b/features/steps/project/issues/issues.rb @@ -356,6 +356,6 @@ class Spinach::Features::ProjectIssues < Spinach::FeatureSteps end def filter_issue(text) - fill_in 'issue_search', with: text + fill_in 'issuable_search', with: text end end diff --git a/features/steps/project/merge_requests.rb b/features/steps/project/merge_requests.rb index df17b5626c6..4a67cf06fba 100644 --- a/features/steps/project/merge_requests.rb +++ b/features/steps/project/merge_requests.rb @@ -493,7 +493,7 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps end step 'I fill in merge request search with "Fe"' do - fill_in 'issue_search', with: "Fe" + fill_in 'issuable_search', with: "Fe" end step 'I click the "Target branch" dropdown' do diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..8cc7a26f1fa 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -33,46 +33,29 @@ module API # # If the token is revoked, then it raises RevokedError. # - # If the token is not found (nil), then it raises TokenNotFoundError. + # If the token is not found (nil), then it returns nil # # Arguments: # # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end - end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + access_token = find_access_token + return nil unless access_token + + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end @@ -96,19 +79,6 @@ module API end module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end - end - private def install_error_responders(base) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 150875ed4f0..714d4ea3dc6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -12,13 +12,30 @@ module API nil end + def private_token + params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + def find_user_from_warden + warden ? warden.authenticate : nil + end + def find_user_by_private_token - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + token = private_token + return nil unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) end def current_user - @current_user ||= (find_user_by_private_token || doorkeeper_guard) + @current_user ||= find_user_by_private_token + @current_user ||= doorkeeper_guard + @current_user ||= find_user_from_warden unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb index 3da04edde70..997377abc55 100644 --- a/lib/ci/mask_secret.rb +++ b/lib/ci/mask_secret.rb @@ -1,9 +1,10 @@ module Ci::MaskSecret class << self - def mask(value, token) + def mask!(value, token) return value unless value.present? && token.present? - value.gsub(token, 'x' * token.length) + value.gsub!(token, 'x' * token.length) + value end end end diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb new file mode 100644 index 00000000000..b6a89f715fd --- /dev/null +++ b/lib/gitlab/database/date_time.rb @@ -0,0 +1,27 @@ +module Gitlab + module Database + module DateTime + # Find the first of the `end_time_attrs` that isn't `NULL`. Subtract from it + # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval + # along with an alias specified by the `as` parameter. + # + # Note: For MySQL, the interval is returned in seconds. + # For PostgreSQL, the interval is returned as an INTERVAL type. + def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) + diff_fn = if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + + query_so_far.project(diff_fn.as(as)) + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb new file mode 100644 index 00000000000..1444d25ebc7 --- /dev/null +++ b/lib/gitlab/database/median.rb @@ -0,0 +1,112 @@ +# https://www.periscopedata.com/blog/medians-in-sql.html +module Gitlab + module Database + module Median + def median_datetime(arel_table, query_so_far, column_sym) + median_queries = + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym) + elsif Gitlab::Database.mysql? + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + + results = Array.wrap(median_queries).map do |query| + ActiveRecord::Base.connection.execute(query) + end + extract_median(results).presence + end + + def extract_median(results) + result = results.compact.first + + if Gitlab::Database.postgresql? + result = result.first.presence + median = result['median'] if result + median.to_f if median + elsif Gitlab::Database.mysql? + result.to_a.flatten.first + end + end + + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + query = arel_table. + from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). + project(average([arel_table[column_sym]], 'median')). + where( + Arel::Nodes::Between.new( + Arel.sql("(select @row_id := @row_id + 1)"), + Arel::Nodes::And.new( + [Arel.sql('@ct/2.0'), + Arel.sql('@ct/2.0 + 1')] + ) + ) + ). + # Disallow negative values + where(arel_table[column_sym].gteq(0)) + + [ + Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), + Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), + Arel.sql("set @row_id := 0;"), + query.to_sql, + Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") + ] + end + + def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + # Create a CTE with the column we're operating on, row number (after sorting by the column + # we're operating on), and count of the table we're operating on (duplicated across) all rows + # of the CTE. For example, if we're looking to find the median of the `projects.star_count` + # column, the CTE might look like this: + # + # star_count | row_id | ct + # ------------+--------+---- + # 5 | 1 | 3 + # 9 | 2 | 3 + # 15 | 3 | 3 + cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( + cte_table, + arel_table. + project( + arel_table[column_sym].as(column_sym.to_s), + Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), + Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), + arel_table.project("COUNT(1)").as('ct')). + # Disallow negative values + where(arel_table[column_sym].gteq(zero_interval))) + + # From the CTE, select either the middle row or the middle two rows (this is accomplished + # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the + # selected rows, and this is the median value. + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). + where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) + ) + ). + with(query_so_far, cte). + to_sql + end + + private + + def average(args, as) + Arel::Nodes::NamedFunction.new("AVG", args, as) + end + + def extract_epoch(arel_attribute) + Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) + end + + # Need to cast '0' to an INTERVAL before we can check if the interval is positive + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c5a11148d33..2c21804fe7a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,7 +11,6 @@ module Gitlab if current_user gon.current_user_id = current_user.id - gon.api_token = current_user.private_token end end end diff --git a/spec/factories/deployments.rb b/spec/factories/deployments.rb index 82591604fcb..6f24bf58d14 100644 --- a/spec/factories/deployments.rb +++ b/spec/factories/deployments.rb @@ -3,11 +3,12 @@ FactoryGirl.define do sha '97de212e80737a608d939f648d959671fb0a0142' ref 'master' tag false + project nil environment factory: :environment after(:build) do |deployment, evaluator| - deployment.project = deployment.environment.project + deployment.project ||= deployment.environment.project end end end diff --git a/spec/features/dashboard_issues_spec.rb b/spec/features/dashboard_issues_spec.rb index 3fb1cb37544..fc914022a59 100644 --- a/spec/features/dashboard_issues_spec.rb +++ b/spec/features/dashboard_issues_spec.rb @@ -21,6 +21,9 @@ describe "Dashboard Issues filtering", feature: true, js: true do click_link 'No Milestone' + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end expect(page).to have_selector('.issue', count: 1) end @@ -29,6 +32,9 @@ describe "Dashboard Issues filtering", feature: true, js: true do click_link 'Any Milestone' + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '2') + end expect(page).to have_selector('.issue', count: 2) end @@ -39,6 +45,9 @@ describe "Dashboard Issues filtering", feature: true, js: true do click_link milestone.title end + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end expect(page).to have_selector('.issue', count: 1) end end diff --git a/spec/features/issues/filter_issues_spec.rb b/spec/features/issues/filter_issues_spec.rb index 69fda27cc61..72f39e2fbca 100644 --- a/spec/features/issues/filter_issues_spec.rb +++ b/spec/features/issues/filter_issues_spec.rb @@ -206,7 +206,7 @@ describe 'Filter issues', feature: true do context 'only text', js: true do it 'filters issues by searched text' do - fill_in 'issue_search', with: 'Bug' + fill_in 'issuable_search', with: 'Bug' page.within '.issues-list' do expect(page).to have_selector('.issue', count: 2) @@ -214,7 +214,7 @@ describe 'Filter issues', feature: true do end it 'does not show any issues' do - fill_in 'issue_search', with: 'testing' + fill_in 'issuable_search', with: 'testing' page.within '.issues-list' do expect(page).not_to have_selector('.issue') @@ -224,12 +224,16 @@ describe 'Filter issues', feature: true do context 'text and dropdown options', js: true do it 'filters by text and label' do - fill_in 'issue_search', with: 'Bug' + fill_in 'issuable_search', with: 'Bug' page.within '.issues-list' do expect(page).to have_selector('.issue', count: 2) end + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '2') + end + click_button 'Label' page.within '.labels-filter' do click_link 'bug' @@ -239,15 +243,23 @@ describe 'Filter issues', feature: true do page.within '.issues-list' do expect(page).to have_selector('.issue', count: 1) end + + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end end it 'filters by text and milestone' do - fill_in 'issue_search', with: 'Bug' + fill_in 'issuable_search', with: 'Bug' page.within '.issues-list' do expect(page).to have_selector('.issue', count: 2) end + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '2') + end + click_button 'Milestone' page.within '.milestone-filter' do click_link '8' @@ -256,15 +268,23 @@ describe 'Filter issues', feature: true do page.within '.issues-list' do expect(page).to have_selector('.issue', count: 1) end + + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end end it 'filters by text and assignee' do - fill_in 'issue_search', with: 'Bug' + fill_in 'issuable_search', with: 'Bug' page.within '.issues-list' do expect(page).to have_selector('.issue', count: 2) end + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '2') + end + click_button 'Assignee' page.within '.dropdown-menu-assignee' do click_link user.name @@ -273,15 +293,23 @@ describe 'Filter issues', feature: true do page.within '.issues-list' do expect(page).to have_selector('.issue', count: 1) end + + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end end it 'filters by text and author' do - fill_in 'issue_search', with: 'Bug' + fill_in 'issuable_search', with: 'Bug' page.within '.issues-list' do expect(page).to have_selector('.issue', count: 2) end + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '2') + end + click_button 'Author' page.within '.dropdown-menu-author' do click_link user.name @@ -290,6 +318,10 @@ describe 'Filter issues', feature: true do page.within '.issues-list' do expect(page).to have_selector('.issue', count: 1) end + + page.within '.issues-state-filters' do + expect(page).to have_selector('.active .badge', text: '1') + end end end end diff --git a/spec/features/issues/reset_filters_spec.rb b/spec/features/issues/reset_filters_spec.rb index 41f218eaa8b..f4d0f13c3d5 100644 --- a/spec/features/issues/reset_filters_spec.rb +++ b/spec/features/issues/reset_filters_spec.rb @@ -37,7 +37,7 @@ feature 'Issues filter reset button', feature: true, js: true do context 'when a text search has been conducted' do it 'resets the text search filter' do - visit_issues(project, issue_search: 'Bug') + visit_issues(project, search: 'Bug') expect(page).to have_css('.issue', count: 1) reset_filters @@ -67,7 +67,7 @@ feature 'Issues filter reset button', feature: true, js: true do context 'when all filters have been applied' do it 'resets all filters' do - visit_issues(project, assignee_id: user.id, author_id: user.id, milestone_title: milestone.title, label_name: bug.name, issue_search: 'Bug') + visit_issues(project, assignee_id: user.id, author_id: user.id, milestone_title: milestone.title, label_name: bug.name, search: 'Bug') expect(page).to have_css('.issue', count: 0) reset_filters diff --git a/spec/features/projects/gfm_autocomplete_load_spec.rb b/spec/features/projects/gfm_autocomplete_load_spec.rb new file mode 100644 index 00000000000..1921ea6d8ae --- /dev/null +++ b/spec/features/projects/gfm_autocomplete_load_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe 'GFM autocomplete loading', feature: true, js: true do + let(:project) { create(:project) } + + before do + login_as :admin + + visit namespace_project_path(project.namespace, project) + end + + it 'does not load on project#show' do + expect(evaluate_script('GitLab.GfmAutoComplete.dataSource')).to eq('') + end + + it 'loads on new issue page' do + visit new_namespace_project_issue_path(project.namespace, project) + + expect(evaluate_script('GitLab.GfmAutoComplete.dataSource')).not_to eq('') + end +end diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb new file mode 100644 index 00000000000..7e2c701e401 --- /dev/null +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +# Integration test that exports a file using the Import/Export feature +# It looks up for any sensitive word inside the JSON, so if a sensitive word is found +# we''l have to either include it adding the model that includes it to the +safe_list+ +# or make sure the attribute is blacklisted in the +import_export.yml+ configuration +feature 'Import/Export - project export integration test', feature: true, js: true do + include Select2Helper + include ExportFileHelper + + let(:user) { create(:admin) } + let(:export_path) { "#{Dir::tmpdir}/import_file_spec" } + let(:config_hash) { YAML.load_file(Gitlab::ImportExport.config_file).deep_stringify_keys } + + let(:sensitive_words) { %w[pass secret token key] } + let(:safe_list) do + { + token: [ProjectHook, Ci::Trigger], + key: [Project, Ci::Variable, :yaml_variables] + } + end + let(:safe_hashes) { { yaml_variables: %w[key value public] } } + + let(:project) { setup_project } + + background do + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + end + + after do + FileUtils.rm_rf(export_path, secure: true) + end + + context 'admin user' do + before do + login_as(user) + end + + scenario 'exports a project successfully' do + visit edit_namespace_project_path(project.namespace, project) + + expect(page).to have_content('Export project') + + click_link 'Export project' + + visit edit_namespace_project_path(project.namespace, project) + + expect(page).to have_content('Download export') + + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) + + project_json_path = File.join(tmpdir, 'project.json') + expect(File).to exist(project_json_path) + + project_hash = JSON.parse(IO.read(project_json_path)) + + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, project_hash) + + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end + end + + def failure_message(key_found, parent, sensitive_word) + <<-MSG + Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + + If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + + Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the + correspondent hash or model as the value. + + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + CURRENT_SPEC: #{__FILE__} + MSG + end + end +end diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb index f707ccf4e93..09cd6369881 100644 --- a/spec/features/projects/import_export/import_file_spec.rb +++ b/spec/features/projects/import_export/import_file_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -feature 'project import', feature: true, js: true do +feature 'Import/Export - project import integration test', feature: true, js: true do include Select2Helper let(:admin) { create(:admin) } diff --git a/spec/finders/issues_finder_spec.rb b/spec/finders/issues_finder_spec.rb index ec8809e6926..40bccb8e50b 100644 --- a/spec/finders/issues_finder_spec.rb +++ b/spec/finders/issues_finder_spec.rb @@ -7,8 +7,8 @@ describe IssuesFinder do let(:project2) { create(:empty_project) } let(:milestone) { create(:milestone, project: project1) } let(:label) { create(:label, project: project2) } - let(:issue1) { create(:issue, author: user, assignee: user, project: project1, milestone: milestone) } - let(:issue2) { create(:issue, author: user, assignee: user, project: project2) } + let(:issue1) { create(:issue, author: user, assignee: user, project: project1, milestone: milestone, title: 'gitlab') } + let(:issue2) { create(:issue, author: user, assignee: user, project: project2, description: 'gitlab') } let(:issue3) { create(:issue, author: user2, assignee: user2, project: project2) } let!(:label_link) { create(:label_link, label: label, target: issue2) } @@ -127,6 +127,22 @@ describe IssuesFinder do end end + context 'filtering by issue term' do + let(:params) { { search: 'git' } } + + it 'returns issues with title and description match for search term' do + expect(issues).to contain_exactly(issue1, issue2) + end + end + + context 'filtering by issue iid' do + let(:params) { { search: issue3.to_reference } } + + it 'returns issue with iid match' do + expect(issues).to contain_exactly(issue3) + end + end + context 'when the user is unauthorized' do let(:search_user) { nil } diff --git a/spec/lib/ci/mask_secret_spec.rb b/spec/lib/ci/mask_secret_spec.rb index 518de76911c..3101bed20fb 100644 --- a/spec/lib/ci/mask_secret_spec.rb +++ b/spec/lib/ci/mask_secret_spec.rb @@ -5,15 +5,23 @@ describe Ci::MaskSecret, lib: true do describe '#mask' do it 'masks exact number of characters' do - expect(subject.mask('token', 'oke')).to eq('txxxn') + expect(mask('token', 'oke')).to eq('txxxn') end it 'masks multiple occurrences' do - expect(subject.mask('token token token', 'oke')).to eq('txxxn txxxn txxxn') + expect(mask('token token token', 'oke')).to eq('txxxn txxxn txxxn') end it 'does not mask if not found' do - expect(subject.mask('token', 'not')).to eq('token') + expect(mask('token', 'not')).to eq('token') + end + + it 'does support null token' do + expect(mask('token', nil)).to eq('token') + end + + def mask(value, token) + subject.mask!(value.dup, token) end end end diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index ed43646330f..de68e32e5b4 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -343,7 +343,7 @@ describe Gitlab::GitAccess, lib: true do end context 'to private project' do - let(:project) { create(:project, :internal) } + let(:project) { create(:project) } it { expect(subject).not_to be_allowed } end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml new file mode 100644 index 00000000000..30968ba2d5f --- /dev/null +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -0,0 +1,175 @@ +--- +issues: +- subscriptions +- award_emoji +- author +- assignee +- updated_by +- milestone +- notes +- label_links +- labels +- todos +- user_agent_detail +- moved_to +- events +events: +- author +- project +- target +notes: +- award_emoji +- project +- noteable +- author +- updated_by +- resolved_by +- todos +- events +label_links: +- target +- label +label: +- subscriptions +- project +- lists +- label_links +- issues +- merge_requests +milestone: +- project +- issues +- labels +- merge_requests +- participants +- events +snippets: +- author +- project +- notes +releases: +- project +project_members: +- created_by +- user +- source +- project +merge_requests: +- subscriptions +- award_emoji +- author +- assignee +- updated_by +- milestone +- notes +- label_links +- labels +- todos +- target_project +- source_project +- merge_user +- merge_request_diffs +- merge_request_diff +- events +merge_request_diff: +- merge_request +pipelines: +- project +- user +- statuses +- builds +- trigger_requests +statuses: +- project +- pipeline +- user +variables: +- project +triggers: +- project +- trigger_requests +deploy_keys: +- user +- deploy_keys_projects +- projects +services: +- project +- service_hook +hooks: +- project +protected_branches: +- project +- merge_access_levels +- push_access_levels +project: +- taggings +- base_tags +- tag_taggings +- tags +- creator +- group +- namespace +- board +- last_event +- services +- campfire_service +- drone_ci_service +- emails_on_push_service +- builds_email_service +- irker_service +- pivotaltracker_service +- hipchat_service +- flowdock_service +- assembla_service +- asana_service +- gemnasium_service +- slack_service +- buildkite_service +- bamboo_service +- teamcity_service +- pushover_service +- jira_service +- redmine_service +- custom_issue_tracker_service +- bugzilla_service +- gitlab_issue_tracker_service +- external_wiki_service +- forked_project_link +- forked_from_project +- forked_project_links +- forks +- merge_requests +- fork_merge_requests +- issues +- labels +- events +- milestones +- notes +- snippets +- hooks +- protected_branches +- project_members +- users +- requesters +- deploy_keys_projects +- deploy_keys +- users_star_projects +- starrers +- releases +- lfs_objects_projects +- lfs_objects +- project_group_links +- invited_groups +- todos +- notification_settings +- import_data +- commit_statuses +- pipelines +- builds +- runner_projects +- runners +- variables +- triggers +- environments +- deployments +- project_feature
\ No newline at end of file diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb new file mode 100644 index 00000000000..2ba344092ce --- /dev/null +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +# Part of the test security suite for the Import/Export feature +# Checks whether there are new attributes in models that are currently being exported as part of the +# project Import/Export feature. +# If there are new attributes, these will have to either be added to this spec in case we want them +# to be included as part of the export, or blacklist them using the import_export.yml configuration file. +# Likewise, new models added to import_export.yml, will need to be added with their correspondent attributes +# to this spec. +describe 'Import/Export attribute configuration', lib: true do + include ConfigurationHelper + + let(:config_hash) { YAML.load_file(Gitlab::ImportExport.config_file).deep_stringify_keys } + let(:relation_names) do + names = names_from_tree(config_hash['project_tree']) + + # Remove duplicated or add missing models + # - project is not part of the tree, so it has to be added manually. + # - milestone, labels have both singular and plural versions in the tree, so remove the duplicates. + names.flatten.uniq - ['milestones', 'labels'] + ['project'] + end + + let(:safe_attributes_file) { 'spec/lib/gitlab/import_export/safe_model_attributes.yml' } + let(:safe_model_attributes) { YAML.load_file(safe_attributes_file) } + + it 'has no new columns' do + relation_names.each do |relation_name| + relation_class = relation_class_for_name(relation_name) + + expect(safe_model_attributes[relation_class.to_s]).not_to be_nil, "Expected exported class #{relation_class.to_s} to exist in safe_model_attributes" + + current_attributes = parsed_attributes(relation_name, relation_class.attribute_names) + safe_attributes = safe_model_attributes[relation_class.to_s] + new_attributes = current_attributes - safe_attributes + + expect(new_attributes).to be_empty, failure_message(relation_class.to_s, new_attributes) + end + end + + def failure_message(relation_class, new_attributes) + <<-MSG + It looks like #{relation_class}, which is exported using the project Import/Export, has new attributes: #{new_attributes.join(',')} + + Please add the attribute(s) to SAFE_MODEL_ATTRIBUTES if you consider this can be exported. + Otherwise, please blacklist the attribute(s) in IMPORT_EXPORT_CONFIG by adding it to its correspondent + model in the +excluded_attributes+ section. + + SAFE_MODEL_ATTRIBUTES: #{File.expand_path(safe_attributes_file)} + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + MSG + end + + class Author < User + end +end diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb new file mode 100644 index 00000000000..9b492d1b9c7 --- /dev/null +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +# Part of the test security suite for the Import/Export feature +# Finds if a new model has been added that can potentially be part of the Import/Export +# If it finds a new model, it will show a +failure_message+ with the options available. +describe 'Import/Export model configuration', lib: true do + include ConfigurationHelper + + let(:config_hash) { YAML.load_file(Gitlab::ImportExport.config_file).deep_stringify_keys } + let(:model_names) do + names = names_from_tree(config_hash['project_tree']) + + # Remove duplicated or add missing models + # - project is not part of the tree, so it has to be added manually. + # - milestone, labels have both singular and plural versions in the tree, so remove the duplicates. + # - User, Author... Models we do not care about for checking models + names.flatten.uniq - ['milestones', 'labels', 'user', 'author'] + ['project'] + end + + let(:all_models_yml) { 'spec/lib/gitlab/import_export/all_models.yml' } + let(:all_models) { YAML.load_file(all_models_yml) } + let(:current_models) { setup_models } + + it 'has no new models' do + model_names.each do |model_name| + new_models = Array(current_models[model_name]) - Array(all_models[model_name]) + expect(new_models).to be_empty, failure_message(model_name.classify, new_models) + end + end + + # List of current models between models, in the format of + # {model: [model_2, model3], ...} + def setup_models + all_models_hash = {} + + model_names.each do |model_name| + model_class = relation_class_for_name(model_name) + + all_models_hash[model_name] = associations_for(model_class) - ['project'] + end + + all_models_hash + end + + def failure_message(parent_model_name, new_models) + <<-MSG + New model(s) <#{new_models.join(',')}> have been added, related to #{parent_model_name}, which is exported by + the Import/Export feature. + + If you think this model should be included in the export, please add it to IMPORT_EXPORT_CONFIG. + Definitely add it to MODELS_JSON to signal that you've handled this error and to prevent it from showing up in the future. + + MODELS_JSON: #{File.expand_path(all_models_yml)} + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + MSG + end +end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml new file mode 100644 index 00000000000..f2d272ca7e2 --- /dev/null +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -0,0 +1,310 @@ +--- +Issue: +- id +- title +- assignee_id +- author_id +- project_id +- created_at +- updated_at +- position +- branch_name +- description +- state +- iid +- updated_by_id +- confidential +- deleted_at +- due_date +- moved_to_id +- lock_version +- milestone_id +- weight +Event: +- id +- target_type +- target_id +- title +- data +- project_id +- created_at +- updated_at +- action +- author_id +Note: +- id +- note +- noteable_type +- author_id +- created_at +- updated_at +- project_id +- attachment +- line_code +- commit_id +- noteable_id +- system +- st_diff +- updated_by_id +- type +- position +- original_position +- resolved_at +- resolved_by_id +- discussion_id +- original_discussion_id +LabelLink: +- id +- label_id +- target_id +- target_type +- created_at +- updated_at +Label: +- id +- title +- color +- project_id +- created_at +- updated_at +- template +- description +- priority +Milestone: +- id +- title +- project_id +- description +- due_date +- created_at +- updated_at +- state +- iid +ProjectSnippet: +- id +- title +- content +- author_id +- project_id +- created_at +- updated_at +- file_name +- type +- visibility_level +Release: +- id +- tag +- description +- project_id +- created_at +- updated_at +ProjectMember: +- id +- access_level +- source_id +- source_type +- user_id +- notification_level +- type +- created_at +- updated_at +- created_by_id +- invite_email +- invite_token +- invite_accepted_at +- requested_at +- expires_at +User: +- id +- username +- email +MergeRequest: +- id +- target_branch +- source_branch +- source_project_id +- author_id +- assignee_id +- title +- created_at +- updated_at +- state +- merge_status +- target_project_id +- iid +- description +- position +- locked_at +- updated_by_id +- merge_error +- merge_params +- merge_when_build_succeeds +- merge_user_id +- merge_commit_sha +- deleted_at +- in_progress_merge_commit_sha +- lock_version +- milestone_id +- approvals_before_merge +- rebase_commit_sha +MergeRequestDiff: +- id +- state +- st_commits +- merge_request_id +- created_at +- updated_at +- base_commit_sha +- real_size +- head_commit_sha +- start_commit_sha +Ci::Pipeline: +- id +- project_id +- ref +- sha +- before_sha +- push_data +- created_at +- updated_at +- tag +- yaml_errors +- committed_at +- gl_project_id +- status +- started_at +- finished_at +- duration +- user_id +CommitStatus: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- gl_project_id +- artifacts_metadata +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +Ci::Variable: +- id +- project_id +- key +- value +- encrypted_value +- encrypted_value_salt +- encrypted_value_iv +- gl_project_id +Ci::Trigger: +- id +- token +- project_id +- deleted_at +- created_at +- updated_at +- gl_project_id +DeployKey: +- id +- user_id +- created_at +- updated_at +- key +- title +- type +- fingerprint +- public +Service: +- id +- type +- title +- project_id +- created_at +- updated_at +- active +- properties +- template +- push_events +- issues_events +- merge_requests_events +- tag_push_events +- note_events +- pipeline_events +- build_events +- category +- default +- wiki_page_events +- confidential_issues_events +ProjectHook: +- id +- url +- project_id +- created_at +- updated_at +- type +- service_id +- push_events +- issues_events +- merge_requests_events +- tag_push_events +- note_events +- pipeline_events +- enable_ssl_verification +- build_events +- wiki_page_events +- token +- group_id +- confidential_issues_events +ProtectedBranch: +- id +- project_id +- name +- created_at +- updated_at +Project: +- description +- issues_enabled +- merge_requests_enabled +- wiki_enabled +- snippets_enabled +- visibility_level +- archived +Author: +- name +ProjectFeature: +- id +- project_id +- merge_requests_access_level +- issues_access_level +- wiki_access_level +- snippets_access_level +- builds_access_level +- created_at +- updated_at
\ No newline at end of file diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index f1857f846dc..550a890797e 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -187,6 +187,37 @@ describe Ci::Pipeline, models: true do end end + describe "merge request metrics" do + let(:project) { FactoryGirl.create :project } + let(:pipeline) { FactoryGirl.create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master', sha: project.repository.commit('master').id) } + let!(:merge_request) { create(:merge_request, source_project: project, source_branch: pipeline.ref) } + + context 'when transitioning to running' do + it 'records the build start time' do + time = Time.now + Timecop.freeze(time) { build.run } + + expect(merge_request.reload.metrics.latest_build_started_at).to be_within(1.second).of(time) + end + + it 'clears the build end time' do + build.run + + expect(merge_request.reload.metrics.latest_build_finished_at).to be_nil + end + end + + context 'when transitioning to success' do + it 'records the build end time' do + build.run + time = Time.now + Timecop.freeze(time) { build.success } + + expect(merge_request.reload.metrics.latest_build_finished_at).to be_within(1.second).of(time) + end + end + end + def create_build(name, queued_at = current, started_from = 0) create(:ci_build, name: name, @@ -468,4 +499,28 @@ describe Ci::Pipeline, models: true do stage_idx: stage_idx) end end + + describe "#merge_requests" do + let(:project) { FactoryGirl.create :project } + let(:pipeline) { FactoryGirl.create(:ci_empty_pipeline, status: 'created', project: project, ref: 'master', sha: project.repository.commit('master').id) } + + it "returns merge requests whose `diff_head_sha` matches the pipeline's SHA" do + merge_request = create(:merge_request, source_project: project, source_branch: pipeline.ref) + + expect(pipeline.merge_requests).to eq([merge_request]) + end + + it "doesn't return merge requests whose source branch doesn't match the pipeline's ref" do + create(:merge_request, source_project: project, source_branch: 'feature', target_branch: 'master') + + expect(pipeline.merge_requests).to be_empty + end + + it "doesn't return merge requests whose `diff_head_sha` doesn't match the pipeline's SHA" do + create(:merge_request, source_project: project, source_branch: pipeline.ref) + allow_any_instance_of(MergeRequest).to receive(:diff_head_sha) { '97de212e80737a608d939f648d959671fb0a0142b' } + + expect(pipeline.merge_requests).to be_empty + end + end end diff --git a/spec/models/cycle_analytics/code_spec.rb b/spec/models/cycle_analytics/code_spec.rb new file mode 100644 index 00000000000..b9381e33914 --- /dev/null +++ b/spec/models/cycle_analytics/code_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe 'CycleAnalytics#code', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :code, + data_fn: -> (context) { { issue: context.create(:issue, project: context.project) } }, + start_time_conditions: [["issue mentioned in a commit", + -> (context, data) do + context.create_commit_referencing_issue(data[:issue]) + end]], + end_time_conditions: [["merge request that closes issue is created", + -> (context, data) do + context.create_merge_request_closing_issue(data[:issue]) + end]], + post_fn: -> (context, data) do + context.merge_merge_requests_closing_issue(data[:issue]) + context.deploy_master + end) + + context "when a regular merge request (that doesn't close the issue) is created" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + + create_commit_referencing_issue(issue) + create_merge_request_closing_issue(issue, message: "Closes nothing") + + merge_merge_requests_closing_issue(issue) + deploy_master + end + + expect(subject.code).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/issue_spec.rb b/spec/models/cycle_analytics/issue_spec.rb new file mode 100644 index 00000000000..e9cc71254ab --- /dev/null +++ b/spec/models/cycle_analytics/issue_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe 'CycleAnalytics#issue', models: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :issue, + data_fn: -> (context) { { issue: context.build(:issue, project: context.project) } }, + start_time_conditions: [["issue created", -> (context, data) { data[:issue].save }]], + end_time_conditions: [["issue associated with a milestone", + -> (context, data) do + if data[:issue].persisted? + data[:issue].update(milestone: context.create(:milestone, project: context.project)) + end + end], + ["list label added to issue", + -> (context, data) do + if data[:issue].persisted? + data[:issue].update(label_ids: [context.create(:label, lists: [context.create(:list)]).id]) + end + end]], + post_fn: -> (context, data) do + if data[:issue].persisted? + context.create_merge_request_closing_issue(data[:issue].reload) + context.merge_merge_requests_closing_issue(data[:issue]) + context.deploy_master + end + end) + + context "when a regular label (instead of a list label) is added to the issue" do + it "returns nil" do + 5.times do + regular_label = create(:label) + issue = create(:issue, project: project) + issue.update(label_ids: [regular_label.id]) + + create_merge_request_closing_issue(issue) + merge_merge_requests_closing_issue(issue) + deploy_master + end + + expect(subject.issue).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/plan_spec.rb b/spec/models/cycle_analytics/plan_spec.rb new file mode 100644 index 00000000000..5b8c96dc992 --- /dev/null +++ b/spec/models/cycle_analytics/plan_spec.rb @@ -0,0 +1,52 @@ +require 'spec_helper' + +describe 'CycleAnalytics#plan', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :plan, + data_fn: -> (context) do + { + issue: context.create(:issue, project: context.project), + branch_name: context.random_git_name + } + end, + start_time_conditions: [["issue associated with a milestone", + -> (context, data) do + data[:issue].update(milestone: context.create(:milestone, project: context.project)) + end], + ["list label added to issue", + -> (context, data) do + data[:issue].update(label_ids: [context.create(:label, lists: [context.create(:list)]).id]) + end]], + end_time_conditions: [["issue mentioned in a commit", + -> (context, data) do + context.create_commit_referencing_issue(data[:issue], branch_name: data[:branch_name]) + end]], + post_fn: -> (context, data) do + context.create_merge_request_closing_issue(data[:issue], source_branch: data[:branch_name]) + context.merge_merge_requests_closing_issue(data[:issue]) + context.deploy_master + end) + + context "when a regular label (instead of a list label) is added to the issue" do + it "returns nil" do + branch_name = random_git_name + label = create(:label) + issue = create(:issue, project: project) + issue.update(label_ids: [label.id]) + create_commit_referencing_issue(issue, branch_name: branch_name) + + create_merge_request_closing_issue(issue, source_branch: branch_name) + merge_merge_requests_closing_issue(issue) + deploy_master + + expect(subject.issue).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/production_spec.rb b/spec/models/cycle_analytics/production_spec.rb new file mode 100644 index 00000000000..1f5e5cab92d --- /dev/null +++ b/spec/models/cycle_analytics/production_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +describe 'CycleAnalytics#production', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :production, + data_fn: -> (context) { { issue: context.build(:issue, project: context.project) } }, + start_time_conditions: [["issue is created", -> (context, data) { data[:issue].save }]], + before_end_fn: lambda do |context, data| + context.create_merge_request_closing_issue(data[:issue]) + context.merge_merge_requests_closing_issue(data[:issue]) + end, + end_time_conditions: + [["merge request that closes issue is deployed to production", -> (context, data) { context.deploy_master }], + ["production deploy happens after merge request is merged (along with other changes)", + lambda do |context, data| + # Make other changes on master + sha = context.project.repository.commit_file(context.user, context.random_git_name, "content", "commit message", 'master', false) + context.project.repository.commit(sha) + + context.deploy_master + end]]) + + context "when a regular merge request (that doesn't close the issue) is merged and deployed" do + it "returns nil" do + 5.times do + merge_request = create(:merge_request) + MergeRequests::MergeService.new(project, user).execute(merge_request) + deploy_master + end + + expect(subject.production).to be_nil + end + end + + context "when the deployment happens to a non-production environment" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + merge_request = create_merge_request_closing_issue(issue) + MergeRequests::MergeService.new(project, user).execute(merge_request) + deploy_master(environment: 'staging') + end + + expect(subject.production).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/review_spec.rb b/spec/models/cycle_analytics/review_spec.rb new file mode 100644 index 00000000000..b6e26d8f261 --- /dev/null +++ b/spec/models/cycle_analytics/review_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe 'CycleAnalytics#review', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :review, + data_fn: -> (context) { { issue: context.create(:issue, project: context.project) } }, + start_time_conditions: [["merge request that closes issue is created", + -> (context, data) do + context.create_merge_request_closing_issue(data[:issue]) + end]], + end_time_conditions: [["merge request that closes issue is merged", + -> (context, data) do + context.merge_merge_requests_closing_issue(data[:issue]) + end]], + post_fn: -> (context, data) { context.deploy_master }) + + context "when a regular merge request (that doesn't close the issue) is created and merged" do + it "returns nil" do + 5.times do + MergeRequests::MergeService.new(project, user).execute(create(:merge_request)) + + deploy_master + end + + expect(subject.review).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/staging_spec.rb b/spec/models/cycle_analytics/staging_spec.rb new file mode 100644 index 00000000000..af1c4477ddb --- /dev/null +++ b/spec/models/cycle_analytics/staging_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe 'CycleAnalytics#staging', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :staging, + data_fn: lambda do |context| + issue = context.create(:issue, project: context.project) + { issue: issue, merge_request: context.create_merge_request_closing_issue(issue) } + end, + start_time_conditions: [["merge request that closes issue is merged", + -> (context, data) do + context.merge_merge_requests_closing_issue(data[:issue]) + end ]], + end_time_conditions: [["merge request that closes issue is deployed to production", + -> (context, data) do + context.deploy_master + end], + ["production deploy happens after merge request is merged (along with other changes)", + lambda do |context, data| + # Make other changes on master + sha = context.project.repository.commit_file( + context.user, + context.random_git_name, + "content", + "commit message", + 'master', + false) + context.project.repository.commit(sha) + + context.deploy_master + end]]) + + context "when a regular merge request (that doesn't close the issue) is merged and deployed" do + it "returns nil" do + 5.times do + merge_request = create(:merge_request) + MergeRequests::MergeService.new(project, user).execute(merge_request) + deploy_master + end + + expect(subject.staging).to be_nil + end + end + + context "when the deployment happens to a non-production environment" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + merge_request = create_merge_request_closing_issue(issue) + MergeRequests::MergeService.new(project, user).execute(merge_request) + deploy_master(environment: 'staging') + end + + expect(subject.staging).to be_nil + end + end +end diff --git a/spec/models/cycle_analytics/summary_spec.rb b/spec/models/cycle_analytics/summary_spec.rb new file mode 100644 index 00000000000..743bc2da33f --- /dev/null +++ b/spec/models/cycle_analytics/summary_spec.rb @@ -0,0 +1,53 @@ +require 'spec_helper' + +describe CycleAnalytics::Summary, models: true do + let(:project) { create(:project) } + let(:from) { Time.now } + let(:user) { create(:user, :admin) } + subject { described_class.new(project, from: from) } + + describe "#new_issues" do + it "finds the number of issues created after the 'from date'" do + Timecop.freeze(5.days.ago) { create(:issue, project: project) } + Timecop.freeze(5.days.from_now) { create(:issue, project: project) } + + expect(subject.new_issues).to eq(1) + end + + it "doesn't find issues from other projects" do + Timecop.freeze(5.days.from_now) { create(:issue, project: create(:project)) } + + expect(subject.new_issues).to eq(0) + end + end + + describe "#commits" do + it "finds the number of commits created after the 'from date'" do + Timecop.freeze(5.days.ago) { create_commit("Test message", project, user, 'master') } + Timecop.freeze(5.days.from_now) { create_commit("Test message", project, user, 'master') } + + expect(subject.commits).to eq(1) + end + + it "doesn't find commits from other projects" do + Timecop.freeze(5.days.from_now) { create_commit("Test message", create(:project), user, 'master') } + + expect(subject.commits).to eq(0) + end + end + + describe "#deploys" do + it "finds the number of deploys made created after the 'from date'" do + Timecop.freeze(5.days.ago) { create(:deployment, project: project) } + Timecop.freeze(5.days.from_now) { create(:deployment, project: project) } + + expect(subject.deploys).to eq(1) + end + + it "doesn't find commits from other projects" do + Timecop.freeze(5.days.from_now) { create(:deployment, project: create(:project)) } + + expect(subject.deploys).to eq(0) + end + end +end diff --git a/spec/models/cycle_analytics/test_spec.rb b/spec/models/cycle_analytics/test_spec.rb new file mode 100644 index 00000000000..89ace0b2742 --- /dev/null +++ b/spec/models/cycle_analytics/test_spec.rb @@ -0,0 +1,94 @@ +require 'spec_helper' + +describe 'CycleAnalytics#test', feature: true do + extend CycleAnalyticsHelpers::TestGeneration + + let(:project) { create(:project) } + let(:from_date) { 10.days.ago } + let(:user) { create(:user, :admin) } + subject { CycleAnalytics.new(project, from: from_date) } + + generate_cycle_analytics_spec( + phase: :test, + data_fn: lambda do |context| + issue = context.create(:issue, project: context.project) + merge_request = context.create_merge_request_closing_issue(issue) + pipeline = context.create(:ci_pipeline, ref: merge_request.source_branch, sha: merge_request.diff_head_sha, project: context.project) + { pipeline: pipeline, issue: issue } + end, + start_time_conditions: [["pipeline is started", -> (context, data) { data[:pipeline].run! }]], + end_time_conditions: [["pipeline is finished", -> (context, data) { data[:pipeline].succeed! }]], + post_fn: -> (context, data) do + context.merge_merge_requests_closing_issue(data[:issue]) + context.deploy_master + end) + + context "when the pipeline is for a regular merge request (that doesn't close an issue)" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + merge_request = create_merge_request_closing_issue(issue) + pipeline = create(:ci_pipeline, ref: "refs/heads/#{merge_request.source_branch}", sha: merge_request.diff_head_sha) + + pipeline.run! + pipeline.succeed! + + merge_merge_requests_closing_issue(issue) + deploy_master + end + + expect(subject.test).to be_nil + end + end + + context "when the pipeline is not for a merge request" do + it "returns nil" do + 5.times do + pipeline = create(:ci_pipeline, ref: "refs/heads/master", sha: project.repository.commit('master').sha) + + pipeline.run! + pipeline.succeed! + + deploy_master + end + + expect(subject.test).to be_nil + end + end + + context "when the pipeline is dropped (failed)" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + merge_request = create_merge_request_closing_issue(issue) + pipeline = create(:ci_pipeline, ref: "refs/heads/#{merge_request.source_branch}", sha: merge_request.diff_head_sha) + + pipeline.run! + pipeline.drop! + + merge_merge_requests_closing_issue(issue) + deploy_master + end + + expect(subject.test).to be_nil + end + end + + context "when the pipeline is cancelled" do + it "returns nil" do + 5.times do + issue = create(:issue, project: project) + merge_request = create_merge_request_closing_issue(issue) + pipeline = create(:ci_pipeline, ref: "refs/heads/#{merge_request.source_branch}", sha: merge_request.diff_head_sha) + + pipeline.run! + pipeline.cancel! + + merge_merge_requests_closing_issue(issue) + deploy_master + end + + expect(subject.test).to be_nil + end + end +end diff --git a/spec/models/issue/metrics_spec.rb b/spec/models/issue/metrics_spec.rb new file mode 100644 index 00000000000..e170b087ebc --- /dev/null +++ b/spec/models/issue/metrics_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +describe Issue::Metrics, models: true do + let(:project) { create(:project) } + + subject { create(:issue, project: project) } + + describe "when recording the default set of issue metrics on issue save" do + context "milestones" do + it "records the first time an issue is associated with a milestone" do + time = Time.now + Timecop.freeze(time) { subject.update(milestone: create(:milestone)) } + metrics = subject.metrics + + expect(metrics).to be_present + expect(metrics.first_associated_with_milestone_at).to be_within(1.second).of(time) + end + + it "does not record the second time an issue is associated with a milestone" do + time = Time.now + Timecop.freeze(time) { subject.update(milestone: create(:milestone)) } + Timecop.freeze(time + 2.hours) { subject.update(milestone: nil) } + Timecop.freeze(time + 6.hours) { subject.update(milestone: create(:milestone)) } + metrics = subject.metrics + + expect(metrics).to be_present + expect(metrics.first_associated_with_milestone_at).to be_within(1.second).of(time) + end + end + + context "list labels" do + it "records the first time an issue is associated with a list label" do + list_label = create(:label, lists: [create(:list)]) + time = Time.now + Timecop.freeze(time) { subject.update(label_ids: [list_label.id]) } + metrics = subject.metrics + + expect(metrics).to be_present + expect(metrics.first_added_to_board_at).to be_within(1.second).of(time) + end + + it "does not record the second time an issue is associated with a list label" do + time = Time.now + first_list_label = create(:label, lists: [create(:list)]) + Timecop.freeze(time) { subject.update(label_ids: [first_list_label.id]) } + second_list_label = create(:label, lists: [create(:list)]) + Timecop.freeze(time + 5.hours) { subject.update(label_ids: [second_list_label.id]) } + metrics = subject.metrics + + expect(metrics).to be_present + expect(metrics.first_added_to_board_at).to be_within(1.second).of(time) + end + end + end +end diff --git a/spec/models/merge_request/metrics_spec.rb b/spec/models/merge_request/metrics_spec.rb new file mode 100644 index 00000000000..a79dd215d41 --- /dev/null +++ b/spec/models/merge_request/metrics_spec.rb @@ -0,0 +1,18 @@ +require 'spec_helper' + +describe MergeRequest::Metrics, models: true do + let(:project) { create(:project) } + + subject { create(:merge_request, source_project: project) } + + describe "when recording the default set of metrics on merge request save" do + it "records the merge time" do + time = Time.now + Timecop.freeze(time) { subject.mark_as_merged } + metrics = subject.metrics + + expect(metrics).to be_present + expect(metrics.merged_at).to be_within(1.second).of(time) + end + end +end diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index eda1cafd65e..a7a06744428 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -33,4 +33,17 @@ describe ProjectPolicy, models: true do it 'returns increasing permissions for each level' do expect(users_permissions).to eq(users_permissions.sort.uniq) end + + it 'does not include the read_issue permission when the issue author is not a member of the private project' do + project = create(:project, :private) + issue = create(:issue, project: project) + user = issue.author + + expect(project.team.member?(issue.author)).to eq(false) + + expect(BasePolicy.class_for(project).abilities(user, project).can_set). + not_to include(:read_issue) + + expect(Ability.allowed?(user, :read_issue, project)).to be_falsy + end end diff --git a/spec/requests/api/api_helpers_spec.rb b/spec/requests/api/api_helpers_spec.rb index bbdf8f03c2b..e66faeed705 100644 --- a/spec/requests/api/api_helpers_spec.rb +++ b/spec/requests/api/api_helpers_spec.rb @@ -36,11 +36,36 @@ describe API::Helpers, api: true do params.delete(API::Helpers::SUDO_PARAM) end + def warden_authenticate_returns(value) + warden = double("warden", authenticate: value) + env['warden'] = warden + end + + def doorkeeper_guard_returns(value) + allow_any_instance_of(self.class).to receive(:doorkeeper_guard){ value } + end + def error!(message, status) raise Exception end describe ".current_user" do + subject { current_user } + + describe "when authenticating via Warden" do + before { doorkeeper_guard_returns false } + + context "fails" do + it { is_expected.to be_nil } + end + + context "succeeds" do + before { warden_authenticate_returns user } + + it { is_expected.to eq(user) } + end + end + describe "when authenticating using a user's private token" do it "returns nil for an invalid token" do env[API::Helpers::PRIVATE_TOKEN_HEADER] = 'invalid token' diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index e3922bec689..74516686921 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -335,7 +335,7 @@ describe 'Git HTTP requests', lib: true do project.team << [user, :reporter] end - shared_examples 'can download code only from own projects' do + shared_examples 'can download code only' do it 'downloads get status 200' do clone_get "#{project.path_with_namespace}.git", user: 'gitlab-ci-token', password: build.token @@ -353,7 +353,7 @@ describe 'Git HTTP requests', lib: true do context 'administrator' do let(:user) { create(:admin) } - it_behaves_like 'can download code only from own projects' + it_behaves_like 'can download code only' it 'downloads from other project get status 403' do clone_get "#{other_project.path_with_namespace}.git", user: 'gitlab-ci-token', password: build.token @@ -365,7 +365,7 @@ describe 'Git HTTP requests', lib: true do context 'regular user' do let(:user) { create(:user) } - it_behaves_like 'can download code only from own projects' + it_behaves_like 'can download code only' it 'downloads from other project get status 404' do clone_get "#{other_project.path_with_namespace}.git", user: 'gitlab-ci-token', password: build.token diff --git a/spec/services/create_deployment_service_spec.rb b/spec/services/create_deployment_service_spec.rb index 41b897f36cd..343b4385bf2 100644 --- a/spec/services/create_deployment_service_spec.rb +++ b/spec/services/create_deployment_service_spec.rb @@ -169,4 +169,83 @@ describe CreateDeploymentService, services: true do end end end + + describe "merge request metrics" do + let(:params) do + { + environment: 'production', + ref: 'master', + tag: false, + sha: '97de212e80737a608d939f648d959671fb0a0142b', + } + end + + let(:merge_request) { create(:merge_request, target_branch: 'master', source_branch: 'feature', source_project: project) } + + context "while updating the 'first_deployed_to_production_at' time" do + before { merge_request.mark_as_merged } + + context "for merge requests merged before the current deploy" do + it "sets the time if the deploy's environment is 'production'" do + time = Time.now + Timecop.freeze(time) { service.execute } + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_within(1.second).of(time) + end + + it "doesn't set the time if the deploy's environment is not 'production'" do + staging_params = params.merge(environment: 'staging') + service = described_class.new(project, user, staging_params) + service.execute + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_nil + end + + it 'does not raise errors if the merge request does not have a metrics record' do + merge_request.metrics.destroy + + expect(merge_request.reload.metrics).to be_nil + expect { service.execute }.not_to raise_error + end + end + + context "for merge requests merged before the previous deploy" do + context "if the 'first_deployed_to_production_at' time is already set" do + it "does not overwrite the older 'first_deployed_to_production_at' time" do + # Previous deploy + time = Time.now + Timecop.freeze(time) { service.execute } + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_within(1.second).of(time) + + # Current deploy + service = described_class.new(project, user, params) + Timecop.freeze(time + 12.hours) { service.execute } + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_within(1.second).of(time) + end + end + + context "if the 'first_deployed_to_production_at' time is not already set" do + it "does not overwrite the older 'first_deployed_to_production_at' time" do + # Previous deploy + time = 5.minutes.from_now + Timecop.freeze(time) { service.execute } + + expect(merge_request.reload.metrics.merged_at).to be < merge_request.reload.metrics.first_deployed_to_production_at + + merge_request.reload.metrics.update(first_deployed_to_production_at: nil) + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_nil + + # Current deploy + service = described_class.new(project, user, params) + Timecop.freeze(time + 12.hours) { service.execute } + + expect(merge_request.reload.metrics.first_deployed_to_production_at).to be_nil + end + end + end + end + end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 22724434a7f..22991c5bc86 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -339,6 +339,43 @@ describe GitPushService, services: true do end end + describe "issue metrics" do + let(:issue) { create :issue, project: project } + let(:commit_author) { create :user } + let(:commit) { project.commit } + let(:commit_time) { Time.now } + + before do + project.team << [commit_author, :developer] + project.team << [user, :developer] + + allow(commit).to receive_messages( + safe_message: "this commit \n mentions #{issue.to_reference}", + references: [issue], + author_name: commit_author.name, + author_email: commit_author.email, + committed_date: commit_time + ) + + allow(project.repository).to receive(:commits_between).and_return([commit]) + end + + context "while saving the 'first_mentioned_in_commit_at' metric for an issue" do + it 'sets the metric for referenced issues' do + execute_service(project, user, @oldrev, @newrev, @ref) + + expect(issue.reload.metrics.first_mentioned_in_commit_at).to be_within(1.second).of(commit_time) + end + + it 'does not set the metric for non-referenced issues' do + non_referenced_issue = create(:issue, project: project) + execute_service(project, user, @oldrev, @newrev, @ref) + + expect(non_referenced_issue.reload.metrics.first_mentioned_in_commit_at).to be_nil + end + end + end + describe "closing issues from pushed commits containing a closing reference" do let(:issue) { create :issue, project: project } let(:other_issue) { create :issue, project: project } diff --git a/spec/services/merge_requests/create_service_spec.rb b/spec/services/merge_requests/create_service_spec.rb index c1e4f8bd96b..b8142889075 100644 --- a/spec/services/merge_requests/create_service_spec.rb +++ b/spec/services/merge_requests/create_service_spec.rb @@ -83,5 +83,34 @@ describe MergeRequests::CreateService, services: true do } end end + + context 'while saving references to issues that the created merge request closes' do + let(:first_issue) { create(:issue, project: project) } + let(:second_issue) { create(:issue, project: project) } + + let(:opts) do + { + title: 'Awesome merge_request', + source_branch: 'feature', + target_branch: 'master', + force_remove_source_branch: '1' + } + end + + before do + project.team << [user, :master] + project.team << [assignee, :developer] + end + + it 'creates a `MergeRequestsClosingIssues` record for each issue' do + issue_closing_opts = opts.merge(description: "Closes #{first_issue.to_reference} and #{second_issue.to_reference}") + service = described_class.new(project, user, issue_closing_opts) + allow(service).to receive(:execute_hooks) + merge_request = service.execute + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to match_array([first_issue.id, second_issue.id]) + end + end end end diff --git a/spec/services/merge_requests/refresh_service_spec.rb b/spec/services/merge_requests/refresh_service_spec.rb index fff86480c6d..a162df5fc34 100644 --- a/spec/services/merge_requests/refresh_service_spec.rb +++ b/spec/services/merge_requests/refresh_service_spec.rb @@ -174,6 +174,58 @@ describe MergeRequests::RefreshService, services: true do end end + context 'merge request metrics' do + let(:issue) { create :issue, project: @project } + let(:commit_author) { create :user } + let(:commit) { project.commit } + + before do + project.team << [commit_author, :developer] + project.team << [user, :developer] + + allow(commit).to receive_messages( + safe_message: "Closes #{issue.to_reference}", + references: [issue], + author_name: commit_author.name, + author_email: commit_author.email, + committed_date: Time.now + ) + + allow_any_instance_of(MergeRequest).to receive(:commits).and_return([commit]) + end + + context 'when the merge request is sourced from the same project' do + it 'creates a `MergeRequestsClosingIssues` record for each issue closed by a commit' do + merge_request = create(:merge_request, target_branch: 'master', source_branch: 'feature', source_project: @project) + refresh_service = service.new(@project, @user) + allow(refresh_service).to receive(:execute_hooks) + refresh_service.execute(@oldrev, @newrev, 'refs/heads/feature') + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to eq([issue.id]) + end + end + + context 'when the merge request is sourced from a different project' do + it 'creates a `MergeRequestsClosingIssues` record for each issue closed by a commit' do + forked_project = create(:project) + create(:forked_project_link, forked_to_project: forked_project, forked_from_project: @project) + + merge_request = create(:merge_request, + target_branch: 'master', + source_branch: 'feature', + target_project: @project, + source_project: forked_project) + refresh_service = service.new(@project, @user) + allow(refresh_service).to receive(:execute_hooks) + refresh_service.execute(@oldrev, @newrev, 'refs/heads/feature') + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to eq([issue.id]) + end + end + end + def reload_mrs @merge_request.reload @fork_merge_request.reload diff --git a/spec/services/merge_requests/update_service_spec.rb b/spec/services/merge_requests/update_service_spec.rb index 6dfeb581975..33db34c0f62 100644 --- a/spec/services/merge_requests/update_service_spec.rb +++ b/spec/services/merge_requests/update_service_spec.rb @@ -263,5 +263,42 @@ describe MergeRequests::UpdateService, services: true do end end end + + context 'while saving references to issues that the updated merge request closes' do + let(:first_issue) { create(:issue, project: project) } + let(:second_issue) { create(:issue, project: project) } + + it 'creates a `MergeRequestsClosingIssues` record for each issue' do + issue_closing_opts = { description: "Closes #{first_issue.to_reference} and #{second_issue.to_reference}" } + service = described_class.new(project, user, issue_closing_opts) + allow(service).to receive(:execute_hooks) + service.execute(merge_request) + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to match_array([first_issue.id, second_issue.id]) + end + + it 'removes `MergeRequestsClosingIssues` records when issues are not closed anymore' do + opts = { + title: 'Awesome merge_request', + description: "Closes #{first_issue.to_reference} and #{second_issue.to_reference}", + source_branch: 'feature', + target_branch: 'master', + force_remove_source_branch: '1' + } + + merge_request = MergeRequests::CreateService.new(project, user, opts).execute + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to match_array([first_issue.id, second_issue.id]) + + service = described_class.new(project, user, description: "not closing any issues") + allow(service).to receive(:execute_hooks) + service.execute(merge_request.reload) + + issue_ids = MergeRequestsClosingIssues.where(merge_request: merge_request).pluck(:issue_id) + expect(issue_ids).to be_empty + end + end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index f81a58899fd..0d152534c38 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -379,6 +379,7 @@ describe NotificationService, services: true do it "emails subscribers of the issue's labels" do subscriber = create(:user) label = create(:label, issues: [issue]) + issue.reload label.toggle_subscription(subscriber) notification.new_issue(issue, @u_disabled) @@ -399,6 +400,7 @@ describe NotificationService, services: true do project.team << [guest, :guest] label = create(:label, issues: [confidential_issue]) + confidential_issue.reload label.toggle_subscription(non_member) label.toggle_subscription(author) label.toggle_subscription(assignee) diff --git a/spec/support/cycle_analytics_helpers.rb b/spec/support/cycle_analytics_helpers.rb new file mode 100644 index 00000000000..e8e760a6187 --- /dev/null +++ b/spec/support/cycle_analytics_helpers.rb @@ -0,0 +1,64 @@ +module CycleAnalyticsHelpers + def create_commit_referencing_issue(issue, branch_name: random_git_name) + project.repository.add_branch(user, branch_name, 'master') + create_commit("Commit for ##{issue.iid}", issue.project, user, branch_name) + end + + def create_commit(message, project, user, branch_name) + filename = random_git_name + oldrev = project.repository.commit(branch_name).sha + + options = { + committer: project.repository.user_to_committer(user), + author: project.repository.user_to_committer(user), + commit: { message: message, branch: branch_name, update_ref: true }, + file: { content: "content", path: filename, update: false } + } + + commit_sha = Gitlab::Git::Blob.commit(project.repository, options) + project.repository.commit(commit_sha) + + GitPushService.new(project, + user, + oldrev: oldrev, + newrev: commit_sha, + ref: 'refs/heads/master').execute + end + + def create_merge_request_closing_issue(issue, message: nil, source_branch: nil) + if !source_branch || project.repository.commit(source_branch).blank? + source_branch = random_git_name + project.repository.add_branch(user, source_branch, 'master') + end + + sha = project.repository.commit_file(user, random_git_name, "content", "commit message", source_branch, false) + project.repository.commit(sha) + + opts = { + title: 'Awesome merge_request', + description: message || "Fixes #{issue.to_reference}", + source_branch: source_branch, + target_branch: 'master' + } + + MergeRequests::CreateService.new(project, user, opts).execute + end + + def merge_merge_requests_closing_issue(issue) + merge_requests = issue.closed_by_merge_requests + merge_requests.each { |merge_request| MergeRequests::MergeService.new(project, user).execute(merge_request) } + end + + def deploy_master(environment: 'production') + CreateDeploymentService.new(project, user, { + environment: environment, + ref: 'master', + tag: false, + sha: project.repository.commit('master').sha + }).execute + end +end + +RSpec.configure do |config| + config.include CycleAnalyticsHelpers +end diff --git a/spec/support/cycle_analytics_helpers/test_generation.rb b/spec/support/cycle_analytics_helpers/test_generation.rb new file mode 100644 index 00000000000..8e19a6c92e2 --- /dev/null +++ b/spec/support/cycle_analytics_helpers/test_generation.rb @@ -0,0 +1,161 @@ +# rubocop:disable Metrics/AbcSize + +# Note: The ABC size is large here because we have a method generating test cases with +# multiple nested contexts. This shouldn't count as a violation. + +module CycleAnalyticsHelpers + module TestGeneration + # Generate the most common set of specs that all cycle analytics phases need to have. + # + # Arguments: + # + # phase: Which phase are we testing? Will call `CycleAnalytics.new.send(phase)` for the final assertion + # data_fn: A function that returns a hash, constituting initial data for the test case + # start_time_conditions: An array of `conditions`. Each condition is an tuple of `condition_name` and `condition_fn`. `condition_fn` is called with + # `context` (no lexical scope, so need to do `context.create` for factories, for example) and `data` (from the `data_fn`). + # Each `condition_fn` is expected to implement a case which consitutes the start of the given cycle analytics phase. + # end_time_conditions: An array of `conditions`. Each condition is an tuple of `condition_name` and `condition_fn`. `condition_fn` is called with + # `context` (no lexical scope, so need to do `context.create` for factories, for example) and `data` (from the `data_fn`). + # Each `condition_fn` is expected to implement a case which consitutes the end of the given cycle analytics phase. + # before_end_fn: This function is run before calling the end time conditions. Used for setup that needs to be run between the start and end conditions. + # post_fn: Code that needs to be run after running the end time conditions. + + def generate_cycle_analytics_spec(phase:, data_fn:, start_time_conditions:, end_time_conditions:, before_end_fn: nil, post_fn: nil) + combinations_of_start_time_conditions = (1..start_time_conditions.size).flat_map { |size| start_time_conditions.combination(size).to_a } + combinations_of_end_time_conditions = (1..end_time_conditions.size).flat_map { |size| end_time_conditions.combination(size).to_a } + + scenarios = combinations_of_start_time_conditions.product(combinations_of_end_time_conditions) + scenarios.each do |start_time_conditions, end_time_conditions| + context "start condition: #{start_time_conditions.map(&:first).to_sentence}" do + context "end condition: #{end_time_conditions.map(&:first).to_sentence}" do + it "finds the median of available durations between the two conditions" do + time_differences = Array.new(5) do |index| + data = data_fn[self] + start_time = (index * 10).days.from_now + end_time = start_time + rand(1..5).days + + start_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(start_time) { condition_fn[self, data] } + end + + # Run `before_end_fn` at the midpoint between `start_time` and `end_time` + Timecop.freeze(start_time + (end_time - start_time) / 2) { before_end_fn[self, data] } if before_end_fn + + end_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(end_time) { condition_fn[self, data] } + end + + Timecop.freeze(end_time + 1.day) { post_fn[self, data] } if post_fn + + end_time - start_time + end + + median_time_difference = time_differences.sort[2] + expect(subject.send(phase)).to be_within(5).of(median_time_difference) + end + + context "when the data belongs to another project" do + let(:other_project) { create(:project) } + + it "returns nil" do + # Use a stub to "trick" the data/condition functions + # into using another project. This saves us from having to + # define separate data/condition functions for this particular + # test case. + allow(self).to receive(:project) { other_project } + + 5.times do + data = data_fn[self] + start_time = Time.now + end_time = rand(1..10).days.from_now + + start_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(start_time) { condition_fn[self, data] } + end + + end_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(end_time) { condition_fn[self, data] } + end + + Timecop.freeze(end_time + 1.day) { post_fn[self, data] } if post_fn + end + + # Turn off the stub before checking assertions + allow(self).to receive(:project).and_call_original + + expect(subject.send(phase)).to be_nil + end + end + + context "when the end condition happens before the start condition" do + it 'returns nil' do + data = data_fn[self] + start_time = Time.now + end_time = start_time + rand(1..5).days + + # Run `before_end_fn` at the midpoint between `start_time` and `end_time` + Timecop.freeze(start_time + (end_time - start_time) / 2) { before_end_fn[self, data] } if before_end_fn + + end_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(start_time) { condition_fn[self, data] } + end + + start_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(end_time) { condition_fn[self, data] } + end + + Timecop.freeze(end_time + 1.day) { post_fn[self, data] } if post_fn + + expect(subject.send(phase)).to be_nil + end + end + end + end + + context "start condition NOT PRESENT: #{start_time_conditions.map(&:first).to_sentence}" do + context "end condition: #{end_time_conditions.map(&:first).to_sentence}" do + it "returns nil" do + 5.times do + data = data_fn[self] + end_time = rand(1..10).days.from_now + + end_time_conditions.each_with_index do |(condition_name, condition_fn), index| + Timecop.freeze(end_time + index.days) { condition_fn[self, data] } + end + + Timecop.freeze(end_time + 1.day) { post_fn[self, data] } if post_fn + end + + expect(subject.send(phase)).to be_nil + end + end + end + + context "start condition: #{start_time_conditions.map(&:first).to_sentence}" do + context "end condition NOT PRESENT: #{end_time_conditions.map(&:first).to_sentence}" do + it "returns nil" do + 5.times do + data = data_fn[self] + start_time = Time.now + + start_time_conditions.each do |condition_name, condition_fn| + Timecop.freeze(start_time) { condition_fn[self, data] } + end + + post_fn[self, data] if post_fn + end + + expect(subject.send(phase)).to be_nil + end + end + end + end + + context "when none of the start / end conditions are matched" do + it "returns nil" do + expect(subject.send(phase)).to be_nil + end + end + end + end +end diff --git a/spec/support/git_helpers.rb b/spec/support/git_helpers.rb new file mode 100644 index 00000000000..93422390ef7 --- /dev/null +++ b/spec/support/git_helpers.rb @@ -0,0 +1,9 @@ +module GitHelpers + def random_git_name + "#{FFaker::Product.brand}-#{FFaker::Product.brand}-#{rand(1000)}" + end +end + +RSpec.configure do |config| + config.include GitHelpers +end diff --git a/spec/support/import_export/configuration_helper.rb b/spec/support/import_export/configuration_helper.rb new file mode 100644 index 00000000000..f752508d48c --- /dev/null +++ b/spec/support/import_export/configuration_helper.rb @@ -0,0 +1,29 @@ +module ConfigurationHelper + # Returns a list of models from hashes/arrays contained in +project_tree+ + def names_from_tree(project_tree) + project_tree.map do |branch_or_model| + branch_or_model = branch_or_model.to_s if branch_or_model.is_a?(Symbol) + + branch_or_model.is_a?(String) ? branch_or_model : names_from_tree(branch_or_model) + end + end + + def relation_class_for_name(relation_name) + relation_name = Gitlab::ImportExport::RelationFactory::OVERRIDES[relation_name.to_sym] || relation_name + relation_name.to_s.classify.constantize + end + + def parsed_attributes(relation_name, attributes) + excluded_attributes = config_hash['excluded_attributes'][relation_name] + included_attributes = config_hash['included_attributes'][relation_name] + + attributes = attributes - JSON[excluded_attributes.to_json] if excluded_attributes + attributes = attributes & JSON[included_attributes.to_json] if included_attributes + + attributes + end + + def associations_for(safe_model) + safe_model.reflect_on_all_associations.map { |assoc| assoc.name.to_s } + end +end diff --git a/spec/support/import_export/export_file_helper.rb b/spec/support/import_export/export_file_helper.rb new file mode 100644 index 00000000000..be0772d6a4a --- /dev/null +++ b/spec/support/import_export/export_file_helper.rb @@ -0,0 +1,133 @@ +require './spec/support/import_export/configuration_helper' + +module ExportFileHelper + include ConfigurationHelper + + ObjectWithParent = Struct.new(:object, :parent, :key_found) + + def setup_project + project = create(:project, :public) + + create(:release, project: project) + + issue = create(:issue, assignee: user, project: project) + snippet = create(:project_snippet, project: project) + label = create(:label, project: project) + milestone = create(:milestone, project: project) + merge_request = create(:merge_request, source_project: project, milestone: milestone) + commit_status = create(:commit_status, project: project) + + create(:label_link, label: label, target: issue) + + ci_pipeline = create(:ci_pipeline, + project: project, + sha: merge_request.diff_head_sha, + ref: merge_request.source_branch, + statuses: [commit_status]) + + create(:ci_build, pipeline: ci_pipeline, project: project) + create(:milestone, project: project) + create(:note, noteable: issue, project: project) + create(:note, noteable: merge_request, project: project) + create(:note, noteable: snippet, project: project) + create(:note_on_commit, + author: user, + project: project, + commit_id: ci_pipeline.sha) + + create(:event, target: milestone, project: project, action: Event::CREATED, author: user) + create(:project_member, :master, user: user, project: project) + create(:ci_variable, project: project) + create(:ci_trigger, project: project) + key = create(:deploy_key) + key.projects << project + create(:service, project: project) + create(:project_hook, project: project, token: 'token') + create(:protected_branch, project: project) + + project + end + + # Expands the compressed file for an exported project into +tmpdir+ + def in_directory_with_expanded_export(project) + Dir.mktmpdir do |tmpdir| + export_file = project.export_project_path + _output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}}) + + yield(exit_status, tmpdir) + end + end + + # Recursively finds key/values including +key+ as part of the key, inside a nested hash + def deep_find_with_parent(sensitive_key_word, object, found = nil) + sensitive_key_found = object_contains_key?(object, sensitive_key_word) + + # Returns the parent object and the object found containing a sensitive word as part of the key + if sensitive_key_found && object[sensitive_key_found] + ObjectWithParent.new(object[sensitive_key_found], object, sensitive_key_found) + elsif object.is_a?(Enumerable) + # Recursively lookup for keys containing sensitive words in a Hash or Array + object_with_parent = nil + + object.find do |*hash_or_array| + object_with_parent = deep_find_with_parent(sensitive_key_word, hash_or_array.last, found) + end + + object_with_parent + end + end + + # Return true if the hash has a key containing a sensitive word + def object_contains_key?(object, sensitive_key_word) + return false unless object.is_a?(Hash) + + object.keys.find { |key| key.include?(sensitive_key_word) } + end + + # Returns the offended ObjectWithParent object if a sensitive word is found inside a hash, + # excluding the whitelisted safe hashes. + def find_sensitive_attributes(sensitive_word, project_hash) + loop do + object_with_parent = deep_find_with_parent(sensitive_word, project_hash) + + return nil unless object_with_parent && object_with_parent.object + + if is_safe_hash?(object_with_parent.parent, sensitive_word) + # It's in the safe list, remove hash and keep looking + object_with_parent.parent.delete(object_with_parent.key_found) + else + return object_with_parent + end + + nil + end + end + + # Returns true if it's one of the excluded models in +safe_list+ + def is_safe_hash?(parent, sensitive_word) + return false unless parent && safe_list[sensitive_word.to_sym] + + # Extra attributes that appear in a model but not in the exported hash. + excluded_attributes = ['type'] + + safe_list[sensitive_word.to_sym].each do |model| + # Check whether this is a hash attribute inside a model + if model.is_a?(Symbol) + return true if (safe_hashes[model] - parent.keys).empty? + else + return true if safe_model?(model, excluded_attributes, parent) + end + end + + false + end + + # Compares model attributes with those those found in the hash + # and returns true if there is a match, ignoring some excluded attributes. + def safe_model?(model, excluded_attributes, parent) + excluded_attributes += associations_for(model) + parsed_model_attributes = parsed_attributes(model.name.underscore, model.attribute_names) + + (parsed_model_attributes - parent.keys - excluded_attributes).empty? + end +end |
